From 48e37bea3ea4e83ddab8227869bbe56b52d9957d Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 14 Jul 2021 16:50:28 -0700 Subject: [PATCH 001/176] Create a shadowed module when overriding in metaflow_custom (#609) When you override an existing module inside a metaflow custom package, we now create a module suffixed with '_orig' (so, for example, if you override 'metaflow.plugins.aws', you can access the original module as 'metaflow.plugins.aws._orig' --- metaflow/__init__.py | 54 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/metaflow/__init__.py b/metaflow/__init__.py index 6d29630c690..a221f3b8ba9 100644 --- a/metaflow/__init__.py +++ b/metaflow/__init__.py @@ -46,30 +46,82 @@ class and related decorators. import sys import types +if sys.version_info[0] >= 3 and sys.version_info[1] >= 4: + import importlib.util + from importlib.machinery import ModuleSpec +else: + # Something random so there is no syntax error + ModuleSpec = None class _LazyLoader(object): # This _LazyLoader implements the Importer Protocol defined in PEP 302 def __init__(self, handled): + # Modules directly loaded (this is either new modules or overrides of existing ones) self._handled = handled if handled else {} + # This is used to revert back to regular loading when trying to load + # the over-ridden module + self._tempexcluded = set() + def find_module(self, fullname, path=None): - if fullname in self._handled: + if fullname in self._tempexcluded: + return None + if fullname in self._handled or \ + (fullname.endswith('._orig') and fullname[:-6] in self._handled): return self return None def load_module(self, fullname): if fullname in sys.modules: return sys.modules[fullname] + if not self._can_handle_orig_module() and fullname.endswith('._orig'): + # We return a nicer error message + raise ImportError( + "Attempting to load '%s' -- loading shadowed modules in Metaflow " + "Custom is only supported in Python 3.4+" % fullname) to_import = self._handled.get(fullname, None) + # We see if we are shadowing an existing module and, if so, we + # will keep track of the module we are shadowing so that it + # may be loaded if needed. We basically will create a ._orig submodule + # of sorts. This functionality only works for Python 3.4+. For anything + # below this, we do not create the _orig module so loading it will + # result in ModuleNotFound + if self._can_handle_orig_module() and not fullname.endswith('._orig'): + try: + # We exclude this module temporarily from what we handle to + # revert back to the non-shadowing mode of import + self._tempexcluded.add(fullname) + spec = importlib.util.find_spec(fullname) + self._handled["%s._orig" % fullname] = spec + finally: + self._tempexcluded.remove(fullname) + if isinstance(to_import, str): to_import = importlib.import_module(to_import) sys.modules[fullname] = to_import elif isinstance(to_import, types.ModuleType): sys.modules[fullname] = to_import + elif self._can_handle_orig_module() and isinstance(to_import, ModuleSpec): + # This loads modules that end in _orig + m = importlib.util.module_from_spec(to_import) + to_import.loader.exec_module(m) + sys.modules[fullname] = m + elif to_import is None and fullname.endswith('._orig'): + # This happens when trying to access a shadowed ._orig module + # when actually, there is no shadowed module; print a nicer message + # Condition is a bit overkill and most likely only checking to_import + # would be OK. Being extra sure in case _LazyLoader is misused and + # a None value is passed in. + raise ImportError( + "Metaflow Custom shadowed module '%s' does not exist" % fullname) else: raise ImportError return sys.modules[fullname] + @staticmethod + def _can_handle_orig_module(): + return sys.version_info[0] >= 3 and sys.version_info[1] >= 4 + from .event_logger import EventLogger From c9acd1e509d7b2dee9c530cbfb7bb9e5ffac2a8c Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Fri, 16 Jul 2021 11:00:08 -0700 Subject: [PATCH 002/176] remove METAFLOW_ATTEMPT (#594) --- metaflow/runtime.py | 2 -- test/core/metaflow_test/formatter.py | 2 +- test/core/tests/tag_catch.py | 8 +++++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/metaflow/runtime.py b/metaflow/runtime.py index 62328556768..8b42e031519 100644 --- a/metaflow/runtime.py +++ b/metaflow/runtime.py @@ -947,8 +947,6 @@ def _launch(self): self.task.ubf_context) env.update(args.get_env()) env['PYTHONUNBUFFERED'] = 'x' - # the env vars are needed by the test framework, nothing else - env['_METAFLOW_ATTEMPT'] = str(self.task.retries) # NOTE bufsize=1 below enables line buffering which is required # by read_logline() below that relies on readline() not blocking # print('running', args) diff --git a/test/core/metaflow_test/formatter.py b/test/core/metaflow_test/formatter.py index f17d2483906..0acfeba60c3 100644 --- a/test/core/metaflow_test/formatter.py +++ b/test/core/metaflow_test/formatter.py @@ -79,7 +79,7 @@ def _flow_lines(self): tags.extend(tag.split('(')[0] for tag in step.tags) yield 0, '# -*- coding: utf-8 -*-' - yield 0, 'from metaflow import FlowSpec, step, Parameter, project, IncludeFile, JSONType' + yield 0, 'from metaflow import FlowSpec, step, Parameter, project, IncludeFile, JSONType, current' yield 0, 'from metaflow_test import assert_equals, '\ 'assert_exception, '\ 'ExpectationFailed, '\ diff --git a/test/core/tests/tag_catch.py b/test/core/tests/tag_catch.py index 851cee07aa6..c347c1eb822 100644 --- a/test/core/tests/tag_catch.py +++ b/test/core/tests/tag_catch.py @@ -1,5 +1,7 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag +from metaflow import current + class TagCatchTest(MetaflowTest): PRIORITY = 2 @@ -8,7 +10,7 @@ class TagCatchTest(MetaflowTest): @steps(0, ['start']) def step_start(self): import os, sys - self.test_attempt = int(os.environ['_METAFLOW_ATTEMPT']) + self.test_attempt = current.retry_count sys.stdout.write('stdout testing logs %d\n' % self.test_attempt) sys.stderr.write('stderr testing logs %d\n' % self.test_attempt) if self.test_attempt < 3: @@ -20,7 +22,7 @@ def step_start(self): @steps(0, ['foreach-split']) def step_split(self): import os - if os.environ['_METAFLOW_ATTEMPT'] == '2': + if current.retry_count == 2: self.this_is_split = True else: raise TestRetry() @@ -29,7 +31,7 @@ def step_split(self): @steps(0, ['join']) def step_join(self): import os - if os.environ['_METAFLOW_ATTEMPT'] == '2': + if current.retry_count == 2: self.test_attempt = inputs[0].test_attempt else: raise TestRetry() From 906b357c0e23acf933eac19595f3f1a6aac8a2c1 Mon Sep 17 00:00:00 2001 From: Romain Date: Fri, 16 Jul 2021 12:43:24 -0700 Subject: [PATCH 003/176] Allow Metaflow Custom packages to define additional suffixes to package (#614) * Allow Metaflow Custom packages to define additional suffixes to package This enables you to package non-python files in your metaflow_custom package and have them included when Metaflow creates the code package. * Addressed comment --- metaflow/package.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/metaflow/package.py b/metaflow/package.py index 79a579c9371..7887aff6d01 100644 --- a/metaflow/package.py +++ b/metaflow/package.py @@ -25,6 +25,10 @@ def __init__(self, flow, environment, echo, suffixes=DEFAULT_SUFFIXES_LIST): self.metaflow_custom_root = None else: self.metaflow_custom_root = os.path.dirname(metaflow_custom.__file__) + self.metaflow_custom_addl_suffixes = getattr( + metaflow_custom, + 'METAFLOW_CUSTOM_PACKAGE_SUFFIXES', + None) environment.init_environment(echo) for step in flow: @@ -34,7 +38,9 @@ def __init__(self, flow, environment, echo, suffixes=DEFAULT_SUFFIXES_LIST): environment) self.blob, self.sha = self._make() - def _walk(self, root, exclude_hidden=True): + def _walk(self, root, exclude_hidden=True, addl_suffixes=None): + if addl_suffixes is None: + addl_suffixes = [] root = to_unicode(root) # handle files/folder with non ascii chars prefixlen = len('%s/' % os.path.dirname(root)) for path, dirs, files in os.walk(root): @@ -46,7 +52,7 @@ def _walk(self, root, exclude_hidden=True): for fname in files: if fname[0] == '.': continue - if any(fname.endswith(suffix) for suffix in self.suffixes): + if any(fname.endswith(suffix) for suffix in self.suffixes + addl_suffixes): p = os.path.join(path, fname) yield p, p[prefixlen:] @@ -61,7 +67,10 @@ def path_tuples(self): yield path_tuple # Metaflow customization if any if self.metaflow_custom_root: - for path_tuple in self._walk(self.metaflow_custom_root, exclude_hidden=False): + for path_tuple in self._walk( + self.metaflow_custom_root, + exclude_hidden=False, + addl_suffixes=self.metaflow_custom_addl_suffixes): yield path_tuple # the package folders for environment for path_tuple in self.environment.add_to_package(): From d0d3d3db22fb6ecfd6212ba4cb1ef1ec56b73771 Mon Sep 17 00:00:00 2001 From: Romain Date: Sat, 17 Jul 2021 10:41:51 -0700 Subject: [PATCH 004/176] Add inputs to task_pre_step (#615) * Add inputs to task_pre_step * Addressed comments * Forgot to remove an import --- metaflow/decorators.py | 3 ++- metaflow/plugins/aws/batch/batch_decorator.py | 3 ++- .../plugins/aws/step_functions/step_functions_decorator.py | 5 +++-- metaflow/plugins/conda/conda_step_decorator.py | 3 ++- metaflow/plugins/timeout_decorator.py | 3 ++- metaflow/task.py | 3 ++- 6 files changed, 13 insertions(+), 7 deletions(-) diff --git a/metaflow/decorators.py b/metaflow/decorators.py index 3fb82c15cd0..9c13e6cf536 100644 --- a/metaflow/decorators.py +++ b/metaflow/decorators.py @@ -273,7 +273,8 @@ def task_pre_step(self, graph, retry_count, max_user_code_retries, - ubf_context): + ubf_context, + inputs): """ Run before the step function in the task context. """ diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index b1e9012377a..c2278e6d5ee 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -208,7 +208,8 @@ def task_pre_step(self, graph, retry_count, max_retries, - ubf_context): + ubf_context, + inputs): if metadata.TYPE == 'local': self.ds_root = ds.root else: diff --git a/metaflow/plugins/aws/step_functions/step_functions_decorator.py b/metaflow/plugins/aws/step_functions/step_functions_decorator.py index 224dfa4b9ef..9b31e62e37b 100644 --- a/metaflow/plugins/aws/step_functions/step_functions_decorator.py +++ b/metaflow/plugins/aws/step_functions/step_functions_decorator.py @@ -20,7 +20,8 @@ def task_pre_step(self, graph, retry_count, max_user_code_retries, - ubf_context): + ubf_context, + inputs): meta = {} meta['aws-step-functions-execution'] = os.environ['METAFLOW_RUN_ID'] meta['aws-step-functions-state-machine'] =\ @@ -89,4 +90,4 @@ def _ttl(self): delta = int(os.environ.get('METAFLOW_SFN_WORKFLOW_TIMEOUT', delta)) # Add 90 days since AWS Step Functions maintains execution history for # that long. - return delta + (90 * 24 * 60 * 60) + int(time.time()) \ No newline at end of file + return delta + (90 * 24 * 60 * 60) + int(time.time()) diff --git a/metaflow/plugins/conda/conda_step_decorator.py b/metaflow/plugins/conda/conda_step_decorator.py index 00dc215925b..5b68738c73c 100644 --- a/metaflow/plugins/conda/conda_step_decorator.py +++ b/metaflow/plugins/conda/conda_step_decorator.py @@ -282,7 +282,8 @@ def task_pre_step(self, graph, retry_count, max_retries, - ubf_context): + ubf_context, + inputs): if self.is_enabled(ubf_context): meta.register_metadata(run_id, step_name, task_id, [MetaDatum(field='conda_env_id', diff --git a/metaflow/plugins/timeout_decorator.py b/metaflow/plugins/timeout_decorator.py index 61ae8b8dbb2..4af34223836 100644 --- a/metaflow/plugins/timeout_decorator.py +++ b/metaflow/plugins/timeout_decorator.py @@ -71,7 +71,8 @@ def task_pre_step(self, graph, retry_count, max_user_code_retries, - ubf_context): + ubf_context, + inputs): if ubf_context != UBF_CONTROL and retry_count <= max_user_code_retries: # enable timeout only when executing user code self.step_name = step_name diff --git a/metaflow/task.py b/metaflow/task.py index 389aa7f8a30..5f911b13eb2 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -387,7 +387,8 @@ def run_step(self, self.flow._graph, retry_count, max_user_code_retries, - self.ubf_context) + self.ubf_context, + inputs) # decorators can actually decorate the step function, # or they can replace it altogether. This functionality From 54b10996db226e9acb267e9f4719a4fc45939277 Mon Sep 17 00:00:00 2001 From: Savin Date: Mon, 19 Jul 2021 13:24:04 -0700 Subject: [PATCH 005/176] Refactor @resources decorator (#617) * Refactor @resources decorator @resources decorator is shared by all compute related decorators - @batch, @lambda, @k8s, @titus. This patch moves it out of batch_decorator.py so that other decorators can cleanly reference it. * Update __init__.py --- metaflow/plugins/__init__.py | 3 +- metaflow/plugins/aws/batch/batch_decorator.py | 87 +++++++------------ metaflow/plugins/resources_decorator.py | 39 +++++++++ 3 files changed, 72 insertions(+), 57 deletions(-) create mode 100644 metaflow/plugins/resources_decorator.py diff --git a/metaflow/plugins/__init__.py b/metaflow/plugins/__init__.py index b386c2dd777..39face9e2b8 100644 --- a/metaflow/plugins/__init__.py +++ b/metaflow/plugins/__init__.py @@ -95,7 +95,8 @@ def _merge_lists(base, overrides, attr): from .timeout_decorator import TimeoutDecorator from .environment_decorator import EnvironmentDecorator from .retry_decorator import RetryDecorator -from .aws.batch.batch_decorator import BatchDecorator, ResourcesDecorator +from .resources_decorator import ResourcesDecorator +from .aws.batch.batch_decorator import BatchDecorator from .aws.step_functions.step_functions_decorator \ import StepFunctionsInternalDecorator from .test_unbounded_foreach_decorator\ diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index c2278e6d5ee..1be46d1ed72 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -10,6 +10,7 @@ from metaflow.datastore.util.s3util import get_s3_client from metaflow.decorators import StepDecorator from metaflow.metaflow_config import DATASTORE_LOCAL_DIR +from metaflow.plugins import ResourcesDecorator from metaflow.plugins.timeout_decorator import get_run_time_limit_for_task from metaflow.metadata import MetaDatum @@ -30,83 +31,57 @@ from urllib.parse import urlparse -class ResourcesDecorator(StepDecorator): - """ - Step decorator to specify the resources needed when executing this step. - This decorator passes this information along to Batch when requesting resources - to execute this step. - This decorator is ignored if the execution of the step does not happen on Batch. - To use, annotate your step as follows: - ``` - @resources(cpu=32) - @step - def myStep(self): - ... - ``` - Parameters - ---------- - cpu : int - Number of CPUs required for this step. Defaults to 1 - gpu : int - Number of GPUs required for this step. Defaults to 0 - memory : int - Memory size (in MB) required for this step. Defaults to 4096 - shared_memory : int - The value for the size (in MiB) of the /dev/shm volume for this step. - This parameter maps to the --shm-size option to docker run . - """ - name = 'resources' - defaults = { - 'cpu': '1', - 'gpu': '0', - 'memory': '4096', - 'shared_memory': None - } - class BatchDecorator(StepDecorator): """ - Step decorator to specify that this step should execute on Batch. - This decorator indicates that your step should execute on Batch. Note that you can - apply this decorator automatically to all steps using the ```--with batch``` argument - when calling run. Step level decorators are overrides and will force a step to execute - on Batch regardless of the ```--with``` specification. + Step decorator to specify that this step should execute on AWS Batch. + + This decorator indicates that your step should execute on AWS Batch. Note + that you can apply this decorator automatically to all steps using the + ```--with batch``` argument when calling run/resume. Step level decorators + within the code are overrides and will force a step to execute on AWS Batch + regardless of the ```--with``` specification. + To use, annotate your step as follows: ``` @batch @step - def myStep(self): + def my_step(self): ... ``` Parameters ---------- cpu : int - Number of CPUs required for this step. Defaults to 1. If @resources is also - present, the maximum value from all decorators is used + Number of CPUs required for this step. Defaults to 1. If @resources is + also present, the maximum value from all decorators is used gpu : int - Number of GPUs required for this step. Defaults to 0. If @resources is also - present, the maximum value from all decorators is used - memory : int - Memory size (in MB) required for this step. Defaults to 4096. If @resources is + Number of GPUs required for this step. Defaults to 0. If @resources is also present, the maximum value from all decorators is used + memory : int + Memory size (in MB) required for this step. Defaults to 4096. If + @resources is also present, the maximum value from all decorators is + used image : string - Image to use when launching on AWS Batch. If not specified, a default image mapping to - the current version of Python is used + Docker image to use when launching on AWS Batch. If not specified, a + default docker image mapping to the current version of Python is used queue : string - Queue to submit the job to. Defaults to the one determined by the environment variable - METAFLOW_BATCH_JOB_QUEUE + AWS Batch Job Queue to submit the job to. Defaults to the one + specified by the environment variable METAFLOW_BATCH_JOB_QUEUE iam_role : string - IAM role that AWS Batch can use to access Amazon S3. Defaults to the one determined by the environment - variable METAFLOW_ECS_S3_ACCESS_IAM_ROLE + AWS IAM role that AWS Batch container uses to access AWS cloud resources + (Amazon S3, Amazon DynamoDb, etc). Defaults to the one specified by the + environment variable METAFLOW_ECS_S3_ACCESS_IAM_ROLE execution_role : string - IAM role that AWS Batch can use to trigger AWS Fargate tasks. Defaults to the one determined by the environment - variable METAFLOW_ECS_FARGATE_EXECUTION_ROLE https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html + AWS IAM role that AWS Batch can use to trigger AWS Fargate tasks. + Defaults to the one determined by the environment variable + METAFLOW_ECS_FARGATE_EXECUTION_ROLE https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html shared_memory : int The value for the size (in MiB) of the /dev/shm volume for this step. This parameter maps to the --shm-size option to docker run. max_swap : int - The total amount of swap memory (in MiB) a container can use for this step. - This parameter is translated to the --memory-swap option to docker run - where the value is the sum of the container memory plus the max_swap value. + The total amount of swap memory (in MiB) a container can use for this + step. This parameter is translated to the --memory-swap option to + docker run where the value is the sum of the container memory plus the + max_swap value. swappiness : int This allows you to tune memory swappiness behavior for this step. A swappiness value of 0 causes swapping not to happen unless absolutely diff --git a/metaflow/plugins/resources_decorator.py b/metaflow/plugins/resources_decorator.py new file mode 100644 index 00000000000..38288a4ce66 --- /dev/null +++ b/metaflow/plugins/resources_decorator.py @@ -0,0 +1,39 @@ +from metaflow.decorators import StepDecorator + + +class ResourcesDecorator(StepDecorator): + """ + Step decorator to specify the resources needed when executing this step. + + This decorator passes this information along to container orchestrator + (AWS Batch, Kubernetes, etc.) when requesting resources to execute this + step. + + This decorator is ignored if the execution of the step happens locally. + + To use, annotate your step as follows: + ``` + @resources(cpu=32) + @step + def my_step(self): + ... + ``` + Parameters + ---------- + cpu : int + Number of CPUs required for this step. Defaults to 1 + gpu : int + Number of GPUs required for this step. Defaults to 0 + memory : int + Memory size (in MB) required for this step. Defaults to 4096 + shared_memory : int + The value for the size (in MiB) of the /dev/shm volume for this step. + This parameter maps to the --shm-size option to docker run . + """ + name = 'resources' + defaults = { + 'cpu': '1', + 'gpu': '0', + 'memory': '4096', + 'shared_memory': None + } \ No newline at end of file From a11f7ea32c2b0f0e5ad8fccad150c62f2906f335 Mon Sep 17 00:00:00 2001 From: Romain Date: Tue, 20 Jul 2021 09:38:22 -0700 Subject: [PATCH 006/176] Add an option to define your own AWS client provider (#620) You can now specify a function that returns an AWS client. This is useful if you want to use something other than boto3 and can be used with the metaflow_custom mechanism to provide your own authentication --- metaflow/metaflow_config.py | 1 + metaflow/plugins/__init__.py | 56 +++++++++++++------- metaflow/plugins/aws/aws_client.py | 84 ++++++++++++++++++------------ 3 files changed, 91 insertions(+), 50 deletions(-) diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py index e9a14a3ff82..1dd233c0ed0 100644 --- a/metaflow/metaflow_config.py +++ b/metaflow/metaflow_config.py @@ -48,6 +48,7 @@ def from_conf(name, default=None): DEFAULT_METADATA = from_conf('METAFLOW_DEFAULT_METADATA', 'local') DEFAULT_MONITOR = from_conf('METAFLOW_DEFAULT_MONITOR', 'nullSidecarMonitor') DEFAULT_PACKAGE_SUFFIXES = from_conf('METAFLOW_DEFAULT_PACKAGE_SUFFIXES', '.py,.R,.RDS') +DEFAULT_AWS_CLIENT_PROVIDER = from_conf('METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER', 'boto3') ### diff --git a/metaflow/plugins/__init__.py b/metaflow/plugins/__init__.py index 39face9e2b8..34912372e79 100644 --- a/metaflow/plugins/__init__.py +++ b/metaflow/plugins/__init__.py @@ -1,6 +1,18 @@ import sys import types +_expected_extensions = { + 'FLOW_DECORATORS': [], + 'STEP_DECORATORS': [], + 'ENVIRONMENTS': [], + 'METADATA_PROVIDERS': [], + 'SIDECARS': {}, + 'LOGGING_SIDECARS': {}, + 'MONITOR_SIDECARS': {}, + 'AWS_CLIENT_PROVIDERS': [], + 'get_plugin_cli': lambda : [] +} + try: import metaflow_custom.plugins as _ext_plugins except ImportError as e: @@ -16,20 +28,12 @@ "if you want to ignore, uninstall metaflow_custom package") raise class _fake(object): - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - - def get_plugin_cli(self): - return [] - - _ext_plugins = _fake( - FLOW_DECORATORS=[], - STEP_DECORATORS=[], - ENVIRONMENTS=[], - METADATA_PROVIDERS=[], - SIDECARS={}, - LOGGING_SIDECARS={}, - MONITOR_SIDECARS={}) + def __getattr__(self, name): + if name in _expected_extensions: + return _expected_extensions[name] + raise AttributeError + + _ext_plugins = _fake() else: # We load into globals whatever we have in extension_module # We specifically exclude any modules that may be included (like sys, os, etc) @@ -61,6 +65,18 @@ def get_plugin_cli(self): # This keeps it cleaner. from metaflow import _LazyLoader sys.meta_path = [_LazyLoader(lazy_load_custom_modules)] + sys.meta_path + + class _wrap(object): + def __init__(self, obj): + self.__dict__ = obj.__dict__ + + def __getattr__(self, name): + if name in _expected_extensions: + return _expected_extensions[name] + raise AttributeError + + _ext_plugins = _wrap(_ext_plugins) + def get_plugin_cli(): @@ -160,11 +176,15 @@ def _merge_lists(base, overrides, attr): SIDECARS.update(LOGGING_SIDECARS) SIDECARS.update(MONITOR_SIDECARS) +from .aws.aws_client import Boto3ClientProvider +AWS_CLIENT_PROVIDERS = _merge_lists( + [Boto3ClientProvider], _ext_plugins.AWS_CLIENT_PROVIDERS, 'name') + # Erase all temporary names to avoid leaking things -# We leave '_ext_plugins' because it is used in a function (so it needs -# to stick around) -for _n in ['ver', 'n', 'o', 'e', 'lazy_load_custom_modules', - '_LazyLoader', '_merge_lists', '_fake', 'addl_modules']: +# We leave '_ext_plugins' and '_expected_extensions' because they are used in +# a function (so they need to stick around) +for _n in ['ver', 'n', 'o', 'e', 'lazy_load_custom_modules', '_LazyLoader', + '_merge_lists', '_fake', '_wrap', 'addl_modules']: try: del globals()[_n] except KeyError: diff --git a/metaflow/plugins/aws/aws_client.py b/metaflow/plugins/aws/aws_client.py index 75723f2ed20..d3f4ac7eaed 100644 --- a/metaflow/plugins/aws/aws_client.py +++ b/metaflow/plugins/aws/aws_client.py @@ -1,37 +1,57 @@ cached_aws_sandbox_creds = None +cached_provider_class = None -def get_aws_client(module, with_error=False, params={}): - from metaflow.exception import MetaflowException - from metaflow.metaflow_config import AWS_SANDBOX_ENABLED, \ - AWS_SANDBOX_STS_ENDPOINT_URL, AWS_SANDBOX_API_KEY - import requests - try: - import boto3 - from botocore.exceptions import ClientError - except (NameError, ImportError): - raise MetaflowException( - "Could not import module 'boto3'. Install boto3 first.") +class Boto3ClientProvider(object): + name = "boto3" - if AWS_SANDBOX_ENABLED: - global cached_aws_sandbox_creds - if cached_aws_sandbox_creds is None: - # authenticate using STS - url = "%s/auth/token" % AWS_SANDBOX_STS_ENDPOINT_URL - headers = { - 'x-api-key': AWS_SANDBOX_API_KEY - } - try: - r = requests.get(url, headers=headers) - r.raise_for_status() - cached_aws_sandbox_creds = r.json() - except requests.exceptions.HTTPError as e: - raise MetaflowException(repr(e)) - if with_error: + @staticmethod + def get_client(module, with_error=False, params={}): + from metaflow.exception import MetaflowException + from metaflow.metaflow_config import AWS_SANDBOX_ENABLED, \ + AWS_SANDBOX_STS_ENDPOINT_URL, AWS_SANDBOX_API_KEY + import requests + try: + import boto3 + from botocore.exceptions import ClientError + except (NameError, ImportError): + raise MetaflowException( + "Could not import module 'boto3'. Install boto3 first.") + + if AWS_SANDBOX_ENABLED: + global cached_aws_sandbox_creds + if cached_aws_sandbox_creds is None: + # authenticate using STS + url = "%s/auth/token" % AWS_SANDBOX_STS_ENDPOINT_URL + headers = { + 'x-api-key': AWS_SANDBOX_API_KEY + } + try: + r = requests.get(url, headers=headers) + r.raise_for_status() + cached_aws_sandbox_creds = r.json() + except requests.exceptions.HTTPError as e: + raise MetaflowException(repr(e)) + if with_error: + return boto3.session.Session( + **cached_aws_sandbox_creds).client(module, **params), ClientError return boto3.session.Session( - **cached_aws_sandbox_creds).client(module, **params), ClientError - return boto3.session.Session( - **cached_aws_sandbox_creds).client(module, **params) - if with_error: - return boto3.client(module, **params), ClientError - return boto3.client(module, **params) + **cached_aws_sandbox_creds).client(module, **params) + if with_error: + return boto3.client(module, **params), ClientError + return boto3.client(module, **params) + + +def get_aws_client(module, with_error=False, params={}): + global cached_provider_class + if cached_provider_class is None: + from metaflow.metaflow_config import DEFAULT_AWS_CLIENT_PROVIDER + from metaflow.plugins import AWS_CLIENT_PROVIDERS + for p in AWS_CLIENT_PROVIDERS: + if p.name == DEFAULT_AWS_CLIENT_PROVIDER: + cached_provider_class = p + break + else: + raise ValueError("Cannot find AWS Client provider %s" + % DEFAULT_AWS_CLIENT_PROVIDER) + return cached_provider_class.get_client(module, with_error, params) From c1e465bf36b5154e73b1278cc328285610e62428 Mon Sep 17 00:00:00 2001 From: Romain Date: Fri, 23 Jul 2021 16:55:21 -0700 Subject: [PATCH 007/176] Add S3 tests (#613) * Add S3 tests * Addressed comments --- test/README.md | 42 +++ test/data/__init__.py | 17 + test/data/s3/__init__.py | 2 + test/data/s3/s3_data.py | 452 +++++++++++++++++++++++ test/data/s3/test_s3.py | 759 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 1272 insertions(+) create mode 100644 test/data/__init__.py create mode 100644 test/data/s3/__init__.py create mode 100644 test/data/s3/s3_data.py create mode 100644 test/data/s3/test_s3.py diff --git a/test/README.md b/test/README.md index 5906965a604..217910e6ba5 100644 --- a/test/README.md +++ b/test/README.md @@ -1,5 +1,47 @@ # Metaflow Test Suite +Metaflow test suite consists of two parts: + + 1. A data test suite for the data layer components (`S3`) based on + [Pytest](http://pytest.org). These tests can be found + under the `test/data` directory. + 2. An integration test harness for the core Metaflow at `test/core`. + The harness generates and executes synthetic Metaflow flows, + exercising all aspects of Metaflow. + +You can run the tests by hand using `pytest` or `run_tests.py` as described +below. + +## Data Test Suite + +The data tests are standard `pytest` suites. In the `s3` folder, you will +find two files: `s3_data.py` which generates synthetic +data and `test_s3.py` which contains the actual tests. + +The test data is cached in S3. If you change anything in the `s3_data.py` +module (or you have another reason for wanting to regenerate test +data), you can regenerate the data easily by changing the S3 test prefix +at `test/data/__init__.py`. The `s3_data.py` detects that data +is missing in S3 and they will upload the data in the new location +automatically. + +### Running data tests by hand + +You can run the data tests using `pytest` as follows: + +``` +cd test/data/ +PYTHONPATH=`pwd`/../../ python3 -m pytest -x -s -v --benchmark-skip +``` + +You can obviously also not skip the benchmarks but be aware that the +benchmarks run for a long time. + +Both Python2 and Python3 are supported. See `python -m pytest --help` +for more information about how to execute `pytest` tests. + +## The Integration Test Harness for Metaflow + The integration test harness for the core Metaflow at `test/core` generates and executes synthetic Metaflow flows, exercising all aspects of Metaflow. The test suite is executed using diff --git a/test/data/__init__.py b/test/data/__init__.py new file mode 100644 index 00000000000..e4f0ca86928 --- /dev/null +++ b/test/data/__init__.py @@ -0,0 +1,17 @@ +import os + +# Can set a default path here. Note that you can update the path +# if you want a fresh set of data +S3ROOT = os.environ.get('METAFLOW_S3_TEST_ROOT') + +from metaflow.datatools.s3util import get_s3_client +s3client, _ = get_s3_client() + +from metaflow import FlowSpec + +# ast parsing in metaflow.graph doesn't like this class +# to be defined in test_s3.py. Defining it here works. +class FakeFlow(FlowSpec): + pass + + diff --git a/test/data/s3/__init__.py b/test/data/s3/__init__.py new file mode 100644 index 00000000000..b2b70f45f2b --- /dev/null +++ b/test/data/s3/__init__.py @@ -0,0 +1,2 @@ +# nothing here + diff --git a/test/data/s3/s3_data.py b/test/data/s3/s3_data.py new file mode 100644 index 00000000000..74009d53f3c --- /dev/null +++ b/test/data/s3/s3_data.py @@ -0,0 +1,452 @@ +import os +import sys +import zlib +import json +from uuid import uuid4 +from hashlib import sha1 +from collections import namedtuple + +try: + # python2 + from urlparse import urlparse +except: + # python3 + from urllib.parse import urlparse + +from metaflow.datatools.s3 import S3PutObject + +from metaflow.util import to_fileobj, to_bytes, url_quote + +import numpy + +from .. import s3client, S3ROOT + +BASIC_METADATA = { + 'no_meta': (None, None), # No metadata at all but going through the calls + 'content_no_meta': ('text/plain', None), # Content-type but no metadata + 'no_content_meta': (None, {'userkey': 'UserValue'}), # No content-type but metadata + 'isolation': ('text/plain', {'content-type': 'text/css'}), # Check isolation of user metadata + 'multiple': ('text/plain', { + 'userkey1': 'UserValue1', 'userkey2': 'UserValue2'}), # Multiple metadata + 'complex': ('text/plain', { + 'utf8-data': u'\u523a\u8eab/means sashimi', + 'with-weird-chars': 'Space and !@#<>:/-+=&%'}), +} + +BASIC_RANGE_INFO = { + 'from_beg': (0, 16), # From beginning + 'exceed_end': (0, 10 * 1024**3), # From beginning, should fetch full file + 'middle': (5, 10), # From middle + 'end': (None, -5), # Fetch from end + 'till_end': (5, None), # Fetch till end +} + +# None for file size denotes missing keys +# To properly support ranges in a useful manner, make files at least 32 bytes +# long +BASIC_DATA = [ + # empty prefixes should not be a problem + ('empty_prefix', {}), + # requesting non-existent data should be handled ok + ('missing_files', {'missing': None}), + # a basic sanity check + ('3_small_files', {'empty_file': 0, + 'kb_file': 1024, + 'mb_file': 1024**2, + 'missing_file': None}), + # S3 paths can be longer than the max allowed filename on Linux + ('long_path', {'/'.join('x' * 300): 1024, + # one medium-size path for list_path test + '/'.join('y' * 10): 32, + 'x/x/x': None}), + # test that nested prefixes work correctly + ('prefix', {'prefix': 32, + 'prefixprefix': 33, + # note that prefix/prefix is both an object and a prefix + 'prefix/prefix': 34, + 'prefix/prefix/prefix': None}), + # same filename as above but a different prefix + ('samefile', {'prefix': 42, 'x': 43, 'empty_file': 1, 'xx': None}), + # crazy file names (it seems '#' characters don't work with boto) + ('crazypath', {u'crazy spaces': 34, + u'\x01\xff': 64, + u'\u523a\u8eab/means sashimi': 33, + u'crazy-!.$%@2_()"\'': 100, + u' /cra._:zy/\x01\x02/p a t h/$this/!!is()': 1000, + u'crazy missing :(': None}) +] + +BIG_DATA = [ + # test a file > 4GB + ('5gb_file', {'5gb_file': 5 * 1024**3}), + # ensure that e.g. paged listings work correctly with many keys + ('3000_files', {str(i): i for i in range(3000)}) +] + +# Large file to use for benchmark, must be in BASIC_DATA or BIG_DATA +BENCHMARK_SMALL_FILE = ('3000_files', {'1': 1}) +BENCHMARK_MEDIUM_FILE = ('3_small_files', {'mb_file': 1024**2}) +BENCHMARK_LARGE_FILE = ('5gb_file', {'5gb_file': 5 * 1024**3}) + +BENCHMARK_SMALL_ITER_MAX = 10001 +BENCHMARK_MEDIUM_ITER_MAX = 501 +BENCHMARK_LARGE_ITER_MAX = 11 + +FAKE_RUN_DATA = [ + # test a run id - just a random run id + ('HelloFlow/56', {'one_a': 512, 'one_b': 1024, 'two_c': 8192}) +] + +PUT_PREFIX = 'put_tests' + +ExpectedResult = namedtuple( + 'ExpectedResult', 'size checksum content_type metadata range') + +class RandomFile(object): + + cached_digests = {} + cached_files = {} + + def __init__(self, prefix, fname, size): + self.key = os.path.join(prefix, fname) + self.prefix = prefix + self.fname = fname + self.size = size + self._data = None + + def _make_data(self): + numpy.random.seed(zlib.adler32(self.key.encode('utf-8')) & 0xffffffff) + self._data = numpy.random.bytes(self.size) + + def checksum(self, start=None, length=None): + if self.size is not None: + start = start if start else 0 + length = length if length and start + length < self.size else self.size + lookup_key = "%s:%d:%d" % (self.key, start, length) + if lookup_key not in self.cached_digests: + if self._data is None: + self._make_data() + if length < 0: + self.cached_digests[lookup_key] =\ + sha1(self._data[length:]).hexdigest() + else: + self.cached_digests[lookup_key] =\ + sha1(self._data[start:start+length]).hexdigest() + return self.cached_digests[lookup_key] + + def size_from_range(self, start, length): + if self.size is None: + return None + if length: + if length > 0: + end = length + start + else: + assert start is None + start = self.size + length + end = self.size + else: + end = self.size + + if end > self.size: + end = self.size + return end - start + + def fileobj(self): + if self.size is not None: + return to_fileobj(self.data) + + @property + def data(self): + if self._data is None and self.size is not None: + self._make_data() + return self._data + + @property + def url(self): + return os.path.join(S3ROOT, self.key) + +def _format_test_cases(dataset, meta=None, ranges=None): + cases = [] + ids = [] + for prefix, filespecs in dataset: + objs = [RandomFile(prefix, fname, size) + for fname, size in filespecs.items()] + objs = {obj.url: (obj, None, None) for obj in objs} + if meta: + # We generate one per meta info + for metaname, (content_type, usermeta) in meta.items(): + objs.update({"%s_%s" % (obj.url, metaname): \ + (obj, content_type, usermeta) for (obj, _, _) in objs.values()}) + files = { + k: {None: ExpectedResult( + size=obj.size, + checksum=obj.checksum(), + content_type=content_type, + metadata=usermeta, + range=None)} for k, (obj, content_type, usermeta) in objs.items()} + if ranges: + # For every file we have in files, we calculate the proper + # checksum and create a new dictionary + for k, (obj, content_type, usermeta) in objs.items(): + for offset, length in ranges.values(): + files[k][(offset, length)] = ExpectedResult( + size=obj.size_from_range(offset, length), + checksum=obj.checksum(offset, length), + content_type=content_type, + metadata=usermeta, + range=(offset, length)) + + ids.append(prefix) + cases.append((S3ROOT, [prefix], files)) + return cases, ids + +def pytest_fakerun_cases(): + cases, ids = _format_test_cases(FAKE_RUN_DATA) + return {'argvalues': cases, 'ids': ids} + +def pytest_basic_case(): + cases, ids = _format_test_cases( + BASIC_DATA, ranges=BASIC_RANGE_INFO, meta=BASIC_METADATA) + return {'argvalues': cases, 'ids': ids} + +def pytest_large_case(): + cases, ids = _format_test_cases(BASIC_DATA, meta=BASIC_METADATA) + cases_big, ids_big = _format_test_cases(BIG_DATA) + cases.extend(cases_big) + ids.extend(ids_big) + return {'argvalues': cases, 'ids': ids} + +def pytest_benchmark_case(): + cases, _ = _format_test_cases([BENCHMARK_LARGE_FILE]) + ids = ['5gb'] + + new_cases, _ = _format_test_cases([BENCHMARK_MEDIUM_FILE]) + cases.extend(new_cases) + ids.append('1mb') + + new_cases, _ = _format_test_cases([BENCHMARK_SMALL_FILE]) + cases.extend(new_cases) + ids.append('1b') + + return {'argvalues': cases, 'ids': ids} + +def pytest_benchmark_many_case(): + large_case = _format_test_cases([BENCHMARK_LARGE_FILE])[0][0] + medium_case = _format_test_cases([BENCHMARK_MEDIUM_FILE])[0][0] + small_case = _format_test_cases([BENCHMARK_SMALL_FILE])[0][0] + + # Configuration: we will form groups of up to BENCHMARK_*_ITER_MAX items + # (count taken from iteration_count). We will also form groups taking from + # all three sets + cases = [] + ids = [] + iteration_count = [0, 1, 10, 50, 500, 10000] + for small_count in iteration_count: + if small_count > BENCHMARK_SMALL_ITER_MAX: + break + for medium_count in iteration_count: + if medium_count > BENCHMARK_MEDIUM_ITER_MAX: + break + for large_count in iteration_count: + if large_count > BENCHMARK_LARGE_ITER_MAX: + break + if small_count + medium_count + large_count == 0: + continue + # At this point, form the test + id_name = "%ds_%dm_%dl" % (small_count, medium_count, large_count) + cases.append(( + S3ROOT, [], + [(small_count, small_case[2]), + (medium_count, medium_case[2]), + (large_count, large_case[2])])) + ids.append(id_name) + return {'argvalues': cases, 'ids': ids} + +def pytest_benchmark_put_case(): + put_prefix = os.path.join(S3ROOT, PUT_PREFIX) + cases = [] + ids = [] + for prefix, filespecs in \ + [BENCHMARK_LARGE_FILE, BENCHMARK_MEDIUM_FILE, BENCHMARK_SMALL_FILE]: + blobs = [] + for fname, size in filespecs.items(): + blobs.append((prefix, fname, size)) + cases.append((put_prefix, blobs, None)) + ids = ['5gb', '1mb', '1b'] + return {'argvalues': cases, 'ids': ids} + +def pytest_benchmark_put_many_case(): + single_cases_and_ids = pytest_benchmark_put_case() + single_cases = single_cases_and_ids['argvalues'] + large_blob = single_cases[0][1][0] + medium_blob = single_cases[1][1][0] + small_blob = single_cases[2][1][0] + put_prefix = os.path.join(S3ROOT, PUT_PREFIX) + # Configuration: we will form groups of up to BENCHMARK_*_ITER_MAX items + # (count taken from iteration_count). We will also form groups taking from + # all three sets + cases = [] + ids = [] + iteration_count = [0, 1, 10, 50, 500, 10000] + for small_count in iteration_count: + if small_count > BENCHMARK_SMALL_ITER_MAX: + break + for medium_count in iteration_count: + if medium_count > BENCHMARK_MEDIUM_ITER_MAX: + break + for large_count in iteration_count: + if large_count > BENCHMARK_LARGE_ITER_MAX: + break + if small_count + medium_count + large_count == 0: + continue + # At this point, form the test + id_name = "%ds_%dm_%dl" % (small_count, medium_count, large_count) + blobs = [ + (small_count, small_blob), (medium_count, medium_blob), + (large_count, large_blob)] + cases.append((put_prefix, blobs, None)) + ids.append(id_name) + return {'argvalues': cases, 'ids': ids} + +def pytest_many_prefixes_case(): + cases, ids = _format_test_cases(BASIC_DATA, meta=BASIC_METADATA) + many_prefixes = [] + many_prefixes_expected = {} + for s3root, [prefix], files in cases: + many_prefixes.append(prefix) + many_prefixes_expected.update(files) + # add many prefixes cases + ids.append('many_prefixes') + cases.append((S3ROOT, many_prefixes, many_prefixes_expected)) + return {'argvalues': cases, 'ids': ids} + +def pytest_put_strings_case(meta=None): + put_prefix = os.path.join(S3ROOT, PUT_PREFIX) + data = [u"unicode: \u523a\u8eab means sashimi", + b"bytes: \x00\x01\x02", + "just a string"] + expected = {} + objs = [] + for text in data: + blob = to_bytes(text) + checksum = sha1(blob).hexdigest() + key = str(uuid4()) + expected[os.path.join(put_prefix, key)] = {None: ExpectedResult( + size=len(blob), + checksum=checksum, + content_type=None, + metadata=None, + range=None)} + objs.append((key, text)) + if meta is not None: + for content_type, usermeta in meta.values(): + key = str(uuid4()) + expected[os.path.join(put_prefix, key)] = {None: ExpectedResult( + size=len(blob), + checksum=checksum, + content_type=content_type, + metadata=usermeta, + range=None)} + objs.append(S3PutObject( + key=key, + value=text, + content_type=content_type, + metadata=usermeta)) + return {'argvalues': [(put_prefix, objs, expected)], + 'ids': ['put_strings']} + +def pytest_put_blobs_case(meta=None): + put_prefix = os.path.join(S3ROOT, PUT_PREFIX) + cases = [] + ids = [] + for prefix, filespecs in BIG_DATA: + expected = {} + blobs = [] + for fname, size in filespecs.items(): + blob = RandomFile(prefix, fname, size) + checksum = blob.checksum() + key = str(uuid4()) + expected[os.path.join(put_prefix, key)] = {None: ExpectedResult( + size=blob.size, + checksum=checksum, + content_type=None, + metadata=None, + range=None)} + blobs.append((key, blob.data)) + if meta is not None: + for content_type, usermeta in meta.values(): + key = str(uuid4()) + expected[os.path.join(put_prefix, key)] = {None: ExpectedResult( + size=len(blob), + checksum=checksum, + content_type=content_type, + metadata=usermeta, + range=None)} + blobs.append(S3PutObject( + key=key, + value=blob.data, + content_type=content_type, + metadata=usermeta)) + ids.append(prefix) + cases.append((put_prefix, blobs, expected)) + return {'argvalues': cases, 'ids': ids} + +def ensure_test_data(): + # update S3ROOT in __init__.py to get a fresh set of data + print('Ensuring that test data exists at %s' % S3ROOT) + mark = urlparse(os.path.join(S3ROOT, 'ALL_OK')) + try: + # Check if the data exists and has been modified in the last + # 29 days (this should be lower than the TTL for your bucket to ensure + # the data is available for the test) + import datetime + today = datetime.date.today() + delta = datetime.timedelta(days=29) + s3client.head_object(Bucket=mark.netloc, Key=mark.path.lstrip('/'), + IfModifiedSince=str(today - delta)) + print('All data ok.') + except: + print('Uploading test data') + def _do_upload(prefix, filespecs, meta=None): + for fname, size in filespecs.items(): + if size is not None: + f = RandomFile(prefix, fname, size) + url = urlparse(f.url) + # For metadata, we don't actually touch RandomFile + # (since it is the same) but we modify the path to post-pend + # the name + print('Case %s: %s started' % (prefix, f.url)) + s3client.upload_fileobj(f.fileobj(), + url.netloc, + url.path.lstrip('/')) + print('Case %s: %s added' % (prefix, f.url)) + if meta is not None: + for metaname, metainfo in meta.items(): + new_url = "%s_%s" % (f.url, metaname) + url = urlparse(new_url) + print('Case %s: %s started' % (prefix, new_url)) + extra = {} + content_type, user_meta = metainfo + if content_type: + extra['ContentType'] = content_type + if user_meta: + new_meta = { + 'metaflow-user-attributes': json.dumps(user_meta)} + extra['Metadata'] = new_meta + s3client.upload_fileobj(f.fileobj(), + url.netloc, + url.path.lstrip('/'), + ExtraArgs=extra) + print('Case %s: %s added' % (prefix, new_url)) + + for prefix, filespecs in BIG_DATA + FAKE_RUN_DATA: + _do_upload(prefix, filespecs) + for prefix, filespecs in BASIC_DATA: + _do_upload(prefix, filespecs, meta=BASIC_METADATA) + + s3client.upload_fileobj(to_fileobj('ok'), + Bucket=mark.netloc, + Key=mark.path.lstrip('/')) + print('Test data uploaded ok') + +ensure_test_data() diff --git a/test/data/s3/test_s3.py b/test/data/s3/test_s3.py new file mode 100644 index 00000000000..263ec63bf1c --- /dev/null +++ b/test/data/s3/test_s3.py @@ -0,0 +1,759 @@ +import os +from re import I +import shutil +from hashlib import sha1 +from tempfile import mkdtemp +from itertools import groupby +import random +from uuid import uuid4 + +import pytest + +from metaflow import current, namespace, Run +from metaflow.datatools.s3 import S3,\ + MetaflowS3AccessDenied,\ + MetaflowS3NotFound,\ + MetaflowS3URLException,\ + MetaflowS3InvalidObject,\ + S3PutObject + +from metaflow.util import to_bytes, unicode_type + +from . import s3_data +from .. import FakeFlow + +try: + # python2 + from urlparse import urlparse +except: + # python3 + from urllib.parse import urlparse + +def assert_results(s3objs, expected, info_should_be_empty=False, info_only=False): + # did we receive all expected objects and nothing else? + if info_only: + info_should_be_empty = False + + assert {s3obj.url for s3obj in s3objs} == set(expected) + for s3obj in s3objs: + # assert that all urls returned are unicode, if not None + assert isinstance(s3obj.key, (unicode_type, type(None))) + assert isinstance(s3obj.url, (unicode_type, type(None))) + assert isinstance(s3obj.prefix, (unicode_type, type(None))) + + # is key actually a suffix? + assert s3obj.url.endswith(s3obj.key) + if s3obj.prefix: + # is prefix actually a prefix? + assert s3obj.url.startswith(s3obj.prefix) + # key must look like a real key + assert 0 < len(s3obj.key) < len(s3obj.url) + else: + # if there's no prefix, the key is the url + assert s3obj.url == s3obj.key + + range_info = s3obj.range_info + if range_info: + range_info = (range_info.request_offset, range_info.request_length) + expected_result = expected[s3obj.url].get(range_info, None) + assert expected_result + size = expected_result.size + checksum = expected_result.checksum + content_type = expected_result.content_type + metadata = expected_result.metadata + if size is None: + assert s3obj.exists == False + assert s3obj.downloaded == False + else: + assert s3obj.exists == True + if info_only: + assert s3obj.downloaded == False + else: + assert s3obj.downloaded == True + # local file exists? + assert os.path.exists(s3obj.path) + # blob is ok? + blob = s3obj.blob + assert len(blob) == size + assert type(blob) == type(b'') + assert sha1(blob).hexdigest() == checksum + # size is ok? + assert s3obj.size == size + if info_should_be_empty: + assert not s3obj.has_info + else: + # Content_type is OK + if content_type is None: + # Default content-type when nothing is suplied + assert s3obj.content_type == 'binary/octet-stream' + else: + assert s3obj.content_type == content_type + # metadata is OK + if metadata is None: + assert s3obj.metadata == None + else: + s3objmetadata = s3obj.metadata + assert s3objmetadata is not None + found = set() + for k, v in metadata.items(): + v1 = s3objmetadata.get(k, None) + assert v1 == v, "Metadata %s mismatch" % k + found.add(k) + extra_keys = set(s3objmetadata.keys()) - found + assert not extra_keys, \ + "Additional metadata present %s" % str(extra_keys) + +def shuffle(objs): + for i, (key, value) in enumerate(objs): + t = random.randrange(i, len(objs)) + key_t, value_t = objs[t] + objs[i], objs[t] = (key, value_t), (key_t, value) + +def deranged_shuffle(objs): + shuffled_objs = objs[:] + while True: + shuffle(shuffled_objs) + for (i, a), (j, b) in zip(objs, shuffled_objs): + if a == b: + break + else: + return shuffled_objs + +@pytest.fixture +def tempdir(): + tmpdir = mkdtemp(dir='.', prefix='metaflow.test.tmp') + yield tmpdir + shutil.rmtree(tmpdir) + +@pytest.mark.parametrize( + argnames=['s3root', 'pathspecs', 'expected'], + **s3_data.pytest_benchmark_case() +) +@pytest.mark.benchmark(max_time=30) +def test_info_one_benchmark(benchmark, s3root, pathspecs, expected): + def _do(): + with S3() as s3: + res = [] + for url in expected: + res.append(s3.info(url)) + return res + res = benchmark(_do) + assert_results(res, expected, info_only=True) + +@pytest.mark.parametrize( + argnames=['s3root', 'pathspecs', 'expected'], + **s3_data.pytest_benchmark_many_case() +) +@pytest.mark.benchmark(max_time=30) +def test_info_many_benchmark(benchmark, s3root, pathspecs, expected): + urls = [] + check_expected = {} + for count, v in expected: + urls.extend(list(v)*count) + if count > 0: + check_expected.update(v) + random.shuffle(urls) + def _do(): + with S3() as s3: + res = s3.info_many(urls) + return res + res = benchmark(_do) + assert_results(res, check_expected, info_only=True) + +@pytest.mark.parametrize( + argnames=['s3root', 'pathspecs', 'expected'], + **s3_data.pytest_benchmark_case() +) +@pytest.mark.benchmark(max_time=60) +def test_get_one_benchmark(benchmark, s3root, pathspecs, expected): + def _do(): + with S3() as s3: + res = [] + for url in expected: + # Use return_missing as this is the most expensive path + res.append(s3.get(url, return_missing=True)) + return res + res = benchmark(_do) + # We do not actually check results because the files will be cleared + # Could be improved if we want to be real precise + # assert_results(res, expected, info_should_be_empty=True) + +@pytest.mark.parametrize( + argnames=['s3root', 'pathspecs', 'expected'], + **s3_data.pytest_benchmark_many_case() +) +@pytest.mark.benchmark(max_time=60) +def test_get_many_benchmark(benchmark, s3root, pathspecs, expected): + urls = [] + check_expected = {} + for count, v in expected: + urls.extend(list(v)*count) + if count > 0: + check_expected.update(v) + random.shuffle(urls) + def _do(): + with S3() as s3: + # Use return_missing as this is the most expensive path + res = s3.get_many(urls, return_missing=True) + return res + res = benchmark(_do) + # assert_results(res, check_expected, info_should_be_empty=True) + +@pytest.mark.parametrize( + argnames=['s3root', 'blobs', 'expected'], + **s3_data.pytest_benchmark_put_case() +) +@pytest.mark.benchmark(max_time=60) +def test_put_one_benchmark(benchmark, tempdir, s3root, blobs, expected): + # We generate the files here to avoid having them saved in the benchmark + # result file which then prevents comparisons + def _generate_files(blobs): + for blob in blobs: + prefix, fname, size = blob + data = s3_data.RandomFile(prefix, fname, size) + key = str(uuid4()) + path = os.path.join(tempdir, key) + with open(path, 'wb') as f: + f.write(data.data) + yield key, path + # Generate all files before the test so we don't time this + all_files = list(_generate_files(blobs)) + def _do(): + with S3(s3root=s3root) as s3: + res = [] + for key, obj in all_files: + key = str(uuid4()) # New "name" every time + res.append(s3.put(key, obj, overwrite=False)) + return res + res = benchmark(_do) + +@pytest.mark.parametrize( + argnames=['s3root', 'blobs', 'expected'], + **s3_data.pytest_benchmark_put_many_case() +) +@pytest.mark.benchmark(max_time=60) +def test_put_many_benchmark(benchmark, tempdir, s3root, blobs, expected): + def _generate_files(blobs): + generated_paths = {} + for blob in blobs: + count, blob_info = blob + if blob_info in generated_paths: + for _ in range(count): + yield str(uuid4()), generated_paths[blob_info] + else: + prefix, fname, size = blob_info + data = s3_data.RandomFile(prefix, fname, size) + key = str(uuid4()) + path = os.path.join(tempdir, key) + with open(path, 'wb') as f: + f.write(data.data) + generated_paths[blob_info] = path + for _ in range(count): + yield str(uuid4()), path + all_files = list(_generate_files(blobs)) + def _do(): + new_files = [(str(uuid4()), path) for _, path in all_files] + with S3(s3root=s3root) as s3: + s3urls = s3.put_files(new_files, overwrite=False) + return s3urls + res = benchmark(_do) + +@pytest.mark.parametrize( + argnames=['s3root', 'pathspecs', 'expected'], + **s3_data.pytest_fakerun_cases() +) +def test_init_options(s3root, pathspecs, expected): + [pathspec] = pathspecs + flow_name, run_id = pathspec.split('/') + plen = len(s3root) + + # option 1) s3root as prefix + with S3(s3root=s3root) as s3: + for url, exp in expected.items(): + # s3root should work as a prefix + s3obj = s3.get(url[plen:]) + assert s3obj.key == url[plen:] + assert_results([s3obj], {url: exp}) + with pytest.raises(MetaflowS3URLException): + s3.get('s3://some/fake/address') + + # option 2) full url as s3root + for url, exp in expected.items(): + with S3(s3root=url) as s3: + s3obj = s3.get() + assert_results([s3obj], {url: exp}) + + # option 3) full urls + with S3() as s3: + for url, exp in expected.items(): + # s3root should work as a prefix + s3obj = s3.get(url) + assert s3obj.key == url + assert_results([s3obj], {url: exp}) + with pytest.raises(MetaflowS3URLException): + s3.get('suffix') + with pytest.raises(MetaflowS3URLException): + s3.get('s3://nopath') + with pytest.raises(MetaflowS3URLException): + s3.get_many(['suffixes']) + with pytest.raises(MetaflowS3URLException): + s3.get_recursive(['suffixes']) + with pytest.raises(MetaflowS3URLException): + s3.get_all() + + # option 4) 'current' environment (fake a running flow) + flow = FakeFlow(use_cli=False) + + parsed = urlparse(s3root) + with pytest.raises(MetaflowS3URLException): + # current not set yet, so this should fail + with S3(run=flow): + pass + + current._set_env(flow_name, + run_id, + 'no_step', + 'no_task', + 'no_origin_run_id', + 'no_ns', + 'no_user') + + with S3(bucket=parsed.netloc, prefix=parsed.path, run=flow) as s3: + for url, exp in expected.items(): + name = url.split('/')[-1] + s3obj = s3.get(name) + assert s3obj.key == name + assert_results([s3obj], {url: exp}) + names = [url.split('/')[-1] for url in expected] + s3objs = s3.get_many(names) + assert {e.key for e in s3objs} == set(names) + assert_results(s3objs, expected) + assert_results(s3.get_all(), expected, info_should_be_empty=True) + + # option 5) run object + namespace(None) + with S3(bucket=parsed.netloc, prefix=parsed.path, run=Run(pathspec)) as s3: + names = [url.split('/')[-1] for url in expected] + assert_results(s3.get_many(names), expected) + +@pytest.mark.parametrize( + argnames=['s3root', 'prefixes', 'expected'], + **s3_data.pytest_basic_case() +) +def test_info_one(s3root, prefixes, expected): + with S3() as s3: + for url, item in expected.items(): + if item[None].size is None: + # ensure that the default return_missing=False works + with pytest.raises(MetaflowS3NotFound): + s3obj = s3.info(url) + # test return_missing=True + s3obj = s3.info(url, return_missing=True) + assert_results( + [s3obj], {url: expected[url]}, info_only=True) + else: + s3obj = s3.info(url) + assert_results( + [s3obj], {url: expected[url]}, info_only=True) + +@pytest.mark.parametrize( + argnames=['s3root', 'prefixes', 'expected'], + **s3_data.pytest_basic_case() +) +def test_info_many(s3root, prefixes, expected): + with S3() as s3: + # 1) test the non-missing case + + # to test result ordering, make sure we are requesting + # keys in a non-lexicographic order + not_missing = [url for url, v in expected.items() + if v[None].size is not None] + urls = list(sorted(not_missing, reverse=True)) + s3objs = s3.info_many(urls) + + # results should come out in the order of keys requested + assert urls == [e.url for e in s3objs] + assert_results( + s3objs, {k: expected[k] for k in not_missing}, info_only=True) + + # 2) test with missing items, default case + if not_missing != list(expected): + with pytest.raises(MetaflowS3NotFound): + s3objs = s3.info_many(list(expected)) + + # 3) test with missing items, return_missing=True + + # to test result ordering, make sure we are requesting + # keys in a non-lexicographic order. Missing files should + # be returned in order too + urls = list(sorted(expected, reverse=True)) + s3objs = s3.info_many(urls, return_missing=True) + assert urls == [e.url for e in s3objs] + assert_results(s3objs, expected, info_only=True) + +@pytest.mark.parametrize( + argnames=['s3root', 'prefixes', 'expected'], + **s3_data.pytest_fakerun_cases() +) +def test_get_exceptions(s3root, prefixes, expected): + # get_many() goes via s3op, get() is a method - test both the code paths + with S3() as s3: + with pytest.raises(MetaflowS3AccessDenied): + s3.get_many(['s3://foobar/foo']) + with pytest.raises(MetaflowS3AccessDenied): + s3.get('s3://foobar/foo') + with S3(s3root=s3root) as s3: + with pytest.raises(MetaflowS3NotFound): + s3.get_many(['this_file_does_not_exist']) + with pytest.raises(MetaflowS3NotFound): + s3.get('this_file_does_not_exist') + +@pytest.mark.parametrize( + argnames=['s3root', 'prefixes', 'expected'], + **s3_data.pytest_basic_case() +) +def test_get_one(s3root, prefixes, expected): + with S3() as s3: + for url, item in expected.items(): + if item[None].size is None: + # ensure that the default return_missing=False works + with pytest.raises(MetaflowS3NotFound): + s3obj = s3.get(url) + # test return_missing=True + s3obj = s3.get(url, return_missing=True) + assert_results( + [s3obj], {url: expected[url]}) + else: + s3obj = s3.get(url, return_info=True) + assert_results( + [s3obj], {url: expected[url]}) + +@pytest.mark.parametrize( + argnames=['s3root', 'prefixes', 'expected'], + **s3_data.pytest_basic_case() +) +def test_get_one_wo_meta(s3root, prefixes, expected): + with S3() as s3: + for url, item in expected.items(): + if item[None].size is None: + # ensure that the default return_missing=False works + with pytest.raises(MetaflowS3NotFound): + s3obj = s3.get(url) + s3obj = s3.get(url, return_missing=True, return_info=False) + assert_results( + [s3obj], {url: expected[url]}, info_should_be_empty=True) + else: + s3obj = s3.get(url, return_info=False) + assert_results( + [s3obj], {url: expected[url]}, info_should_be_empty=True) + +@pytest.mark.parametrize( + argnames=['s3root', 'prefixes', 'expected'], + **s3_data.pytest_large_case() +) +def test_get_all(s3root, prefixes, expected): + expected_exists = {url: v + for url, v in expected.items() + if v[None].size is not None} + for prefix in prefixes: + with S3(s3root=os.path.join(s3root, prefix)) as s3: + s3objs = s3.get_all() + # results should be in lexicographic order + assert list(sorted(e.url for e in s3objs))\ + == [e.url for e in s3objs] + assert_results(s3objs, expected_exists, info_should_be_empty=True) + +@pytest.mark.parametrize( + argnames=['s3root', 'prefixes', 'expected'], + **s3_data.pytest_basic_case() +) +def test_get_all_with_meta(s3root, prefixes, expected): + expected_exists = {url: v + for url, v in expected.items() + if v[None].size is not None} + for prefix in prefixes: + with S3(s3root=os.path.join(s3root, prefix)) as s3: + s3objs = s3.get_all(return_info=True) + # results should be in lexicographic order + assert list(sorted(e.url for e in s3objs))\ + == [e.url for e in s3objs] + assert_results(s3objs, expected_exists) + +@pytest.mark.parametrize( + argnames=['s3root', 'prefixes', 'expected'], + **s3_data.pytest_basic_case() +) +def test_get_many(s3root, prefixes, expected): + with S3() as s3: + # 1) test the non-missing case + + # to test result ordering, make sure we are requesting + # keys in a non-lexicographic order + not_missing = [url for url, v in expected.items() + if v[None].size is not None] + urls = list(sorted(not_missing, reverse=True)) + s3objs = s3.get_many(urls, return_info=True) + + # results should come out in the order of keys requested + assert urls == [e.url for e in s3objs] + assert_results( + s3objs, {k: expected[k] for k in not_missing}) + + # 2) test with missing items, default case + if not_missing != list(expected): + with pytest.raises(MetaflowS3NotFound): + s3objs = s3.get_many(list(expected), return_info=True) + + # 3) test with missing items, return_missing=True + + # to test result ordering, make sure we are requesting + # keys in a non-lexicographic order. Missing files should + # be returned in order too + urls = list(sorted(expected, reverse=True)) + s3objs = s3.get_many(urls, return_missing=True, return_info=True) + assert urls == [e.url for e in s3objs] + assert_results(s3objs, expected) + +@pytest.mark.parametrize( + argnames=['s3root', 'prefixes', 'expected'], + **s3_data.pytest_many_prefixes_case() +) +def test_list_paths(s3root, prefixes, expected): + def urls_by_prefix(prefix): + root = os.path.join(s3root, prefix) + for url, v in expected.items(): + if url.startswith(root) and v[None].size is not None: + yield url + + # 1) test that list_paths() without arguments works + matches = {prefix: frozenset(urls_by_prefix(prefix)) for prefix in prefixes} + non_empty = {prefix for prefix, urls in matches.items() if urls} + + with S3(s3root=s3root) as s3: + s3objs = s3.list_paths() + # found_prefixes is a subset of paths under s3root + found_prefixes = [e for e in s3objs if e.key in prefixes] + # we expect to find all non-empty prefixes under the s3root + assert {e.key for e in found_prefixes} == non_empty + # they should be all marked as non-existent objects, just prefixes + assert all(not e.exists for e in found_prefixes) + # they should be all marked as not downloaded + assert all(not e.downloaded for e in found_prefixes) + + # 2) test querying by many prefixes + with S3(s3root=s3root) as s3: + s3objs = s3.list_paths(prefixes) + assert frozenset(e.prefix.rstrip('/').split('/')[-1] + for e in s3objs) == non_empty + + for prefix, exp in matches.items(): + exists = frozenset(e.url for e in s3objs + if e.prefix == prefix and e.exists) + not_exists = frozenset(e.url for e in s3objs + if e.prefix == prefix and not e.exists) + # every object should be expected + assert all(e in exp for e in exists) + # not existing ones are prefixes, they shouldn't match + assert all(e not in exp for e in not_exists) + + # 3) eventually list_paths should hit the leaf + for url, v in expected.items(): + if v[None].size is None: + with S3() as s3: + # querying a non-existent object should return + # prefixes or nothing + s3objs = s3.list_paths([url]) + assert [e for e in s3objs if e.exists] == [] + else: + suffix = url[len(s3root):] + expected_keys = suffix.split('/') + if len(expected_keys) > 20: + # speed optimization: exclude crazy long paths + continue + got_url = s3root + for idx, expected_key in enumerate(expected_keys): + with S3(s3root=got_url) as s3: + s3objs = s3.list_paths() + # are we at the leaf? + if idx == len(expected_keys) - 1: + # a leaf object should always exist + [match] = [e for e in s3objs + if e.key == expected_key and e.exists] + else: + # a non-leaf may match objects that are also prefixes + [match] = [e for e in s3objs + if e.key == expected_key and not e.exists] + # prefix + key == url + assert os.path.join(match.prefix, match.key) ==\ + match.url.rstrip('/') + got_url = match.url + + # the leaf should be the object itself + assert match.url == url + +@pytest.mark.parametrize( + argnames=['s3root', 'prefixes', 'expected'], + **s3_data.pytest_many_prefixes_case() +) +def test_list_recursive(s3root, prefixes, expected): + not_missing = [url for url, v in expected.items() + if v[None].size is not None] + with S3(s3root=s3root) as s3: + s3objs = s3.list_recursive(prefixes) + assert frozenset(e.url for e in s3objs) == frozenset(not_missing) + # ensure that there are no duplicates + assert len(s3objs) == len(not_missing) + # list_recursive returns leaves only + assert all(e.exists for e in s3objs) + +@pytest.mark.parametrize( + argnames=['s3root', 'prefixes', 'expected'], + **s3_data.pytest_many_prefixes_case() +) +def test_get_recursive(s3root, prefixes, expected): + expected_exists = {url: v + for url, v in expected.items() + if v[None].size is not None} + local_files = [] + with S3(s3root=s3root) as s3: + s3objs = s3.get_recursive(prefixes) + + # we need to deduce which prefixes actually produce results + nonempty_prefixes = list( + filter(lambda p: any(url.startswith(os.path.join(s3root, p)) + for url in expected_exists), + prefixes) + ) + + # prefixes must be returned in the order of prefixes requested + plen = len(s3root) + grouped = list(groupby(s3objs, lambda e: e.prefix[plen:])) + + assert nonempty_prefixes == [prefix for prefix, _ in grouped] + # for each prefix, the results should be in lexicographic order + for prefix, objs in grouped: + urls = [e.url for e in objs] + assert list(sorted(urls)) == urls + + assert_results(s3objs, expected_exists, info_should_be_empty=True) + + # if there are multiple prefixes, it is a bit harder to know + # what's the expected set of results. We do this test only + # for the single-prefix case for now + if len(prefixes) == 1: + [prefix] = prefixes + s3root = os.path.join(s3root, prefix) + keys = {url[len(s3root) + 1:] for url in expected_exists} + assert {e.key for e in s3objs} == keys + + local_files = [s3obj.path for s3obj in s3objs] + # local files must not exist outside of the S3 context + for path in local_files: + assert not os.path.exists(path) + +def test_put_exceptions(): + with S3() as s3: + with pytest.raises(MetaflowS3InvalidObject): + s3.put_many([('a', 1)]) + with pytest.raises(MetaflowS3InvalidObject): + s3.put('a', 1) + with pytest.raises(MetaflowS3NotFound): + s3.put_files([('a', '/non-existent/local-file')]) + with pytest.raises(MetaflowS3URLException): + s3.put_many([('foo', 'bar')]) + +@pytest.mark.parametrize( + argnames=['s3root', 'objs', 'expected'], + **s3_data.pytest_put_strings_case() +) +def test_put_many(s3root, objs, expected): + with S3(s3root=s3root) as s3: + s3urls = s3.put_many(objs) + assert list(dict(s3urls)) == list(dict(objs)) + # results must be in the same order as the keys requested + for i in range(len(s3urls)): + assert objs[i][0] == s3urls[i][0] + with S3() as s3: + s3objs = s3.get_many(dict(s3urls).values()) + assert_results(s3objs, expected) + with S3(s3root=s3root) as s3: + s3objs = s3.get_many(list(dict(objs))) + assert {s3obj.key for s3obj in s3objs} == {key for key, _ in objs} + + # upload shuffled objs with overwrite disabled + shuffled_objs = deranged_shuffle(objs) + with S3(s3root=s3root) as s3: + overwrite_disabled_s3urls = s3.put_many(shuffled_objs, overwrite=False) + assert len(overwrite_disabled_s3urls) == 0 + with S3() as s3: + s3objs = s3.get_many(dict(s3urls).values()) + assert_results(s3objs, expected) + + +@pytest.mark.parametrize( + argnames=['s3root', 'objs', 'expected'], + **s3_data.pytest_put_strings_case() +) +def test_put_one(s3root, objs, expected): + with S3(s3root=s3root) as s3: + for key, obj in objs: + s3url = s3.put(key, obj) + assert s3url in expected + s3obj = s3.get(key) + assert s3obj.key == key + assert_results([s3obj], {s3url: expected[s3url]}) + assert s3obj.blob == to_bytes(obj) + # put with overwrite disabled + s3url = s3.put(key, "random_value", overwrite=False) + assert s3url in expected + s3obj = s3.get(key) + assert s3obj.key == key + assert_results([s3obj], {s3url: expected[s3url]}) + assert s3obj.blob == to_bytes(obj) + +@pytest.mark.parametrize( + argnames=['s3root', 'blobs', 'expected'], + **s3_data.pytest_put_blobs_case() +) +def test_put_files(tempdir, s3root, blobs, expected): + def _files(blobs): + for blob in blobs: + key = getattr(blob, 'key', blob[0]) + data = getattr(blob, 'value', blob[1]) + content_type = getattr(blob, 'content_type', None) + metadata = getattr(blob, 'metadata', None) + path = os.path.join(tempdir, key) + with open(path, 'wb') as f: + f.write(data) + yield S3PutObject( + key=key, + value=path, + content_type=content_type, + metadata=metadata) + with S3(s3root=s3root) as s3: + s3urls = s3.put_files(_files(blobs)) + assert list(dict(s3urls)) == list(dict(blobs)) + + with S3() as s3: + # get urls + s3objs = s3.get_many(dict(s3urls).values()) + assert_results(s3objs, expected) + + with S3(s3root=s3root) as s3: + # get keys + s3objs = s3.get_many(key for key, blob in blobs) + assert {s3obj.key for s3obj in s3objs} == {key for key, _ in blobs} + + # upload shuffled blobs with overwrite disabled + shuffled_blobs = blobs[:] + shuffle(shuffled_blobs) + with S3(s3root=s3root) as s3: + overwrite_disabled_s3urls = s3.put_files(_files(shuffled_blobs), overwrite=False) + assert len(overwrite_disabled_s3urls) == 0 + + with S3() as s3: + s3objs = s3.get_many(dict(s3urls).values()) + assert_results(s3objs, expected) + with S3(s3root=s3root) as s3: + s3objs = s3.get_many(key for key, blob in shuffled_blobs) + assert {s3obj.key for s3obj in s3objs} == {key for key, _ in shuffled_blobs} From 1d28f85c1d13a231e43d2afa9109da0f07936123 Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Mon, 26 Jul 2021 20:50:18 -0700 Subject: [PATCH 008/176] silence tar timestamp warnings (#627) --- metaflow/metaflow_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/metaflow_environment.py b/metaflow/metaflow_environment.py index 018b7093170..14ccca1618d 100644 --- a/metaflow/metaflow_environment.py +++ b/metaflow/metaflow_environment.py @@ -96,7 +96,7 @@ def get_package_commands(self, code_package_url): "mflog \'Failed to download code package from %s " "after 6 tries. Exiting...\' && exit 1; " "fi" % code_package_url, - "tar xf job.tar", + "TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar", "mflog \'Task is starting.\'", ] return cmds From aa7c73bf8d7f43c30646e6a8a2a172fc8ec59cdb Mon Sep 17 00:00:00 2001 From: Savin Date: Thu, 29 Jul 2021 14:14:39 -0700 Subject: [PATCH 009/176] Handle None as default parameters properly in AWS Step Functions (#630) A parameter specification like - ``` Parameter(name="test_param", type=int, default=None) ``` will result in an error even though the default has been specified ``` Flow failed: The value of parameter test_param is ambiguous. It does not have a default and it is not required. ``` This PR fixes this issue. --- .../plugins/aws/step_functions/step_functions.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/metaflow/plugins/aws/step_functions/step_functions.py b/metaflow/plugins/aws/step_functions/step_functions.py index 03372938e66..04a838525a5 100644 --- a/metaflow/plugins/aws/step_functions/step_functions.py +++ b/metaflow/plugins/aws/step_functions/step_functions.py @@ -310,26 +310,17 @@ def _process_parameters(self): "case-insensitive." % param.name) seen.add(norm) - valuetype = param.kwargs.get('type', str) - value = deploy_time_eval(param.kwargs.get('default')) - required = param.kwargs.get('required', False) - # Throw an exception if the flow has optional parameters - # with no default value. - if value is None and required is False: - raise MetaflowException("The value of parameter *%s* is " - "ambiguous. It does not have a " - "default and it is not required." - % param.name) - + is_required = param.kwargs.get('required', False) # Throw an exception if a schedule is set for a flow with required # parameters with no defaults. We currently don't have any notion # of data triggers in AWS Event Bridge. - if value is None and required and has_schedule: + if 'default' not in param.kwargs and is_required and has_schedule: raise MetaflowException("The parameter *%s* does not have a " "default and is required. Scheduling " "such parameters via AWS Event Bridge " "is not currently supported." % param.name) + value = deploy_time_eval(param.kwargs.get('default')) parameters.append(dict(name=param.name, value=value)) return parameters From 0575f8d55c94a6969c35a41675bdb69cc8387c44 Mon Sep 17 00:00:00 2001 From: Romain Date: Thu, 29 Jul 2021 15:51:10 -0700 Subject: [PATCH 010/176] IncludeFile now returns the included file in the client (#607) * DRAFT: IncludeFile now returns the included file in the client and CLI THIS IS NOT FINISHED; DO NOT MERGE AS IS. * Fix the tests * Forgot to update type check for multiple encoding --- metaflow/client/core.py | 3 + metaflow/includefile.py | 47 ++++++++++---- test/core/metaflow_test/cli_check.py | 5 +- test/core/metaflow_test/metadata_check.py | 1 + test/core/tests/basic_include.py | 74 ++++++++++++++--------- 5 files changed, 89 insertions(+), 41 deletions(-) diff --git a/metaflow/client/core.py b/metaflow/client/core.py index 84ba4946a3a..a0f5017729c 100644 --- a/metaflow/client/core.py +++ b/metaflow/client/core.py @@ -11,6 +11,7 @@ MetaflowNamespaceMismatch,\ MetaflowInternalError +from metaflow.includefile import IncludedFile from metaflow.metaflow_config import DEFAULT_METADATA from metaflow.plugins import ENVIRONMENTS, METADATA_PROVIDERS from metaflow.unbounded_foreach import CONTROL_TASK_TAG @@ -711,6 +712,8 @@ def data(self): sha = self._object['sha'] with filecache.get_data(ds_type, self.path_components[0], sha) as f: obj = pickle.load(f) + if isinstance(obj, IncludedFile): + return obj.decode(self.id) return obj # TODO add diff --git a/metaflow/includefile.py b/metaflow/includefile.py index 569f028e43b..e9e2ee6005c 100644 --- a/metaflow/includefile.py +++ b/metaflow/includefile.py @@ -165,6 +165,27 @@ def put(self, key, obj, overwrite=True): DATACLIENTS = {'local': Local, 's3': S3} +class IncludedFile(object): + # Thin wrapper to indicate to the MF client that this object is special + # and should be handled as an IncludedFile when returning it (ie: fetching + # the actual content) + + def __init__(self, descriptor): + self._descriptor = json.dumps(descriptor) + + @property + def descriptor(self): + return self._descriptor + + def decode(self, name, var_type='Artifact'): + ok, file_type, err = LocalFile.is_file_handled(self._descriptor) + if not ok: + raise MetaflowException("%s '%s' could not be loaded: %s" % (var_type, name, err)) + if file_type is None or isinstance(file_type, LocalFile): + raise MetaflowException("%s '%s' was not properly converted" % (var_type, name)) + return file_type.load(self._descriptor) + + class LocalFile(): def __init__(self, is_text, encoding, path): self._is_text = is_text @@ -173,7 +194,16 @@ def __init__(self, is_text, encoding, path): @classmethod def is_file_handled(cls, path): + # This returns a tuple: + # - True/False indicating whether the file is handled + # - None if we need to create a handler for the file, a LocalFile if + # we already know what to do with the file or a Uploader if the file + # is already present remotely (either starting with s3:// or local://) + # - An error message if file is not handled if path: + if isinstance(path, IncludedFile): + path = path.descriptor + decoded_value = Uploader.decode_value(to_unicode(path)) if decoded_value['type'] == 'self': return True, LocalFile( @@ -295,15 +325,10 @@ def __init__( name, required=required, help=help, type=FilePathClass(is_text, encoding), **kwargs) - def load_parameter(self, val): - if val is None: - return val - ok, file_type, err = LocalFile.is_file_handled(val) - if not ok: - raise MetaflowException("Parameter '%s' could not be loaded: %s" % (self.name, err)) - if file_type is None or isinstance(file_type, LocalFile): - raise MetaflowException("Parameter '%s' was not properly converted" % self.name) - return file_type.load(val) + def load_parameter(self, v): + if v is None: + return v + return v.decode(self.name, var_type='Parameter') class Uploader(): @@ -316,11 +341,11 @@ def __init__(self, client_class): @staticmethod def encode_url(url_type, url, **kwargs): # Avoid encoding twice (default -> URL -> _convert method of FilePath for example) - if url is None or len(url) == 0 or url[0] == '{': + if isinstance(url, IncludedFile): return url return_value = {'type': url_type, 'url': url} return_value.update(kwargs) - return json.dumps(return_value) + return IncludedFile(return_value) @staticmethod def decode_value(value): diff --git a/test/core/metaflow_test/cli_check.py b/test/core/metaflow_test/cli_check.py index 7c46785c617..47ddfdb70fa 100644 --- a/test/core/metaflow_test/cli_check.py +++ b/test/core/metaflow_test/cli_check.py @@ -4,6 +4,7 @@ import json from tempfile import NamedTemporaryFile +from metaflow.includefile import IncludedFile from metaflow.util import is_stringish from . import MetaflowCheck, AssertArtifactFailed, AssertLogFailed, truncate @@ -39,6 +40,8 @@ def assert_artifact(self, step, name, value, fields=None): for field, v in fields.items(): if is_stringish(artifact): data = json.loads(artifact) + elif isinstance(artifact, IncludedFile): + data = json.loads(artifact.descriptor) else: data = artifact if not isinstance(data, dict): @@ -92,4 +95,4 @@ def get_log(self, step, logtype): 'logs', '--%s' % logtype, '%s/%s' % (self.run_id, step)] - return self.run_cli(cmd, capture_output=True).decode('utf-8') \ No newline at end of file + return self.run_cli(cmd, capture_output=True).decode('utf-8') diff --git a/test/core/metaflow_test/metadata_check.py b/test/core/metaflow_test/metadata_check.py index cc9d0bfd21e..ad11276bea9 100644 --- a/test/core/metaflow_test/metadata_check.py +++ b/test/core/metaflow_test/metadata_check.py @@ -1,4 +1,5 @@ import json + from metaflow.util import is_stringish from . import MetaflowCheck, AssertArtifactFailed, AssertLogFailed, assert_equals, assert_exception, truncate diff --git a/test/core/tests/basic_include.py b/test/core/tests/basic_include.py index fa9f0404b1d..b531311b305 100644 --- a/test/core/tests/basic_include.py +++ b/test/core/tests/basic_include.py @@ -41,33 +41,49 @@ def step_all(self): pass def check_results(self, flow, checker): - for step in flow: - checker.assert_artifact( - step.name, - 'myfile_txt', - None, - fields={'type': 'uploader-v1', - 'is_text': True, - 'encoding': None}) - checker.assert_artifact( - step.name, - 'myfile_utf8', - None, - fields={'type': 'uploader-v1', - 'is_text': True, - 'encoding': 'utf8'}) - checker.assert_artifact( - step.name, - 'myfile_binary', - None, - fields={'type': 'uploader-v1', - 'is_text': False, - 'encoding': None}) - checker.assert_artifact( - step.name, - 'myfile_overriden', - None, - fields={'type': 'uploader-v1', - 'is_text': True, - 'encoding': None}) + run = checker.get_run() + if run is None: + # CliChecker does not return a run object; we check to make sure + # the returned value is the blob describing the artifact + # (this may be improved in the future) + for step in flow: + checker.assert_artifact( + step.name, + 'myfile_txt', + None, + fields={'type': 'uploader-v1', + 'is_text': True, + 'encoding': None}) + checker.assert_artifact( + step.name, + 'myfile_utf8', + None, + fields={'type': 'uploader-v1', + 'is_text': True, + 'encoding': 'utf8'}) + checker.assert_artifact( + step.name, + 'myfile_binary', + None, + fields={'type': 'uploader-v1', + 'is_text': False, + 'encoding': None}) + checker.assert_artifact( + step.name, + 'myfile_overriden', + None, + fields={'type': 'uploader-v1', + 'is_text': True, + 'encoding': None}) + else: + # In the case of the client, we check the value. + for step in flow: + checker.assert_artifact(step.name, 'myfile_txt', + "Regular Text File") + checker.assert_artifact(step.name, 'myfile_utf8', + u"UTF Text File \u5e74") + checker.assert_artifact(step.name, 'myfile_binary', + u"UTF Text File \u5e74".encode(encoding='utf8')) + checker.assert_artifact(step.name, 'myfile_overriden', + "Override Text File") From ed877dd28a643ef7ada7d1f921e3cabfdf8011a5 Mon Sep 17 00:00:00 2001 From: Savin Date: Thu, 29 Jul 2021 16:00:13 -0700 Subject: [PATCH 011/176] Add resource tags to AWS Batch jobs (#631) * Add resource tags to AWS Batch jobs This PR assumes that Batch:TagResource is whitelisted for the role which submits the AWS Batch job. * propagate tags * add env var * better commens * add production token --- metaflow/metaflow_config.py | 6 ++++++ metaflow/plugins/aws/batch/batch.py | 10 +++++++++- metaflow/plugins/aws/batch/batch_client.py | 9 ++++++++- metaflow/plugins/aws/step_functions/step_functions.py | 3 +++ 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py index 1dd233c0ed0..befa3013682 100644 --- a/metaflow/metaflow_config.py +++ b/metaflow/metaflow_config.py @@ -111,6 +111,12 @@ def from_conf(name, default=None): BATCH_METADATA_SERVICE_URL = from_conf('METAFLOW_SERVICE_INTERNAL_URL', METADATA_SERVICE_URL) BATCH_METADATA_SERVICE_HEADERS = METADATA_SERVICE_HEADERS +# Assign resource tags to AWS Batch jobs. Set to False by default since +# it requires `Batch:TagResource` permissions which may not be available +# in all Metaflow deployments. Hopefully, some day we can flip the +# default to True. +BATCH_EMIT_TAGS = from_conf("METAFLOW_BATCH_EMIT_TAGS", False) + ### # AWS Step Functions configuration ### diff --git a/metaflow/plugins/aws/batch/batch.py b/metaflow/plugins/aws/batch/batch.py index c2f91f3c0f6..9d72b41b828 100644 --- a/metaflow/plugins/aws/batch/batch.py +++ b/metaflow/plugins/aws/batch/batch.py @@ -11,7 +11,7 @@ from metaflow.exception import MetaflowException, MetaflowInternalError from metaflow.metaflow_config import BATCH_METADATA_SERVICE_URL, DATATOOLS_S3ROOT, \ DATASTORE_LOCAL_DIR, DATASTORE_SYSROOT_S3, DEFAULT_METADATA, \ - BATCH_METADATA_SERVICE_HEADERS + BATCH_METADATA_SERVICE_HEADERS, BATCH_EMIT_TAGS from metaflow import util from .batch_client import BatchClient @@ -214,6 +214,14 @@ def create_job( if attrs: for key, value in attrs.items(): job.parameter(key, value) + # Tags for AWS Batch job (for say cost attribution) + if BATCH_EMIT_TAGS: + for key in ['metaflow.flow_name', 'metaflow.run_id', + 'metaflow.step_name', 'metaflow.version', + 'metaflow.run_id.$', 'metaflow.user', + 'metaflow.owner', 'metaflow.production_token']: + if key in attrs: + job.tag(key, attrs.get(key)) return job def launch_job( diff --git a/metaflow/plugins/aws/batch/batch_client.py b/metaflow/plugins/aws/batch/batch_client.py index 192e982541f..c4c1ed65334 100644 --- a/metaflow/plugins/aws/batch/batch_client.py +++ b/metaflow/plugins/aws/batch/batch_client.py @@ -139,7 +139,10 @@ def _register_job_definition(self, 'type': 'MEMORY' } ] - } + }, + # This propagates the AWS Batch resource tags to the underlying + # ECS tasks. + 'propagateTags': True } if platform == 'FARGATE' or platform == 'FARGATE_SPOT': @@ -319,6 +322,10 @@ def timeout_in_secs(self, timeout_in_secs): self.payload['timeout']['attemptDurationSeconds'] = timeout_in_secs return self + def tag(self, key, value): + self.payload['tags'][key] = str(value) + return self + def parameter(self, key, value): self.payload['parameters'][key] = str(value) return self diff --git a/metaflow/plugins/aws/step_functions/step_functions.py b/metaflow/plugins/aws/step_functions/step_functions.py index 04a838525a5..167cd96fcef 100644 --- a/metaflow/plugins/aws/step_functions/step_functions.py +++ b/metaflow/plugins/aws/step_functions/step_functions.py @@ -814,6 +814,9 @@ def batch(self, job): to_pascalcase(job.payload['retryStrategy'])) \ .parameter('Timeout', to_pascalcase(job.payload['timeout'])) + # tags may not be present in all scenarios + if 'tags' in job.payload: + self.parameter('Tags', job.payload['tags']) return self def dynamo_db(self, table_name, primary_key, values): From 9f832e62b3d4288acae8de483dc5709d660dc347 Mon Sep 17 00:00:00 2001 From: Savin Date: Thu, 29 Jul 2021 17:08:05 -0700 Subject: [PATCH 012/176] New patch release (#633) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1ab335be548..609fc3adccb 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '2.3.2' +version = '2.3.3' setup(name='metaflow', version=version, From 74bb0d7ed7d31ad250621b0087287b6364af597f Mon Sep 17 00:00:00 2001 From: Savin Date: Fri, 6 Aug 2021 14:51:18 -0700 Subject: [PATCH 013/176] Bug fix with s3tail exception (#635) --- metaflow/datastore/util/s3tail.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/datastore/util/s3tail.py b/metaflow/datastore/util/s3tail.py index f0c3f04b694..f336e8ba08f 100644 --- a/metaflow/datastore/util/s3tail.py +++ b/metaflow/datastore/util/s3tail.py @@ -74,5 +74,5 @@ def _fill_buf(self): elif code[0] == '5': return None else: - raise Exception('Retrieving %s/%s failed: %s' % (self.bucket, self.key, code)) + raise Exception('Retrieving %s/%s failed: %s' % (self._bucket, self._key, code)) From 0633e35c3ffd85a9c7774e056f68ab9e374c41b9 Mon Sep 17 00:00:00 2001 From: Savin Date: Wed, 11 Aug 2021 13:19:07 -0700 Subject: [PATCH 014/176] Revert "IncludeFile now returns the included file in the client (#607)" (#637) This reverts commit 0575f8d55c94a6969c35a41675bdb69cc8387c44. --- metaflow/client/core.py | 3 - metaflow/includefile.py | 47 ++++---------- test/core/metaflow_test/cli_check.py | 5 +- test/core/metaflow_test/metadata_check.py | 1 - test/core/tests/basic_include.py | 74 +++++++++-------------- 5 files changed, 41 insertions(+), 89 deletions(-) diff --git a/metaflow/client/core.py b/metaflow/client/core.py index a0f5017729c..84ba4946a3a 100644 --- a/metaflow/client/core.py +++ b/metaflow/client/core.py @@ -11,7 +11,6 @@ MetaflowNamespaceMismatch,\ MetaflowInternalError -from metaflow.includefile import IncludedFile from metaflow.metaflow_config import DEFAULT_METADATA from metaflow.plugins import ENVIRONMENTS, METADATA_PROVIDERS from metaflow.unbounded_foreach import CONTROL_TASK_TAG @@ -712,8 +711,6 @@ def data(self): sha = self._object['sha'] with filecache.get_data(ds_type, self.path_components[0], sha) as f: obj = pickle.load(f) - if isinstance(obj, IncludedFile): - return obj.decode(self.id) return obj # TODO add diff --git a/metaflow/includefile.py b/metaflow/includefile.py index e9e2ee6005c..569f028e43b 100644 --- a/metaflow/includefile.py +++ b/metaflow/includefile.py @@ -165,27 +165,6 @@ def put(self, key, obj, overwrite=True): DATACLIENTS = {'local': Local, 's3': S3} -class IncludedFile(object): - # Thin wrapper to indicate to the MF client that this object is special - # and should be handled as an IncludedFile when returning it (ie: fetching - # the actual content) - - def __init__(self, descriptor): - self._descriptor = json.dumps(descriptor) - - @property - def descriptor(self): - return self._descriptor - - def decode(self, name, var_type='Artifact'): - ok, file_type, err = LocalFile.is_file_handled(self._descriptor) - if not ok: - raise MetaflowException("%s '%s' could not be loaded: %s" % (var_type, name, err)) - if file_type is None or isinstance(file_type, LocalFile): - raise MetaflowException("%s '%s' was not properly converted" % (var_type, name)) - return file_type.load(self._descriptor) - - class LocalFile(): def __init__(self, is_text, encoding, path): self._is_text = is_text @@ -194,16 +173,7 @@ def __init__(self, is_text, encoding, path): @classmethod def is_file_handled(cls, path): - # This returns a tuple: - # - True/False indicating whether the file is handled - # - None if we need to create a handler for the file, a LocalFile if - # we already know what to do with the file or a Uploader if the file - # is already present remotely (either starting with s3:// or local://) - # - An error message if file is not handled if path: - if isinstance(path, IncludedFile): - path = path.descriptor - decoded_value = Uploader.decode_value(to_unicode(path)) if decoded_value['type'] == 'self': return True, LocalFile( @@ -325,10 +295,15 @@ def __init__( name, required=required, help=help, type=FilePathClass(is_text, encoding), **kwargs) - def load_parameter(self, v): - if v is None: - return v - return v.decode(self.name, var_type='Parameter') + def load_parameter(self, val): + if val is None: + return val + ok, file_type, err = LocalFile.is_file_handled(val) + if not ok: + raise MetaflowException("Parameter '%s' could not be loaded: %s" % (self.name, err)) + if file_type is None or isinstance(file_type, LocalFile): + raise MetaflowException("Parameter '%s' was not properly converted" % self.name) + return file_type.load(val) class Uploader(): @@ -341,11 +316,11 @@ def __init__(self, client_class): @staticmethod def encode_url(url_type, url, **kwargs): # Avoid encoding twice (default -> URL -> _convert method of FilePath for example) - if isinstance(url, IncludedFile): + if url is None or len(url) == 0 or url[0] == '{': return url return_value = {'type': url_type, 'url': url} return_value.update(kwargs) - return IncludedFile(return_value) + return json.dumps(return_value) @staticmethod def decode_value(value): diff --git a/test/core/metaflow_test/cli_check.py b/test/core/metaflow_test/cli_check.py index 47ddfdb70fa..7c46785c617 100644 --- a/test/core/metaflow_test/cli_check.py +++ b/test/core/metaflow_test/cli_check.py @@ -4,7 +4,6 @@ import json from tempfile import NamedTemporaryFile -from metaflow.includefile import IncludedFile from metaflow.util import is_stringish from . import MetaflowCheck, AssertArtifactFailed, AssertLogFailed, truncate @@ -40,8 +39,6 @@ def assert_artifact(self, step, name, value, fields=None): for field, v in fields.items(): if is_stringish(artifact): data = json.loads(artifact) - elif isinstance(artifact, IncludedFile): - data = json.loads(artifact.descriptor) else: data = artifact if not isinstance(data, dict): @@ -95,4 +92,4 @@ def get_log(self, step, logtype): 'logs', '--%s' % logtype, '%s/%s' % (self.run_id, step)] - return self.run_cli(cmd, capture_output=True).decode('utf-8') + return self.run_cli(cmd, capture_output=True).decode('utf-8') \ No newline at end of file diff --git a/test/core/metaflow_test/metadata_check.py b/test/core/metaflow_test/metadata_check.py index ad11276bea9..cc9d0bfd21e 100644 --- a/test/core/metaflow_test/metadata_check.py +++ b/test/core/metaflow_test/metadata_check.py @@ -1,5 +1,4 @@ import json - from metaflow.util import is_stringish from . import MetaflowCheck, AssertArtifactFailed, AssertLogFailed, assert_equals, assert_exception, truncate diff --git a/test/core/tests/basic_include.py b/test/core/tests/basic_include.py index b531311b305..fa9f0404b1d 100644 --- a/test/core/tests/basic_include.py +++ b/test/core/tests/basic_include.py @@ -41,49 +41,33 @@ def step_all(self): pass def check_results(self, flow, checker): - run = checker.get_run() - if run is None: - # CliChecker does not return a run object; we check to make sure - # the returned value is the blob describing the artifact - # (this may be improved in the future) - for step in flow: - checker.assert_artifact( - step.name, - 'myfile_txt', - None, - fields={'type': 'uploader-v1', - 'is_text': True, - 'encoding': None}) - checker.assert_artifact( - step.name, - 'myfile_utf8', - None, - fields={'type': 'uploader-v1', - 'is_text': True, - 'encoding': 'utf8'}) - checker.assert_artifact( - step.name, - 'myfile_binary', - None, - fields={'type': 'uploader-v1', - 'is_text': False, - 'encoding': None}) - checker.assert_artifact( - step.name, - 'myfile_overriden', - None, - fields={'type': 'uploader-v1', - 'is_text': True, - 'encoding': None}) - else: - # In the case of the client, we check the value. - for step in flow: - checker.assert_artifact(step.name, 'myfile_txt', - "Regular Text File") - checker.assert_artifact(step.name, 'myfile_utf8', - u"UTF Text File \u5e74") - checker.assert_artifact(step.name, 'myfile_binary', - u"UTF Text File \u5e74".encode(encoding='utf8')) - checker.assert_artifact(step.name, 'myfile_overriden', - "Override Text File") + for step in flow: + checker.assert_artifact( + step.name, + 'myfile_txt', + None, + fields={'type': 'uploader-v1', + 'is_text': True, + 'encoding': None}) + checker.assert_artifact( + step.name, + 'myfile_utf8', + None, + fields={'type': 'uploader-v1', + 'is_text': True, + 'encoding': 'utf8'}) + checker.assert_artifact( + step.name, + 'myfile_binary', + None, + fields={'type': 'uploader-v1', + 'is_text': False, + 'encoding': None}) + checker.assert_artifact( + step.name, + 'myfile_overriden', + None, + fields={'type': 'uploader-v1', + 'is_text': True, + 'encoding': None}) From 8e892455e4c61c6b779ffa098952f341dd18f458 Mon Sep 17 00:00:00 2001 From: Savin Date: Wed, 11 Aug 2021 13:20:05 -0700 Subject: [PATCH 015/176] patch release (#639) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 609fc3adccb..004a29dab81 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '2.3.3' +version = '2.3.4' setup(name='metaflow', version=version, From 9d06447bb7ad85bc082ef04cbcb47b44758be19c Mon Sep 17 00:00:00 2001 From: Romain Date: Mon, 16 Aug 2021 16:06:26 -0700 Subject: [PATCH 016/176] Update Unbounded Foreach Test in the case of a Conda environment (#626) --- metaflow/plugins/test_unbounded_foreach_decorator.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/metaflow/plugins/test_unbounded_foreach_decorator.py b/metaflow/plugins/test_unbounded_foreach_decorator.py index cd6ea91c278..c7693ff039b 100644 --- a/metaflow/plugins/test_unbounded_foreach_decorator.py +++ b/metaflow/plugins/test_unbounded_foreach_decorator.py @@ -69,8 +69,12 @@ def control_task_step_func(self, flow, graph, retry_count): step_name = current.step_name control_task_id = current.task_id (_, split_step_name, split_task_id) = control_task_id.split('-')[1:] - - executable = self.environment.executable(step_name) + # If we are running inside Conda, we use the base executable FIRST; + # the conda environment will then be used when runtime_step_cli is + # called. This is so that it can properly set up all the metaflow + # aliases needed. + env_to_use = getattr(self.environment, 'base_env', self.environment) + executable = env_to_use.executable(step_name) script = sys.argv[0] # Access the `unbounded_foreach` param using `flow` (as datastore). From 7beaff65b38c085a7af2a4b2b73cf82a4135dbc1 Mon Sep 17 00:00:00 2001 From: Romain Date: Mon, 16 Aug 2021 18:01:26 -0700 Subject: [PATCH 017/176] Properly allow full toplevel module overrides in metaflow_custom (#623) * Properly allow full toplevel module overrides in metaflow_custom * Modified test to follow new format * Improve module aliasing for metaflow_custom (#625) --- metaflow/__init__.py | 114 +++++++++++++----- .../core/metaflow_custom/toplevel/__init__.py | 5 - .../core/metaflow_custom/toplevel/toplevel.py | 5 + 3 files changed, 90 insertions(+), 34 deletions(-) create mode 100644 test/core/metaflow_custom/toplevel/toplevel.py diff --git a/metaflow/__init__.py b/metaflow/__init__.py index a221f3b8ba9..53abf5d6eb5 100644 --- a/metaflow/__init__.py +++ b/metaflow/__init__.py @@ -53,8 +53,12 @@ class and related decorators. # Something random so there is no syntax error ModuleSpec = None + class _LazyLoader(object): # This _LazyLoader implements the Importer Protocol defined in PEP 302 + # TODO: Need to move to find_spec, exec_module and create_module as + # find_module and load_module are deprecated + def __init__(self, handled): # Modules directly loaded (this is either new modules or overrides of existing ones) self._handled = handled if handled else {} @@ -63,12 +67,23 @@ def __init__(self, handled): # the over-ridden module self._tempexcluded = set() + # This is used when loading a module alias to load any submodule + self._alias_to_orig = {} + def find_module(self, fullname, path=None): if fullname in self._tempexcluded: return None if fullname in self._handled or \ (fullname.endswith('._orig') and fullname[:-6] in self._handled): return self + name_parts = fullname.split('.') + if len(name_parts) > 1 and name_parts[-1] != '_orig': + # We check if we had an alias created for this module and if so, + # we are going to load it to properly fully create aliases all + # the way down. + parent_name = '.'.join(name_parts[:-1]) + if parent_name in self._alias_to_orig: + return self return None def load_module(self, fullname): @@ -80,27 +95,37 @@ def load_module(self, fullname): "Attempting to load '%s' -- loading shadowed modules in Metaflow " "Custom is only supported in Python 3.4+" % fullname) to_import = self._handled.get(fullname, None) - # We see if we are shadowing an existing module and, if so, we - # will keep track of the module we are shadowing so that it - # may be loaded if needed. We basically will create a ._orig submodule - # of sorts. This functionality only works for Python 3.4+. For anything - # below this, we do not create the _orig module so loading it will - # result in ModuleNotFound - if self._can_handle_orig_module() and not fullname.endswith('._orig'): - try: - # We exclude this module temporarily from what we handle to - # revert back to the non-shadowing mode of import - self._tempexcluded.add(fullname) - spec = importlib.util.find_spec(fullname) - self._handled["%s._orig" % fullname] = spec - finally: - self._tempexcluded.remove(fullname) + + # If to_import is None, two cases: + # - we are loading a ._orig module + # - OR we are loading a submodule + if to_import is None: + if fullname.endswith('._orig'): + try: + # We exclude this module temporarily from what we handle to + # revert back to the non-shadowing mode of import + self._tempexcluded.add(fullname) + to_import = importlib.util.find_spec(fullname) + finally: + self._tempexcluded.remove(fullname) + else: + name_parts = fullname.split('.') + submodule = name_parts[-1] + parent_name = '.'.join(name_parts[:-1]) + to_import = '.'.join( + [self._alias_to_orig[parent_name], submodule]) if isinstance(to_import, str): - to_import = importlib.import_module(to_import) - sys.modules[fullname] = to_import + try: + to_import_mod = importlib.import_module(to_import) + except ImportError: + raise ImportError( + "No module found '%s' (aliasing %s)" % (fullname, to_import)) + sys.modules[fullname] = to_import_mod + self._alias_to_orig[fullname] = to_import_mod.__name__ elif isinstance(to_import, types.ModuleType): sys.modules[fullname] = to_import + self._alias_to_orig[fullname] = to_import.__name__ elif self._can_handle_orig_module() and isinstance(to_import, ModuleSpec): # This loads modules that end in _orig m = importlib.util.module_from_spec(to_import) @@ -122,6 +147,35 @@ def load_module(self, fullname): def _can_handle_orig_module(): return sys.version_info[0] >= 3 and sys.version_info[1] >= 4 +# We load the module overrides *first* explicitly. Non overrides can be loaded +# in toplevel as well but these can be loaded first if needed. Note that those +# modules should be careful not to include anything in Metaflow at their top-level +# as it is likely to not work. +try: + import metaflow_custom.toplevel.module_overrides as extension_module +except ImportError as e: + ver = sys.version_info[0] * 10 + sys.version_info[1] + if ver >= 36: + # e.name is set to the name of the package that fails to load + # so don't error ONLY IF the error is importing this module (but do + # error if there is a transitive import error) + if not (isinstance(e, ModuleNotFoundError) and \ + e.name in ['metaflow_custom', 'metaflow_custom.toplevel', + 'metaflow_custom.toplevel.module_overrides']): + print( + "Cannot load metaflow_custom top-level configuration -- " + "if you want to ignore, uninstall metaflow_custom package") + raise +else: + # We load only modules + lazy_load_custom_modules = {} + for n, o in extension_module.__dict__.items(): + if isinstance(o, types.ModuleType) and o.__package__ and \ + o.__package__.startswith('metaflow_custom'): + lazy_load_custom_modules['metaflow.%s' % n] = o + if lazy_load_custom_modules: + # Prepend to make sure custom package overrides things + sys.meta_path = [_LazyLoader(lazy_load_custom_modules)] + sys.meta_path from .event_logger import EventLogger @@ -162,10 +216,10 @@ def _can_handle_orig_module(): parallel_map from .metaflow_profile import profile - +# Now override everything other than modules __version_addl__ = None try: - import metaflow_custom.toplevel as extension_module + import metaflow_custom.toplevel.toplevel as extension_module except ImportError as e: ver = sys.version_info[0] * 10 + sys.version_info[1] if ver >= 36: @@ -173,7 +227,8 @@ def _can_handle_orig_module(): # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) if not (isinstance(e, ModuleNotFoundError) and \ - e.name in ['metaflow_custom', 'metaflow_custom.toplevel']): + e.name in ['metaflow_custom', 'metaflow_custom.toplevel', + 'metaflow_custom.toplevel.toplevel']): print( "Cannot load metaflow_custom top-level configuration -- " "if you want to ignore, uninstall metaflow_custom package") @@ -199,18 +254,19 @@ def _can_handle_orig_module(): if lazy_load_custom_modules: # Prepend to make sure custom package overrides things sys.meta_path = [_LazyLoader(lazy_load_custom_modules)] + sys.meta_path + __version_addl__ = getattr(extension_module, '__mf_customization__', '') if extension_module.__version__: __version_addl__ = '%s(%s)' % (__version_addl__, extension_module.__version__) -finally: - # Erase all temporary names to avoid leaking things - for _n in ['ver', 'n', 'o', 'e', 'lazy_load_custom_modules', - 'extension_module', 'addl_modules']: - try: - del globals()[_n] - except KeyError: - pass - del globals()['_n'] + +# Erase all temporary names to avoid leaking things +for _n in ['ver', 'n', 'o', 'e', 'lazy_load_custom_modules', + 'extension_module', 'addl_modules']: + try: + del globals()[_n] + except KeyError: + pass +del globals()['_n'] import pkg_resources try: diff --git a/test/core/metaflow_custom/toplevel/__init__.py b/test/core/metaflow_custom/toplevel/__init__.py index f71ff11e0a4..e69de29bb2d 100644 --- a/test/core/metaflow_custom/toplevel/__init__.py +++ b/test/core/metaflow_custom/toplevel/__init__.py @@ -1,5 +0,0 @@ -__mf_customization__ = 'test' - -tl_value = 42 - -__version__ = None \ No newline at end of file diff --git a/test/core/metaflow_custom/toplevel/toplevel.py b/test/core/metaflow_custom/toplevel/toplevel.py new file mode 100644 index 00000000000..f71ff11e0a4 --- /dev/null +++ b/test/core/metaflow_custom/toplevel/toplevel.py @@ -0,0 +1,5 @@ +__mf_customization__ = 'test' + +tl_value = 42 + +__version__ = None \ No newline at end of file From 0d052e848f2a2f574d33319fed574cde0b0e4e38 Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Fri, 20 Aug 2021 08:40:55 -0700 Subject: [PATCH 018/176] allow to mount host volumes in AWS Batch (#640) --- metaflow/plugins/aws/batch/batch.py | 11 +++++---- metaflow/plugins/aws/batch/batch_cli.py | 5 +++- metaflow/plugins/aws/batch/batch_client.py | 24 +++++++++++++++---- metaflow/plugins/aws/batch/batch_decorator.py | 3 ++- .../aws/step_functions/step_functions.py | 3 ++- 5 files changed, 35 insertions(+), 11 deletions(-) diff --git a/metaflow/plugins/aws/batch/batch.py b/metaflow/plugins/aws/batch/batch.py index 9d72b41b828..e887508c389 100644 --- a/metaflow/plugins/aws/batch/batch.py +++ b/metaflow/plugins/aws/batch/batch.py @@ -164,7 +164,8 @@ def create_job( max_swap=None, swappiness=None, env={}, - attrs={} + attrs={}, + host_volumes=None, ): job_name = self._job_name( attrs.get('metaflow.user'), @@ -186,7 +187,7 @@ def create_job( .execution_role(execution_role) \ .job_def(image, iam_role, queue, execution_role, shared_memory, - max_swap, swappiness) \ + max_swap, swappiness, host_volumes=host_volumes) \ .cpu(cpu) \ .gpu(gpu) \ .memory(memory) \ @@ -244,6 +245,7 @@ def launch_job( shared_memory=None, max_swap=None, swappiness=None, + host_volumes=None, env={}, attrs={}, ): @@ -272,8 +274,9 @@ def launch_job( shared_memory, max_swap, swappiness, - env, - attrs + env=env, + attrs=attrs, + host_volumes=host_volumes ) self.job = job.execute() diff --git a/metaflow/plugins/aws/batch/batch_cli.py b/metaflow/plugins/aws/batch/batch_cli.py index 0f80188a23b..7c359f3b538 100644 --- a/metaflow/plugins/aws/batch/batch_cli.py +++ b/metaflow/plugins/aws/batch/batch_cli.py @@ -169,6 +169,7 @@ def kill(ctx, run_id, user, my_runs): @click.option("--swappiness", help="Swappiness requirement for AWS Batch.") #TODO: Maybe remove it altogether since it's not used here @click.option('--ubf-context', default=None, type=click.Choice([None])) +@click.option('--mount-host-volumes', default=None) @click.pass_context def step( ctx, @@ -187,6 +188,7 @@ def step( shared_memory=None, max_swap=None, swappiness=None, + host_volumes=None, **kwargs ): def echo(msg, stream='stderr', batch_id=None): @@ -294,7 +296,8 @@ def echo(msg, stream='stderr', batch_id=None): max_swap=max_swap, swappiness=swappiness, env=env, - attrs=attrs + attrs=attrs, + host_volumes=host_volumes, ) except Exception as e: print(e) diff --git a/metaflow/plugins/aws/batch/batch_client.py b/metaflow/plugins/aws/batch/batch_client.py index c4c1ed65334..80229802671 100644 --- a/metaflow/plugins/aws/batch/batch_client.py +++ b/metaflow/plugins/aws/batch/batch_client.py @@ -101,7 +101,8 @@ def _register_job_definition(self, execution_role, shared_memory, max_swap, - swappiness): + swappiness, + host_volumes): # identify platform from any compute environment associated with the # queue if AWS_SANDBOX_ENABLED: @@ -189,6 +190,19 @@ def _register_job_definition(self, job_definition['containerProperties'] \ ['linuxParameters']['maxSwap'] = int(max_swap) + if host_volumes: + volume_paths = host_volumes.split(',') + job_definition['containerProperties']['volumes'] = [] + job_definition['containerProperties']['mountPoints'] = [] + for host_path in volume_paths: + name = host_path.replace('/', '_') + job_definition['containerProperties']['volumes'].append( + {'name': name, 'host': {'sourcePath': host_path}} + ) + job_definition['containerProperties']['mountPoints'].append( + {"sourceVolume": name, "containerPath": host_path} + ) + # check if job definition already exists def_name = 'metaflow_%s' % \ hashlib.sha224(str(job_definition).encode('utf-8')).hexdigest() @@ -219,7 +233,8 @@ def job_def(self, execution_role, shared_memory, max_swap, - swappiness): + swappiness, + host_volumes): self.payload['jobDefinition'] = \ self._register_job_definition(image, iam_role, @@ -227,7 +242,8 @@ def job_def(self, execution_role, shared_memory, max_swap, - swappiness) + swappiness, + host_volumes) return self def job_name(self, job_name): @@ -629,4 +645,4 @@ def _fill_buf(self): events = self._get_events() for event in events: self._buf.append(event['message']) - self._pos = event['timestamp'] \ No newline at end of file + self._pos = event['timestamp'] diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index 1be46d1ed72..389ee490a2a 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -99,7 +99,8 @@ def my_step(self): 'execution_role': ECS_FARGATE_EXECUTION_ROLE, 'shared_memory': None, 'max_swap': None, - 'swappiness': None + 'swappiness': None, + 'host_volumes': None, } package_url = None package_sha = None diff --git a/metaflow/plugins/aws/step_functions/step_functions.py b/metaflow/plugins/aws/step_functions/step_functions.py index 167cd96fcef..0b489c30c3d 100644 --- a/metaflow/plugins/aws/step_functions/step_functions.py +++ b/metaflow/plugins/aws/step_functions/step_functions.py @@ -613,7 +613,8 @@ def _batch(self, node): max_swap=resources['max_swap'], swappiness=resources['swappiness'], env=env, - attrs=attrs + attrs=attrs, + host_volumes=resources['host_volumes'], ) \ .attempts(total_retries + 1) From 11466d6362f552486e46c1bd9531d1bee77d4317 Mon Sep 17 00:00:00 2001 From: Savin Date: Mon, 23 Aug 2021 14:26:40 -0700 Subject: [PATCH 019/176] Handle parameters as `foreach vars` (#652) * Handle parameters as `foreach vars` * Change fix to work more upstream The reason _find_input was not seeing the proper value for the parameters was because _init_parameters, which is supposed to initialize them, calls _get_parameters which needlessly calls input which calls _find_input. This removes the needless call to input which solves the problem and has the added benefit of being slightly more efficient. Co-authored-by: Romain Cledat --- metaflow/flowspec.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/metaflow/flowspec.py b/metaflow/flowspec.py index 57862992c45..0757ca81dd6 100644 --- a/metaflow/flowspec.py +++ b/metaflow/flowspec.py @@ -43,6 +43,7 @@ class FlowSpec(object): # Name starting with '__', methods, functions and Parameters do not need # to be listed. _EPHEMERAL = {'_EPHEMERAL', + '_NON_PARAMETERS', '_datastore', '_cached_input', '_graph', @@ -50,6 +51,11 @@ class FlowSpec(object): '_steps', 'index', 'input'} + # When checking for parameters, we look at dir(self) but we want to exclude + # attributes that are definitely not parameters and may be expensive to + # compute (like anything related to the `foreach_stack`). We don't need to exclude + # names starting with `_` as those are already excluded from `_get_parameters`. + _NON_PARAMETERS = {'cmd', 'foreach_stack', 'index', 'input', 'script_name', 'name'} _flow_decorators = {} @@ -95,7 +101,7 @@ def script_name(self): def _get_parameters(self): for var in dir(self): - if var[0] == '_': + if var[0] == '_' or var in self._NON_PARAMETERS: continue try: val = getattr(self, var) @@ -223,7 +229,6 @@ def nest_2(self): for i, frame in enumerate(self._foreach_stack)] def _find_input(self, stack_index=None): - if stack_index is None: stack_index = len(self._foreach_stack) - 1 From 2d5000c201ea0c7c3651018e560718c2243f9779 Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Mon, 23 Aug 2021 14:27:52 -0700 Subject: [PATCH 020/176] host_volumes for batch now takes an array (#655) --- metaflow/plugins/aws/batch/batch_cli.py | 2 +- metaflow/plugins/aws/batch/batch_client.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/metaflow/plugins/aws/batch/batch_cli.py b/metaflow/plugins/aws/batch/batch_cli.py index 7c359f3b538..3195d9051ed 100644 --- a/metaflow/plugins/aws/batch/batch_cli.py +++ b/metaflow/plugins/aws/batch/batch_cli.py @@ -169,7 +169,7 @@ def kill(ctx, run_id, user, my_runs): @click.option("--swappiness", help="Swappiness requirement for AWS Batch.") #TODO: Maybe remove it altogether since it's not used here @click.option('--ubf-context', default=None, type=click.Choice([None])) -@click.option('--mount-host-volumes', default=None) +@click.option('--host-volumes', multiple=True) @click.pass_context def step( ctx, diff --git a/metaflow/plugins/aws/batch/batch_client.py b/metaflow/plugins/aws/batch/batch_client.py index 80229802671..c7cde40190c 100644 --- a/metaflow/plugins/aws/batch/batch_client.py +++ b/metaflow/plugins/aws/batch/batch_client.py @@ -191,10 +191,9 @@ def _register_job_definition(self, ['linuxParameters']['maxSwap'] = int(max_swap) if host_volumes: - volume_paths = host_volumes.split(',') job_definition['containerProperties']['volumes'] = [] job_definition['containerProperties']['mountPoints'] = [] - for host_path in volume_paths: + for host_path in host_volumes: name = host_path.replace('/', '_') job_definition['containerProperties']['volumes'].append( {'name': name, 'host': {'sourcePath': host_path}} From be03f610eb1604305397d9dbdcdc351e7736dda0 Mon Sep 17 00:00:00 2001 From: Jeff Raubitschek Date: Mon, 23 Aug 2021 14:32:01 -0700 Subject: [PATCH 021/176] Do not override cov params if rcfile is set (#653) --- metaflow/cli.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metaflow/cli.py b/metaflow/cli.py index bee907b098f..c5addef2aed 100644 --- a/metaflow/cli.py +++ b/metaflow/cli.py @@ -1,4 +1,5 @@ import inspect +import os import sys import traceback from datetime import datetime @@ -845,10 +846,11 @@ def start(ctx, if coverage: from coverage import Coverage + no_covrc = "COVERAGE_RCFILE" not in os.environ cov = Coverage(data_suffix=True, auto_data=True, - source=['metaflow'], - branch=True) + source=['metaflow'] if no_covrc else None, + branch=True if no_covrc else None) cov.start() cli_args._set_top_kwargs(ctx.params) From c487951b182e71b9a9adda92c66b393d99b69e6f Mon Sep 17 00:00:00 2001 From: Savin Date: Mon, 23 Aug 2021 14:33:12 -0700 Subject: [PATCH 022/176] Patch release - 2.3.5 (#656) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 004a29dab81..fc95704931a 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '2.3.4' +version = '2.3.5' setup(name='metaflow', version=version, From fbd6e6a593775c4ab7d51a3f95b453be56232b56 Mon Sep 17 00:00:00 2001 From: Savin Date: Mon, 23 Aug 2021 15:15:57 -0700 Subject: [PATCH 023/176] Update typo in documentation (#657) --- docs/concurrency.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concurrency.md b/docs/concurrency.md index c9c98074dd4..841268eb9e4 100644 --- a/docs/concurrency.md +++ b/docs/concurrency.md @@ -76,7 +76,7 @@ subcommand is also used to clone many datastores concurrently during #### How to Observe -Set the environment variable `METAFLOW_DEBUG_SUBPROCESS=1` to see the +Set the environment variable `METAFLOW_DEBUG_SUBCOMMAND=1` to see the exact command line that is used to launch a subcommand task. You can re-execute the task simply by re-executing the command line manually. However, be careful when re-executing commands from real runs, as you From 53eb56db015332ae0db72a046a0bf2e6f4f1951d Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Tue, 24 Aug 2021 00:27:38 -0700 Subject: [PATCH 024/176] Add data tests script (#658) --- .github/workflows/s3_test.yml | 40 +++++++++++++++++++++++++++ metaflow/datatools/s3util.py | 52 +++++++++++++++++++++++++++++++++++ test/data/s3/s3_data.py | 8 +++--- test/data/s3/test_s3.py | 5 ---- 4 files changed, 96 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/s3_test.yml create mode 100644 metaflow/datatools/s3util.py diff --git a/.github/workflows/s3_test.yml b/.github/workflows/s3_test.yml new file mode 100644 index 00000000000..5922d2cb70a --- /dev/null +++ b/.github/workflows/s3_test.yml @@ -0,0 +1,40 @@ +name: pr-datastore-tests + +on: + pull_request_target: + types: + - opened + - synchronize + - labeled + +jobs: + test_data: + if: (github.event.action == 'labeled' && (github.event.label.name == 'approved' || github.event.label.name == 'ok-to-test')) || (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved'))) + name: S3 data tests on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest] + + steps: + - uses: actions/checkout@v2 + with: + ref: refs/pull/${{ github.event.pull_request.number }}/merge + submodules: recursive + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Install Python 3.x dependencies + run: | + python3 -m pip install --upgrade pip + python3 -m pip install tox numpy pytest click boto3 requests pylint pytest-benchmark + - name: Execute Data tests + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + METAFLOW_S3_TEST_ROOT: ${{ secrets.METAFLOW_S3_TEST_ROOT }} + run: | + cd test/data + PYTHONPATH=$(pwd)/../../ python3 -m pytest --benchmark-skip -x -s -v \ No newline at end of file diff --git a/metaflow/datatools/s3util.py b/metaflow/datatools/s3util.py new file mode 100644 index 00000000000..749c97f49f3 --- /dev/null +++ b/metaflow/datatools/s3util.py @@ -0,0 +1,52 @@ +from __future__ import print_function +import random +import time +import sys +import os + +from metaflow.exception import MetaflowException +from metaflow.metaflow_config import S3_ENDPOINT_URL, S3_VERIFY_CERTIFICATE + +S3_NUM_RETRIES = 7 + + +TEST_S3_RETRY = 'TEST_S3_RETRY' in os.environ + +def get_s3_client(): + from metaflow.plugins.aws.aws_client import get_aws_client + return get_aws_client( + 's3', + with_error=True, + params={'endpoint_url': S3_ENDPOINT_URL, 'verify': S3_VERIFY_CERTIFICATE }) + +# decorator to retry functions that access S3 +def aws_retry(f): + def retry_wrapper(self, *args, **kwargs): + last_exc = None + for i in range(S3_NUM_RETRIES): + try: + ret = f(self, *args, **kwargs) + if TEST_S3_RETRY and i == 0: + raise Exception("TEST_S3_RETRY env var set. " + "Pretending that an S3 op failed. " + "This is not a real failure.") + else: + return ret + except MetaflowException as ex: + # MetaflowExceptions are not related to AWS, don't retry + raise + except Exception as ex: + try: + function_name = f.func_name + except AttributeError: + function_name = f.__name__ + sys.stderr.write("S3 datastore operation %s failed (%s). " + "Retrying %d more times..\n" + % (function_name, ex, S3_NUM_RETRIES - i)) + self.reset_client(hard_reset=True) + last_exc = ex + # exponential backoff for real failures + if not (TEST_S3_RETRY and i == 0): + time.sleep(2**i + random.randint(0, 5)) + raise last_exc + return retry_wrapper diff --git a/test/data/s3/s3_data.py b/test/data/s3/s3_data.py index 74009d53f3c..9fee0314e4f 100644 --- a/test/data/s3/s3_data.py +++ b/test/data/s3/s3_data.py @@ -415,16 +415,16 @@ def _do_upload(prefix, filespecs, meta=None): # For metadata, we don't actually touch RandomFile # (since it is the same) but we modify the path to post-pend # the name - print('Case %s: %s started' % (prefix, f.url)) + print('Test data case %s: upload to %s started' % (prefix, f.url)) s3client.upload_fileobj(f.fileobj(), url.netloc, url.path.lstrip('/')) - print('Case %s: %s added' % (prefix, f.url)) + print('Test data case %s: uploaded to %s' % (prefix, f.url)) if meta is not None: for metaname, metainfo in meta.items(): new_url = "%s_%s" % (f.url, metaname) url = urlparse(new_url) - print('Case %s: %s started' % (prefix, new_url)) + print('Test data case %s: upload to %s started' % (prefix, new_url)) extra = {} content_type, user_meta = metainfo if content_type: @@ -437,7 +437,7 @@ def _do_upload(prefix, filespecs, meta=None): url.netloc, url.path.lstrip('/'), ExtraArgs=extra) - print('Case %s: %s added' % (prefix, new_url)) + print('Test data case %s: uploaded to %s' % (prefix, new_url)) for prefix, filespecs in BIG_DATA + FAKE_RUN_DATA: _do_upload(prefix, filespecs) diff --git a/test/data/s3/test_s3.py b/test/data/s3/test_s3.py index 263ec63bf1c..d30f607f0d4 100644 --- a/test/data/s3/test_s3.py +++ b/test/data/s3/test_s3.py @@ -330,11 +330,6 @@ def test_init_options(s3root, pathspecs, expected): assert_results(s3objs, expected) assert_results(s3.get_all(), expected, info_should_be_empty=True) - # option 5) run object - namespace(None) - with S3(bucket=parsed.netloc, prefix=parsed.path, run=Run(pathspec)) as s3: - names = [url.split('/')[-1] for url in expected] - assert_results(s3.get_many(names), expected) @pytest.mark.parametrize( argnames=['s3root', 'prefixes', 'expected'], From a3b72acf5b89b6566b905db745462f90bf441ee1 Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Tue, 24 Aug 2021 20:52:55 -0700 Subject: [PATCH 025/176] Add testing context for aws bat ch (#669) --- test/core/contexts.json | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/core/contexts.json b/test/core/contexts.json index cedeac3c36a..6d0e79b238c 100644 --- a/test/core/contexts.json +++ b/test/core/contexts.json @@ -91,6 +91,40 @@ "disabled_tests": [ "S3FailureTest" ] + }, + { + "name": "python3-batch", + "disabled": true, + "python": "python3", + "top_options": [ + "--event-logger=nullSidecarLogger", + "--no-pylint", + "--quiet", + "--with=batch", + "--datastore=s3" + ], + "env": { + "METAFLOW_USER": "tester", + "METAFLOW_RUN_BOOL_PARAM": "False", + "METAFLOW_RUN_NO_DEFAULT_PARAM": "test_str", + "METAFLOW_DEFAULT_METADATA": "service" + }, + "run_options": [ + "--max-workers", "50", + "--max-num-splits", "10000", + "--tag", "\u523a\u8eab means sashimi", + "--tag", "multiple tags should be ok" + ], + "checks": ["python3-cli", "python3-metadata"], + "disabled_tests": [ + "LargeArtifactTest", + "WideForeachTest", + "TagCatchTest", + "BasicUnboundedForeachTest", + "NestedUnboundedForeachTest", + "DetectSegFaultTest", + "TimeoutDecoratorTest" + ] } ], "checks": { From c1af4c7cc7826a88930b571bf12ebe1454160433 Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Tue, 31 Aug 2021 16:38:58 -0700 Subject: [PATCH 026/176] allow dots in host_volumes (#676) --- metaflow/plugins/aws/batch/batch_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/plugins/aws/batch/batch_client.py b/metaflow/plugins/aws/batch/batch_client.py index c7cde40190c..b40997258c2 100644 --- a/metaflow/plugins/aws/batch/batch_client.py +++ b/metaflow/plugins/aws/batch/batch_client.py @@ -194,7 +194,7 @@ def _register_job_definition(self, job_definition['containerProperties']['volumes'] = [] job_definition['containerProperties']['mountPoints'] = [] for host_path in host_volumes: - name = host_path.replace('/', '_') + name = host_path.replace('/', '_').replace('.', '_') job_definition['containerProperties']['volumes'].append( {'name': name, 'host': {'sourcePath': host_path}} ) From 6aa9a7512ba4796e77153832adb20be6c41f976a Mon Sep 17 00:00:00 2001 From: bishax Date: Wed, 1 Sep 2021 00:41:00 +0100 Subject: [PATCH 027/176] Conda environment now applys underlying environment's decorators (#660) * Conda environment now applys underlying environment's decorators * Add tuples instead of unpacking with * for py2 compatibility --- metaflow/plugins/conda/conda_environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metaflow/plugins/conda/conda_environment.py b/metaflow/plugins/conda/conda_environment.py index 125afdadb20..f7ec0928e92 100644 --- a/metaflow/plugins/conda/conda_environment.py +++ b/metaflow/plugins/conda/conda_environment.py @@ -36,8 +36,8 @@ def validate_environment(self, echo): return self.base_env.validate_environment(echo) def decospecs(self): - # Apply conda decorator to all steps - return ('conda', ) + # Apply conda decorator and base environment's decorators to all steps + return ('conda',) + self.base_env.decospecs() def _get_conda_decorator(self, step_name): step = next(step for step in self.flow if step.name == step_name) From d453859d74db3d06f7e874da6978eb8954204d39 Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Wed, 1 Sep 2021 16:03:04 -0700 Subject: [PATCH 028/176] Representation change for `task.exception` (#679) * Added test for exception handling. - Added Repr to fix error representation problem * Fix test doc string * nit fixes. --- metaflow/exception.py | 3 +++ test/core/tests/task_exception.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 test/core/tests/task_exception.py diff --git a/metaflow/exception.py b/metaflow/exception.py index e4124db4c68..cc862990e94 100644 --- a/metaflow/exception.py +++ b/metaflow/exception.py @@ -30,6 +30,9 @@ def __getstate__(self): def __setstate__(self, state): self.__dict__ = state + + def __repr__(self): + return str(self) def __str__(self): if self.stacktrace: diff --git a/test/core/tests/task_exception.py b/test/core/tests/task_exception.py new file mode 100644 index 00000000000..d50c8c998fd --- /dev/null +++ b/test/core/tests/task_exception.py @@ -0,0 +1,25 @@ +from metaflow_test import MetaflowTest, ExpectationFailed, steps + + + +class TaskExceptionTest(MetaflowTest): + """ + A test to validate if exceptions are stored and retrieved correctly + """ + PRIORITY = 1 + SHOULD_FAIL = True + + @steps(0, ['singleton-end'], required=True) + def step_start(self): + raise KeyError('Something has gone wrong') + + @steps(2, ['all']) + def step_all(self): + pass + + def check_results(self, flow, checker): + run = checker.get_run() + if run is not None: + for task in run['end']: + assert_equals('KeyError' in str(task.exception), True) + assert_equals(task.exception.exception,"'Something has gone wrong'") From cdd58cfa39eca8be0a59a471923f277ce909b7a3 Mon Sep 17 00:00:00 2001 From: bishax Date: Tue, 7 Sep 2021 23:54:09 +0100 Subject: [PATCH 029/176] Fixes recursion error when METAFLOW_DEFAULT_ENVIRONMENT=conda (#674) * Fixes recursion error when METAFLOW_DEFAULT_ENVIRONMENT=conda * Add explanation --- metaflow/plugins/conda/conda_environment.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/metaflow/plugins/conda/conda_environment.py b/metaflow/plugins/conda/conda_environment.py index f7ec0928e92..faa15d08c89 100644 --- a/metaflow/plugins/conda/conda_environment.py +++ b/metaflow/plugins/conda/conda_environment.py @@ -23,8 +23,14 @@ def __init__(self, flow): # any calls we don't handle specifically to that one. from ...plugins import ENVIRONMENTS from metaflow.metaflow_config import DEFAULT_ENVIRONMENT - self.base_env = [e for e in ENVIRONMENTS + [MetaflowEnvironment] - if e.TYPE == DEFAULT_ENVIRONMENT][0](self.flow) + + if DEFAULT_ENVIRONMENT == self.TYPE: + # If the default environment is Conda itself then fallback on + # the default 'default environment' + self.base_env = MetaflowEnvironment(self.flow) + else: + self.base_env = [e for e in ENVIRONMENTS + [MetaflowEnvironment] + if e.TYPE == DEFAULT_ENVIRONMENT][0](self.flow) def init_environment(self, echo): # Print a message for now From 78b7ec34bd5796b16c9853a3fba8c482221aa15e Mon Sep 17 00:00:00 2001 From: Savin Date: Wed, 8 Sep 2021 11:28:51 -0700 Subject: [PATCH 030/176] Patch release - 2.3.6 (#687) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fc95704931a..347b7c3f1cb 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '2.3.5' +version = '2.3.6' setup(name='metaflow', version=version, From dc310d032b15d977c6e5e7bfb3f2df96ce8c0d6e Mon Sep 17 00:00:00 2001 From: Savin Date: Thu, 9 Sep 2021 13:30:19 -0700 Subject: [PATCH 031/176] Terminate task-level sidecars after `task_finished` hook has been executed (#689) --- metaflow/task.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metaflow/task.py b/metaflow/task.py index 5f911b13eb2..fa22e6a7629 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -515,10 +515,6 @@ def run_step(self, output.save_metadata('task_end', {}) output.persist(self.flow) - # terminate side cars - logger.terminate() - self.metadata.stop_heartbeat() - # this writes a success marker indicating that the # "transaction" is done output.done() @@ -532,3 +528,7 @@ def run_step(self, self.flow._task_ok, retry_count, max_user_code_retries) + + # terminate side cars + logger.terminate() + self.metadata.stop_heartbeat() \ No newline at end of file From 8cfc01a23aee3fcce45c11700b179f32fa95a985 Mon Sep 17 00:00:00 2001 From: Savin Date: Thu, 9 Sep 2021 13:31:35 -0700 Subject: [PATCH 032/176] Modify datastore availability for `task_pre_step` decorator hook (#688) * Modify datastore availability for `task_pre_step` decorator hook * remove spurious change --- metaflow/task.py | 60 ++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/metaflow/task.py b/metaflow/task.py index fa22e6a7629..98f18f3beeb 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -375,32 +375,6 @@ def run_step(self, # should either be set prior to running the user code or listed in # FlowSpec._EPHEMERAL to allow for proper merging/importing of # user artifacts in the user's step code. - decorators = step_func.decorators - for deco in decorators: - - deco.task_pre_step(step_name, - output, - self.metadata, - run_id, - task_id, - self.flow, - self.flow._graph, - retry_count, - max_user_code_retries, - self.ubf_context, - inputs) - - # decorators can actually decorate the step function, - # or they can replace it altogether. This functionality - # is used e.g. by catch_decorator which switches to a - # fallback code if the user code has failed too many - # times. - step_func = deco.task_decorate(step_func, - self.flow, - self.flow._graph, - retry_count, - max_user_code_retries, - self.ubf_context) if join_type: # Join step: @@ -424,7 +398,6 @@ def run_step(self, # We take Parameter values from the first input, # which is always safe since parameters are read-only current._update_env({'parameter_names': self._init_parameters(inputs[0])}) - self._exec_step_function(step_func, input_obj) else: # Linear step: # We are running with a single input context. @@ -441,6 +414,37 @@ def run_step(self, # We take Parameter values from the first input, # which is always safe since parameters are read-only current._update_env({'parameter_names': self._init_parameters(inputs[0])}) + + decorators = step_func.decorators + for deco in decorators: + + deco.task_pre_step(step_name, + output, + self.metadata, + run_id, + task_id, + self.flow, + self.flow._graph, + retry_count, + max_user_code_retries, + self.ubf_context, + inputs) + + # decorators can actually decorate the step function, + # or they can replace it altogether. This functionality + # is used e.g. by catch_decorator which switches to a + # fallback code if the user code has failed too many + # times. + step_func = deco.task_decorate(step_func, + self.flow, + self.flow._graph, + retry_count, + max_user_code_retries, + self.ubf_context) + + if join_type: + self._exec_step_function(step_func, input_obj) + else: self._exec_step_function(step_func) for deco in decorators: @@ -531,4 +535,4 @@ def run_step(self, # terminate side cars logger.terminate() - self.metadata.stop_heartbeat() \ No newline at end of file + self.metadata.stop_heartbeat() From 4e077744b9785a0ee2364ac16d2d6ab3ec080532 Mon Sep 17 00:00:00 2001 From: Savin Date: Thu, 9 Sep 2021 13:31:51 -0700 Subject: [PATCH 033/176] Fix error message for _version call for metadata service (#690) --- metaflow/plugins/metadata/service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metaflow/plugins/metadata/service.py b/metaflow/plugins/metadata/service.py index 5e6db9eb49b..0bb493eddea 100644 --- a/metaflow/plugins/metadata/service.py +++ b/metaflow/plugins/metadata/service.py @@ -351,14 +351,14 @@ def _version(cls, monitor): elif resp.status_code != 503: raise ServiceException('Metadata request (%s) failed' ' (code %s): %s' % - (path, resp.status_code, resp.text), + (url, resp.status_code, resp.text), resp.status_code, resp.text) time.sleep(2**i) if resp: raise ServiceException('Metadata request (%s) failed (code %s): %s' - % (path, resp.status_code, resp.text), + % (url, resp.status_code, resp.text), resp.status_code, resp.text) else: - raise ServiceException('Metadata request (%s) failed' % path) + raise ServiceException('Metadata request (%s) failed' % url) From 9cafaca9f9160bcf90c949ffb4231481c91f783f Mon Sep 17 00:00:00 2001 From: Savin Date: Thu, 9 Sep 2021 14:25:08 -0700 Subject: [PATCH 034/176] Rename `metaflow_custom` to `metaflow_extensions` (#691) * Change metaflow_custom to metaflow_extensions * Rename metaflow_custom to metaflow_extensions * more changes * fix test --- metaflow/__init__.py | 38 +++++++++---------- metaflow/datatools/__init__.py | 18 ++++----- metaflow/exception.py | 10 ++--- metaflow/metaflow_config.py | 8 ++-- metaflow/package.py | 18 ++++----- metaflow/plugins/__init__.py | 24 ++++++------ .../plugins/conda/conda_step_decorator.py | 6 +-- metaflow/plugins/env_escape/__init__.py | 10 ++--- .../__init__.py | 0 .../config/__init__.py | 0 .../config/metaflow_config.py | 0 .../exceptions/__init__.py | 0 .../plugins/__init__.py | 0 .../plugins/flow_options.py | 0 .../plugins/nondecoplugin/__init__.py | 0 .../plugins/test_step_decorator.py | 0 .../toplevel/__init__.py | 0 .../toplevel/toplevel.py | 2 +- test/core/metaflow_test/formatter.py | 2 +- .../tests/{customization.py => extensions.py} | 2 +- test/core/tests/flow_options.py | 2 +- 21 files changed, 70 insertions(+), 70 deletions(-) rename test/core/{metaflow_custom => metaflow_extensions}/__init__.py (100%) rename test/core/{metaflow_custom => metaflow_extensions}/config/__init__.py (100%) rename test/core/{metaflow_custom => metaflow_extensions}/config/metaflow_config.py (100%) rename test/core/{metaflow_custom => metaflow_extensions}/exceptions/__init__.py (100%) rename test/core/{metaflow_custom => metaflow_extensions}/plugins/__init__.py (100%) rename test/core/{metaflow_custom => metaflow_extensions}/plugins/flow_options.py (100%) rename test/core/{metaflow_custom => metaflow_extensions}/plugins/nondecoplugin/__init__.py (100%) rename test/core/{metaflow_custom => metaflow_extensions}/plugins/test_step_decorator.py (100%) rename test/core/{metaflow_custom => metaflow_extensions}/toplevel/__init__.py (100%) rename test/core/{metaflow_custom => metaflow_extensions}/toplevel/toplevel.py (53%) rename test/core/tests/{customization.py => extensions.py} (93%) diff --git a/metaflow/__init__.py b/metaflow/__init__.py index 53abf5d6eb5..ac138a1fb67 100644 --- a/metaflow/__init__.py +++ b/metaflow/__init__.py @@ -93,7 +93,7 @@ def load_module(self, fullname): # We return a nicer error message raise ImportError( "Attempting to load '%s' -- loading shadowed modules in Metaflow " - "Custom is only supported in Python 3.4+" % fullname) + "Extensions are only supported in Python 3.4+" % fullname) to_import = self._handled.get(fullname, None) # If to_import is None, two cases: @@ -138,7 +138,7 @@ def load_module(self, fullname): # would be OK. Being extra sure in case _LazyLoader is misused and # a None value is passed in. raise ImportError( - "Metaflow Custom shadowed module '%s' does not exist" % fullname) + "Metaflow Extensions shadowed module '%s' does not exist" % fullname) else: raise ImportError return sys.modules[fullname] @@ -152,7 +152,7 @@ def _can_handle_orig_module(): # modules should be careful not to include anything in Metaflow at their top-level # as it is likely to not work. try: - import metaflow_custom.toplevel.module_overrides as extension_module + import metaflow_extensions.toplevel.module_overrides as extension_module except ImportError as e: ver = sys.version_info[0] * 10 + sys.version_info[1] if ver >= 36: @@ -160,21 +160,21 @@ def _can_handle_orig_module(): # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) if not (isinstance(e, ModuleNotFoundError) and \ - e.name in ['metaflow_custom', 'metaflow_custom.toplevel', - 'metaflow_custom.toplevel.module_overrides']): + e.name in ['metaflow_extensions', 'metaflow_extensions.toplevel', + 'metaflow_extensions.toplevel.module_overrides']): print( - "Cannot load metaflow_custom top-level configuration -- " - "if you want to ignore, uninstall metaflow_custom package") + "Cannot load metaflow_extensions top-level configuration -- " + "if you want to ignore, uninstall metaflow_extensions package") raise else: # We load only modules lazy_load_custom_modules = {} for n, o in extension_module.__dict__.items(): if isinstance(o, types.ModuleType) and o.__package__ and \ - o.__package__.startswith('metaflow_custom'): + o.__package__.startswith('metaflow_extensions'): lazy_load_custom_modules['metaflow.%s' % n] = o if lazy_load_custom_modules: - # Prepend to make sure custom package overrides things + # Prepend to make sure extensions package overrides things sys.meta_path = [_LazyLoader(lazy_load_custom_modules)] + sys.meta_path from .event_logger import EventLogger @@ -219,7 +219,7 @@ def _can_handle_orig_module(): # Now override everything other than modules __version_addl__ = None try: - import metaflow_custom.toplevel.toplevel as extension_module + import metaflow_extensions.toplevel.toplevel as extension_module except ImportError as e: ver = sys.version_info[0] * 10 + sys.version_info[1] if ver >= 36: @@ -227,35 +227,35 @@ def _can_handle_orig_module(): # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) if not (isinstance(e, ModuleNotFoundError) and \ - e.name in ['metaflow_custom', 'metaflow_custom.toplevel', - 'metaflow_custom.toplevel.toplevel']): + e.name in ['metaflow_extensions', 'metaflow_extensions.toplevel', + 'metaflow_extensions.toplevel.toplevel']): print( - "Cannot load metaflow_custom top-level configuration -- " - "if you want to ignore, uninstall metaflow_custom package") + "Cannot load metaflow_extensions top-level configuration -- " + "if you want to ignore, uninstall metaflow_extensions package") raise else: # We load into globals whatever we have in extension_module # We specifically exclude any modules that may be included (like sys, os, etc) - # *except* for ones that are part of metaflow_custom (basically providing + # *except* for ones that are part of metaflow_extensions (basically providing # an aliasing mechanism) lazy_load_custom_modules = {} addl_modules = extension_module.__dict__.get('__mf_promote_submodules__') if addl_modules: - # We make an alias for these modules which the metaflow_custom author + # We make an alias for these modules which the metaflow_extensions author # wants to expose but that may not be loaded yet lazy_load_custom_modules = { - 'metaflow.%s' % k: 'metaflow_custom.%s' % k for k in addl_modules} + 'metaflow.%s' % k: 'metaflow_extensions.%s' % k for k in addl_modules} for n, o in extension_module.__dict__.items(): if not n.startswith('__') and not isinstance(o, types.ModuleType): globals()[n] = o elif isinstance(o, types.ModuleType) and o.__package__ and \ - o.__package__.startswith('metaflow_custom'): + o.__package__.startswith('metaflow_extensions'): lazy_load_custom_modules['metaflow.%s' % n] = o if lazy_load_custom_modules: # Prepend to make sure custom package overrides things sys.meta_path = [_LazyLoader(lazy_load_custom_modules)] + sys.meta_path - __version_addl__ = getattr(extension_module, '__mf_customization__', '') + __version_addl__ = getattr(extension_module, '__mf_extensions__', '') if extension_module.__version__: __version_addl__ = '%s(%s)' % (__version_addl__, extension_module.__version__) diff --git a/metaflow/datatools/__init__.py b/metaflow/datatools/__init__.py index 7d951803d85..5fb290219fe 100644 --- a/metaflow/datatools/__init__.py +++ b/metaflow/datatools/__init__.py @@ -3,9 +3,9 @@ from .s3 import MetaflowS3Exception, S3 -# Import any additional datatools defined by a Metaflow custom package +# Import any additional datatools defined by a Metaflow extensions package try: - import metaflow_custom.datatools as extension_module + import metaflow_extensions.datatools as extension_module except ImportError as e: ver = sys.version_info[0] * 10 + sys.version_info[1] if ver >= 36: @@ -13,29 +13,29 @@ # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) if not (isinstance(e, ModuleNotFoundError) and \ - e.name in ['metaflow_custom', 'metaflow_custom.datatools']): + e.name in ['metaflow_extensions', 'metaflow_extensions.datatools']): print( - "Cannot load metaflow_custom exceptions -- " - "if you want to ignore, uninstall metaflow_custom package") + "Cannot load metaflow_extensions exceptions -- " + "if you want to ignore, uninstall metaflow_extensions package") raise else: # We load into globals whatever we have in extension_module # We specifically exclude any modules that may be included (like sys, os, etc) - # *except* for ones that are part of metaflow_custom (basically providing + # *except* for ones that are part of metaflow_extensions (basically providing # an aliasing mechanism) lazy_load_custom_modules = {} addl_modules = extension_module.__dict__.get('__mf_promote_submodules__') if addl_modules: - # We make an alias for these modules which the metaflow_custom author + # We make an alias for these modules which the metaflow_extensions author # wants to expose but that may not be loaded yet lazy_load_custom_modules = { - 'metaflow.datatools.%s' % k: 'metaflow_custom.datatools.%s' % k + 'metaflow.datatools.%s' % k: 'metaflow_extensions.datatools.%s' % k for k in addl_modules} for n, o in extension_module.__dict__.items(): if not n.startswith('__') and not isinstance(o, types.ModuleType): globals()[n] = o elif isinstance(o, types.ModuleType) and o.__package__ and \ - o.__package__.startswith('metaflow_custom'): + o.__package__.startswith('metaflow_extensions'): lazy_load_custom_modules['metaflow.datatools.%s' % n] = o if lazy_load_custom_modules: from metaflow import _LazyLoader diff --git a/metaflow/exception.py b/metaflow/exception.py index cc862990e94..24cfcd6378d 100644 --- a/metaflow/exception.py +++ b/metaflow/exception.py @@ -123,9 +123,9 @@ def __init__(self, msg, unhandled): super(MissingInMergeArtifactsException, self).__init__(msg) self.artifact_names = unhandled -# Import any exceptions defined by a Metaflow custom package +# Import any exceptions defined by a Metaflow extensions package try: - import metaflow_custom.exceptions as extension_module + import metaflow_extensions.exceptions as extension_module except ImportError as e: ver = sys.version_info[0] * 10 + sys.version_info[1] if ver >= 36: @@ -133,10 +133,10 @@ def __init__(self, msg, unhandled): # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) if not (isinstance(e, ModuleNotFoundError) and \ - e.name in ['metaflow_custom', 'metaflow_custom.exceptions']): + e.name in ['metaflow_extensions', 'metaflow_extensions.exceptions']): print( - "Cannot load metaflow_custom exceptions -- " - "if you want to ignore, uninstall metaflow_custom package") + "Cannot load metaflow_extensions exceptions -- " + "if you want to ignore, uninstall metaflow_extensions package") raise else: # We load into globals whatever we have in extension_module diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py index befa3013682..8ce7738b3ce 100644 --- a/metaflow/metaflow_config.py +++ b/metaflow/metaflow_config.py @@ -228,7 +228,7 @@ def get_pinned_conda_libs(python_version): # Check if there is a an extension to Metaflow to load and override everything try: - import metaflow_custom.config.metaflow_config as extension_module + import metaflow_extensions.config.metaflow_config as extension_module except ImportError as e: ver = sys.version_info[0] * 10 + sys.version_info[1] if ver >= 36: @@ -236,10 +236,10 @@ def get_pinned_conda_libs(python_version): # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) if not (isinstance(e, ModuleNotFoundError) and \ - e.name in ['metaflow_custom', 'metaflow_custom.config']): + e.name in ['metaflow_extensions', 'metaflow_extensions.config']): print( - "Cannot load metaflow_custom configuration -- " - "if you want to ignore, uninstall metaflow_custom package") + "Cannot load metaflow_extensions configuration -- " + "if you want to ignore, uninstall metaflow_extensions package") raise else: # We load into globals whatever we have in extension_module diff --git a/metaflow/package.py b/metaflow/package.py index 7887aff6d01..105d7c35a10 100644 --- a/metaflow/package.py +++ b/metaflow/package.py @@ -20,14 +20,14 @@ def __init__(self, flow, environment, echo, suffixes=DEFAULT_SUFFIXES_LIST): self.environment = environment self.metaflow_root = os.path.dirname(__file__) try: - import metaflow_custom + import metaflow_extensions except ImportError: - self.metaflow_custom_root = None + self.metaflow_extensions_root = None else: - self.metaflow_custom_root = os.path.dirname(metaflow_custom.__file__) - self.metaflow_custom_addl_suffixes = getattr( - metaflow_custom, - 'METAFLOW_CUSTOM_PACKAGE_SUFFIXES', + self.metaflow_extensions_root = os.path.dirname(metaflow_extensions.__file__) + self.metaflow_extensions_addl_suffixes = getattr( + metaflow_extensions, + 'METAFLOW_EXTENSIONS_PACKAGE_SUFFIXES', None) environment.init_environment(echo) @@ -66,11 +66,11 @@ def path_tuples(self): for path_tuple in self._walk(self.metaflow_root, exclude_hidden=False): yield path_tuple # Metaflow customization if any - if self.metaflow_custom_root: + if self.metaflow_extensions_root: for path_tuple in self._walk( - self.metaflow_custom_root, + self.metaflow_extensions_root, exclude_hidden=False, - addl_suffixes=self.metaflow_custom_addl_suffixes): + addl_suffixes=self.metaflow_extensions_addl_suffixes): yield path_tuple # the package folders for environment for path_tuple in self.environment.add_to_package(): diff --git a/metaflow/plugins/__init__.py b/metaflow/plugins/__init__.py index 34912372e79..5c62fc43d01 100644 --- a/metaflow/plugins/__init__.py +++ b/metaflow/plugins/__init__.py @@ -14,7 +14,7 @@ } try: - import metaflow_custom.plugins as _ext_plugins + import metaflow_extensions.plugins as _ext_plugins except ImportError as e: ver = sys.version_info[0] * 10 + sys.version_info[1] if ver >= 36: @@ -22,10 +22,10 @@ # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) if not (isinstance(e, ModuleNotFoundError) and \ - e.name in ['metaflow_custom', 'metaflow_custom.plugins']): + e.name in ['metaflow_extensions', 'metaflow_extensions.plugins']): print( - "Cannot load metaflow_custom plugins -- " - "if you want to ignore, uninstall metaflow_custom package") + "Cannot load metaflow_extensions plugins -- " + "if you want to ignore, uninstall metaflow_extensions package") raise class _fake(object): def __getattr__(self, name): @@ -37,31 +37,31 @@ def __getattr__(self, name): else: # We load into globals whatever we have in extension_module # We specifically exclude any modules that may be included (like sys, os, etc) - # *except* for ones that are part of metaflow_custom (basically providing + # *except* for ones that are part of metaflow_extensions (basically providing # an aliasing mechanism) lazy_load_custom_modules = {} addl_modules = _ext_plugins.__dict__.get('__mf_promote_submodules__') if addl_modules: - # We make an alias for these modules which the metaflow_custom author + # We make an alias for these modules which the metaflow_extensions author # wants to expose but that may not be loaded yet lazy_load_custom_modules = { - 'metaflow.plugins.%s' % k: 'metaflow_custom.plugins.%s' % k + 'metaflow.plugins.%s' % k: 'metaflow_extensions.plugins.%s' % k for k in addl_modules} for n, o in _ext_plugins.__dict__.items(): if not n.startswith('__') and not isinstance(o, types.ModuleType): globals()[n] = o elif isinstance(o, types.ModuleType) and o.__package__ and \ - o.__package__.startswith('metaflow_custom'): + o.__package__.startswith('metaflow_extensions'): lazy_load_custom_modules['metaflow.plugins.%s' % n] = o if lazy_load_custom_modules: - # NOTE: We load things first to have metaflow_custom override things here. + # NOTE: We load things first to have metaflow_extensions override things here. # This does mean that for modules that have the same name (for example, - # if metaflow_custom.plugins also provides a conda module), it needs + # if metaflow_extensions.plugins also provides a conda module), it needs # to provide whatever is expected below (so for example a `conda_step_decorator` # file with a `CondaStepDecorator` class). - # We do this because we want metaflow_custom to fully override things + # We do this because we want metaflow_extensions to fully override things # and if we did not change sys.meta_path here, the lines below would - # load the non metaflow_custom modules providing for possible confusion. + # load the non metaflow_extensions modules providing for possible confusion. # This keeps it cleaner. from metaflow import _LazyLoader sys.meta_path = [_LazyLoader(lazy_load_custom_modules)] + sys.meta_path diff --git a/metaflow/plugins/conda/conda_step_decorator.py b/metaflow/plugins/conda/conda_step_decorator.py index 5b68738c73c..93e42e26dbd 100644 --- a/metaflow/plugins/conda/conda_step_decorator.py +++ b/metaflow/plugins/conda/conda_step_decorator.py @@ -219,9 +219,9 @@ def runtime_init(self, flow, graph, package, run_id): self.metaflow_home = tempfile.mkdtemp(dir='/tmp') self.addl_paths = None os.symlink(path_to_metaflow, os.path.join(self.metaflow_home, 'metaflow')) - # Do the same for metaflow_custom + # Do the same for metaflow_extensions try: - import metaflow_custom as m + import metaflow_extensions as m except ImportError: # No additional check needed because if we are here, we already checked # for other issues when loading at the toplevel @@ -230,7 +230,7 @@ def runtime_init(self, flow, graph, package, run_id): custom_paths = list(m.__path__) if len(custom_paths) == 1: # Regular package - os.symlink(custom_paths[0], os.path.join(self.metaflow_home, 'metaflow_custom')) + os.symlink(custom_paths[0], os.path.join(self.metaflow_home, 'metaflow_extensions')) else: # Namespace package; we don't symlink but add the additional paths # for the conda interpreter diff --git a/metaflow/plugins/env_escape/__init__.py b/metaflow/plugins/env_escape/__init__.py index 37554375d8c..07ee682685b 100644 --- a/metaflow/plugins/env_escape/__init__.py +++ b/metaflow/plugins/env_escape/__init__.py @@ -66,7 +66,7 @@ def generate_trampolines(python_path): paths = [os.path.dirname(os.path.abspath(__file__)) + "/configurations"] try: - import metaflow_custom.plugins.env_escape as custom_escape + import metaflow_extensions.plugins.env_escape as custom_escape except ImportError as e: ver = sys.version_info[0] * 10 + sys.version_info[1] if ver >= 36: @@ -74,11 +74,11 @@ def generate_trampolines(python_path): # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) if not (isinstance(e, ModuleNotFoundError) and e.name in [ - 'metaflow_custom', 'metaflow_custom.plugins', - 'metaflow_custom.plugins.env_escape']): + 'metaflow_extensions', 'metaflow_extensions.plugins', + 'metaflow_extensions.plugins.env_escape']): print( - "Cannot load metaflow_custom env escape configurations -- " - "if you want to ignore, uninstall metaflow_custom package") + "Cannot load metaflow_extensions env escape configurations -- " + "if you want to ignore, uninstall metaflow_extensions package") raise else: paths.append(os.path.dirname(os.path.abspath(custom_escape.__file__)) + diff --git a/test/core/metaflow_custom/__init__.py b/test/core/metaflow_extensions/__init__.py similarity index 100% rename from test/core/metaflow_custom/__init__.py rename to test/core/metaflow_extensions/__init__.py diff --git a/test/core/metaflow_custom/config/__init__.py b/test/core/metaflow_extensions/config/__init__.py similarity index 100% rename from test/core/metaflow_custom/config/__init__.py rename to test/core/metaflow_extensions/config/__init__.py diff --git a/test/core/metaflow_custom/config/metaflow_config.py b/test/core/metaflow_extensions/config/metaflow_config.py similarity index 100% rename from test/core/metaflow_custom/config/metaflow_config.py rename to test/core/metaflow_extensions/config/metaflow_config.py diff --git a/test/core/metaflow_custom/exceptions/__init__.py b/test/core/metaflow_extensions/exceptions/__init__.py similarity index 100% rename from test/core/metaflow_custom/exceptions/__init__.py rename to test/core/metaflow_extensions/exceptions/__init__.py diff --git a/test/core/metaflow_custom/plugins/__init__.py b/test/core/metaflow_extensions/plugins/__init__.py similarity index 100% rename from test/core/metaflow_custom/plugins/__init__.py rename to test/core/metaflow_extensions/plugins/__init__.py diff --git a/test/core/metaflow_custom/plugins/flow_options.py b/test/core/metaflow_extensions/plugins/flow_options.py similarity index 100% rename from test/core/metaflow_custom/plugins/flow_options.py rename to test/core/metaflow_extensions/plugins/flow_options.py diff --git a/test/core/metaflow_custom/plugins/nondecoplugin/__init__.py b/test/core/metaflow_extensions/plugins/nondecoplugin/__init__.py similarity index 100% rename from test/core/metaflow_custom/plugins/nondecoplugin/__init__.py rename to test/core/metaflow_extensions/plugins/nondecoplugin/__init__.py diff --git a/test/core/metaflow_custom/plugins/test_step_decorator.py b/test/core/metaflow_extensions/plugins/test_step_decorator.py similarity index 100% rename from test/core/metaflow_custom/plugins/test_step_decorator.py rename to test/core/metaflow_extensions/plugins/test_step_decorator.py diff --git a/test/core/metaflow_custom/toplevel/__init__.py b/test/core/metaflow_extensions/toplevel/__init__.py similarity index 100% rename from test/core/metaflow_custom/toplevel/__init__.py rename to test/core/metaflow_extensions/toplevel/__init__.py diff --git a/test/core/metaflow_custom/toplevel/toplevel.py b/test/core/metaflow_extensions/toplevel/toplevel.py similarity index 53% rename from test/core/metaflow_custom/toplevel/toplevel.py rename to test/core/metaflow_extensions/toplevel/toplevel.py index f71ff11e0a4..a8a47dc3e15 100644 --- a/test/core/metaflow_custom/toplevel/toplevel.py +++ b/test/core/metaflow_extensions/toplevel/toplevel.py @@ -1,4 +1,4 @@ -__mf_customization__ = 'test' +__mf_extensions__ = 'test' tl_value = 42 diff --git a/test/core/metaflow_test/formatter.py b/test/core/metaflow_test/formatter.py index 0acfeba60c3..e4a202fb6db 100644 --- a/test/core/metaflow_test/formatter.py +++ b/test/core/metaflow_test/formatter.py @@ -142,7 +142,7 @@ def _check_lines(self): '"test_flow.py", '\ '"*/click/*", '\ '"*/site-packages/*", '\ - '"*/core/metaflow_custom/*", '\ + '"*/core/metaflow_extensions/*", '\ '"*/core/metaflow_test/*"])' yield 0, 'cov.start()' yield 0, 'import sys' diff --git a/test/core/tests/customization.py b/test/core/tests/extensions.py similarity index 93% rename from test/core/tests/customization.py rename to test/core/tests/extensions.py index 4d3f85624e6..dce22a1dc49 100644 --- a/test/core/tests/customization.py +++ b/test/core/tests/extensions.py @@ -2,7 +2,7 @@ class CustomizationTest(MetaflowTest): """ - Test that the metaflow_custom module is properly loaded + Test that the metaflow_extensions module is properly loaded """ PRIORITY = 0 diff --git a/test/core/tests/flow_options.py b/test/core/tests/flow_options.py index 6cec2d1c6f9..99e4f96249f 100644 --- a/test/core/tests/flow_options.py +++ b/test/core/tests/flow_options.py @@ -3,7 +3,7 @@ class FlowOptionsTest(MetaflowTest): """ - Test that the metaflow_custom module is properly loaded + Test that the metaflow_extensions module is properly loaded """ PRIORITY = 0 HEADER = """ From 76f7d239090b1198223ba022eb4a9984d5da8cf4 Mon Sep 17 00:00:00 2001 From: Savin Date: Thu, 9 Sep 2021 14:40:03 -0700 Subject: [PATCH 035/176] rename metaflow-extensions test (#693) --- test/core/tests/extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/core/tests/extensions.py b/test/core/tests/extensions.py index dce22a1dc49..6669e07095e 100644 --- a/test/core/tests/extensions.py +++ b/test/core/tests/extensions.py @@ -1,6 +1,6 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag -class CustomizationTest(MetaflowTest): +class ExtensionsTest(MetaflowTest): """ Test that the metaflow_extensions module is properly loaded """ From bd3705a9856d837f14ffa6f3fe16b658408cd979 Mon Sep 17 00:00:00 2001 From: Savin Date: Thu, 9 Sep 2021 16:54:33 -0700 Subject: [PATCH 036/176] [Perf] Only passdown parameters to datastore for the join step (#694) --- metaflow/task.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/metaflow/task.py b/metaflow/task.py index 98f18f3beeb..0a8c5ab888c 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -52,7 +52,7 @@ def _exec_step_function(self, step_function, input_obj=None): else: step_function(input_obj) - def _init_parameters(self, parameter_ds): + def _init_parameters(self, parameter_ds, passdown=True): # overwrite Parameters in the flow object vars = [] for var, param in self.flow._get_parameters(): @@ -66,8 +66,10 @@ def property_setter( setattr(self.flow.__class__, var, property(fget=property_setter)) - vars.append(var) - self.flow._datastore.passdown_partial(parameter_ds, vars) + if passdown: + vars.append(var) + if passdown: + self.flow._datastore.passdown_partial(parameter_ds, vars) return vars def _init_data(self, run_id, join_type, input_paths): @@ -397,7 +399,7 @@ def run_step(self, # initialize parameters (if they exist) # We take Parameter values from the first input, # which is always safe since parameters are read-only - current._update_env({'parameter_names': self._init_parameters(inputs[0])}) + current._update_env({'parameter_names': self._init_parameters(inputs[0], passdown=True)}) else: # Linear step: # We are running with a single input context. @@ -413,7 +415,7 @@ def run_step(self, # initialize parameters (if they exist) # We take Parameter values from the first input, # which is always safe since parameters are read-only - current._update_env({'parameter_names': self._init_parameters(inputs[0])}) + current._update_env({'parameter_names': self._init_parameters(inputs[0], passdown=False)}) decorators = step_func.decorators for deco in decorators: From 2f15f437ca1b0f9e72a3af15dad0f1722a477874 Mon Sep 17 00:00:00 2001 From: Savin Date: Fri, 10 Sep 2021 08:00:33 -0700 Subject: [PATCH 037/176] Emit project related tags from within project decorator (#695) * Emit project related tags from within project decorator * update test --- metaflow/cli.py | 24 +++++++++---------- metaflow/decorators.py | 22 ++++++++++++++--- metaflow/metadata/metadata.py | 3 --- .../aws/step_functions/schedule_decorator.py | 10 +++++++- .../plugins/conda/conda_flow_decorator.py | 10 +++++++- metaflow/plugins/project_decorator.py | 4 ++++ .../plugins/flow_options.py | 2 +- 7 files changed, 53 insertions(+), 22 deletions(-) diff --git a/metaflow/cli.py b/metaflow/cli.py index c5addef2aed..7d5e67aefde 100644 --- a/metaflow/cli.py +++ b/metaflow/cli.py @@ -870,19 +870,6 @@ def start(ctx, if e.TYPE == environment][0](ctx.obj.flow) ctx.obj.environment.validate_environment(echo) - ctx.obj.datastore = DATASTORES[datastore] - ctx.obj.datastore_root = datastore_root - - # It is important to initialize flow decorators early as some of the - # things they provide may be used by some of the objects initialize after. - decorators._init_flow_decorators(ctx.obj.flow, - ctx.obj.graph, - ctx.obj.environment, - ctx.obj.datastore, - ctx.obj.logger, - echo, - deco_options) - ctx.obj.monitor = Monitor(monitor, ctx.obj.environment, ctx.obj.flow.name) ctx.obj.monitor.start() @@ -898,6 +885,17 @@ def start(ctx, ctx.obj.datastore.get_datastore_root_from_config(ctx.obj.echo) ctx.obj.datastore_root = ctx.obj.datastore.datastore_root = datastore_root + # It is important to initialize flow decorators early as some of the + # things they provide may be used by some of the objects initialize after. + decorators._init_flow_decorators(ctx.obj.flow, + ctx.obj.graph, + ctx.obj.environment, + ctx.obj.datastore, + ctx.obj.metadata, + ctx.obj.logger, + echo, + deco_options) + if decospecs: decorators._attach_decorators(ctx.obj.flow, decospecs) diff --git a/metaflow/decorators.py b/metaflow/decorators.py index 9c13e6cf536..398f44b18eb 100644 --- a/metaflow/decorators.py +++ b/metaflow/decorators.py @@ -135,7 +135,15 @@ def __init__(self, *args, **kwargs): self._flow_decorators.append(self) super(FlowDecorator, self).__init__(*args, **kwargs) - def flow_init(self, flow, graph, environment, datastore, logger, echo, options): + def flow_init(self, + flow, + graph, + environment, + datastore, + metadata, + logger, + echo, + options): """ Called when all decorators have been created for this flow. """ @@ -431,10 +439,18 @@ def _attach_decorators_to_step(step, decospecs): deco = decos[deconame]._parse_decorator_spec(decospec) step.decorators.append(deco) -def _init_flow_decorators(flow, graph, environment, datastore, logger, echo, deco_options): +def _init_flow_decorators(flow, + graph, + environment, + datastore, + metadata, + logger, + echo, + deco_options): for deco in flow._flow_decorators.values(): opts = {option: deco_options[option] for option in deco.options} - deco.flow_init(flow, graph, environment, datastore, logger, echo, opts) + deco.flow_init(flow, graph, environment, + datastore, metadata, logger, echo, opts) def _init_step_decorators(flow, graph, environment, datastore, logger): diff --git a/metaflow/metadata/metadata.py b/metaflow/metadata/metadata.py index 13952938fe2..acd6803e318 100644 --- a/metaflow/metadata/metadata.py +++ b/metaflow/metadata/metadata.py @@ -462,9 +462,6 @@ def _tags(self): tags.append('metaflow_r_version:' + env['metaflow_r_version']) if 'r_version_code' in env: tags.append('r_version:' + env['r_version_code']) - if 'project_name' in current: - tags.append('project:' + current.project_name) - tags.append('project_branch:' + current.branch_name) return tags def _register_code_package_metadata(self, run_id, step_name, task_id): diff --git a/metaflow/plugins/aws/step_functions/schedule_decorator.py b/metaflow/plugins/aws/step_functions/schedule_decorator.py index 1181a6c3cfb..2dd96ef852b 100644 --- a/metaflow/plugins/aws/step_functions/schedule_decorator.py +++ b/metaflow/plugins/aws/step_functions/schedule_decorator.py @@ -8,7 +8,15 @@ class ScheduleDecorator(FlowDecorator): 'daily': True, 'hourly': False} - def flow_init(self, flow, graph, environment, datastore, logger, echo, options): + def flow_init(self, + flow, + graph, + environment, + datastore, + metadata, + logger, + echo, + options): # Currently supports quartz cron expressions in UTC as defined in # https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions if self.attributes['cron']: diff --git a/metaflow/plugins/conda/conda_flow_decorator.py b/metaflow/plugins/conda/conda_flow_decorator.py index 48789e106cf..552bd0a9b7e 100644 --- a/metaflow/plugins/conda/conda_flow_decorator.py +++ b/metaflow/plugins/conda/conda_flow_decorator.py @@ -37,7 +37,15 @@ class MyFlow(FlowSpec): 'python': None, 'disabled': None} - def flow_init(self, flow, graph, environment, datastore, logger, echo, options): + def flow_init(self, + flow, + graph, + environment, + datastore, + metadata, + logger, + echo, + options): if environment.TYPE != 'conda': raise InvalidEnvironmentException('The *@conda* decorator requires ' '--environment=conda') \ No newline at end of file diff --git a/metaflow/plugins/project_decorator.py b/metaflow/plugins/project_decorator.py index 8bd40848fc0..96fbc9d6ae1 100644 --- a/metaflow/plugins/project_decorator.py +++ b/metaflow/plugins/project_decorator.py @@ -35,6 +35,7 @@ def flow_init(self, graph, environment, datastore, + metadata, logger, echo, options): @@ -54,6 +55,9 @@ def flow_init(self, 'is_user_branch': is_user_branch, 'is_production': options['production'], 'project_flow_name': project_flow_name}) + metadata.add_sticky_tags(sys_tags=[ + 'project:%s' % project_name, + 'project_branch:%s' % branch_name]) def get_top_level_options(self): return list(self._option_values.items()) diff --git a/test/core/metaflow_extensions/plugins/flow_options.py b/test/core/metaflow_extensions/plugins/flow_options.py index 8d78eca5f11..9e860a66069 100644 --- a/test/core/metaflow_extensions/plugins/flow_options.py +++ b/test/core/metaflow_extensions/plugins/flow_options.py @@ -13,5 +13,5 @@ class FlowDecoratorWithOptions(FlowDecorator): ) } - def flow_init(self, flow, graph, environment, flow_datastore, logger, echo, options): + def flow_init(self, flow, graph, environment, flow_datastore, metadata, logger, echo, options): current._update_env({'foobar_value': options['foobar']}) \ No newline at end of file From 63ee02ba3612c0d61ecd5c67116637939a2c17a7 Mon Sep 17 00:00:00 2001 From: Savin Date: Sun, 12 Sep 2021 18:18:26 -0700 Subject: [PATCH 038/176] Datatools changes for convergence (#697) * datatools changes for convergence * move read_chunks up * update * update datatools --- metaflow/datatools/s3.py | 188 ++++++++---------- metaflow/datatools/s3op.py | 85 ++++---- .../{datastore/util => datatools}/s3tail.py | 0 metaflow/datatools/s3util.py | 16 ++ metaflow/plugins/aws/batch/batch.py | 2 +- metaflow/plugins/aws/batch/batch_cli.py | 2 +- metaflow/plugins/aws/batch/batch_decorator.py | 2 +- 7 files changed, 139 insertions(+), 156 deletions(-) rename metaflow/{datastore/util => datatools}/s3tail.py (100%) diff --git a/metaflow/datatools/s3.py b/metaflow/datatools/s3.py index 2ae410439c1..0df4eca7836 100644 --- a/metaflow/datatools/s3.py +++ b/metaflow/datatools/s3.py @@ -29,10 +29,13 @@ # python3 from urllib.parse import urlparse -from metaflow.datastore.util.s3util import get_s3_client +from .s3util import get_s3_client, read_in_chunks try: import boto3 + from boto3.s3.transfer import TransferConfig + DOWNLOAD_FILE_THRESHOLD = 2 * TransferConfig().multipart_threshold + DOWNLOAD_MAX_CHUNK = 2 * 1024 * 1024 * 1024 - 1 boto_found = True except: boto_found = False @@ -73,7 +76,6 @@ class S3Object(object): """ This object represents a path or an object in S3, with an optional local copy. - Get or list calls return one or more of S3Objects. """ @@ -154,7 +156,6 @@ def path(self): """ Path to the local file corresponding to the object downloaded. This file gets deleted automatically when a S3 scope exits. - Returns None if this S3Object has not been downloaded. """ return self._path @@ -163,7 +164,6 @@ def path(self): def blob(self): """ Contents of the object as a byte string. - Returns None if this S3Object has not been downloaded. """ if self._path: @@ -174,7 +174,6 @@ def blob(self): def text(self): """ Contents of the object as a Unicode string. - Returns None if this S3Object has not been downloaded. """ if self._path: @@ -184,7 +183,6 @@ def text(self): def size(self): """ Size of the object in bytes. - Returns None if the key does not correspond to an object in S3. """ return self._size @@ -233,6 +231,28 @@ def __str__(self): def __repr__(self): return str(self) + +class S3Client(object): + def __init__(self): + self._s3_client = None + self._s3_error = None + + @property + def client(self): + if self._s3_client is None: + self.reset_client() + return self._s3_client + + @property + def error(self): + if self._s3_error is None: + self.reset_client() + return self._s3_error + + def reset_client(self): + self._s3_client, self._s3_error = get_s3_client() + + class S3(object): @classmethod @@ -244,29 +264,23 @@ def __init__(self, bucket=None, prefix=None, run=None, - s3root=None): + s3root=None, + **kwargs): """ Initialize a new context for S3 operations. This object is used as a context manager for a with statement. - There are two ways to initialize this object depending whether you want to bind paths to a Metaflow run or not. - 1. With a run object: - run: (required) Either a FlowSpec object (typically 'self') or a Run object corresponding to an existing Metaflow run. These are used to add a version suffix in the S3 path. bucket: (optional) S3 bucket. prefix: (optional) S3 prefix. - 2. Without a run object: - s3root: (optional) An S3 root URL for all operations. If this is not specified, all operations require a full S3 URL. - These options are supported in both the modes: - tmproot: (optional) Root path for temporary files (default: '.') """ @@ -304,7 +318,7 @@ def __init__(self, # 3. use the client only with full URLs self._s3root = None - self._s3_client = None + self._s3_client = kwargs.get('external_client', S3Client()) self._tmpdir = mkdtemp(dir=tmproot, prefix='metaflow.s3.') def __enter__(self): @@ -313,9 +327,6 @@ def __enter__(self): def __exit__(self, *args): self.close() - def __del__(self): - self.close() - def close(self): """ Delete all temporary files downloaded in this context. @@ -328,11 +339,6 @@ def close(self): except: pass - def reset_client(self, hard_reset=False): - if hard_reset or self._s3_client is None: - from metaflow.datastore.util.s3util import get_s3_client - self._s3_client, self._s3_client_error = get_s3_client() - def _url(self, key_value): # NOTE: All URLs are handled as Unicode objects (unicode in py2, # string in py3) internally. We expect that all URLs passed to this @@ -390,20 +396,14 @@ def list_paths(self, keys=None): specified, listings are done in parallel. The returned S3Objects have .exists == False if the url refers to a prefix, not an existing S3 object. - Args: keys: (required) a list of suffixes for paths to list. - Returns: a list of S3Objects (not downloaded) - Example: - Consider the following paths in S3: - A/B/C D/E - In this case, list_paths(['A', 'D']), returns ['A/B', 'D/E']. The first S3Object has .exists == False, since it does not refer to an object in S3. It is just a prefix. @@ -427,20 +427,14 @@ def list_recursive(self, keys=None): specified, listings are done in parallel. The returned S3Objects have always .exists == True, since they refer to existing objects in S3. - Args: keys: (required) a list of suffixes for paths to list. - Returns: a list of S3Objects (not downloaded) - Example: - Consider the following paths in S3: - A/B/C D/E - In this case, list_recursive(['A', 'D']), returns ['A/B/C', 'D/E']. """ def _list(keys): @@ -456,13 +450,11 @@ def _list(keys): def info(self, key=None, return_missing=False): """ Get information about a single object from S3 - Args: key: (optional) a suffix identifying the object. return_missing: (optional, default False) if set to True, do not raise an exception for a missing key but return it as an S3Object with .exists == False. - Returns: an S3Object containing information about the object. The downloaded property will be false and exists will indicate whether @@ -473,42 +465,36 @@ def info(self, key=None, return_missing=False): def _info(s3, tmp): resp = s3.head_object(Bucket=src.netloc, Key=src.path.lstrip('/"')) - with open('%s' % tmp, mode='w') as f: - args = { - 'content_type': resp['ContentType'], - 'metadata': resp['Metadata'], - 'size': resp['ContentLength']} - json.dump(args, f) + return { + 'content_type': resp['ContentType'], + 'metadata': resp['Metadata'], + 'size': resp['ContentLength']} - path_info = None + info_results = None try: - path_info = self._one_boto_op(_info, url) + _, info_results = self._one_boto_op(_info, url, need_tmp_file=False) except MetaflowS3NotFound: if return_missing: - path_info = None + info_results = None else: raise - if path_info: - with open(path_info, 'r') as f: - info = json.load(f) + if info_results: return S3Object( self._s3root, url, path=None, - size=info['size'], - content_type=info['content_type'], - metadata=info['metadata']) + size=info_results['size'], + content_type=info_results['content_type'], + metadata=info_results['metadata']) return S3Object(self._s3root, url, None) def info_many(self, keys, return_missing=False): """ Get information about many objects from S3 in parallel. - Args: keys: (required) a list of suffixes identifying the objects. return_missing: (optional, default False) if set to True, do not raise an exception for a missing key but return it as an S3Object with .exists == False. - Returns: a list of S3Objects corresponding to the objects requested. The downloaded property will be false and exists will indicate whether @@ -549,7 +535,6 @@ def _head(): def get(self, key=None, return_missing=False, return_info=True): """ Get a single object from S3. - Args: key: (optional) a suffix identifying the object. Can also be an object containing the properties `key`, `offset` and @@ -559,7 +544,6 @@ def get(self, key=None, return_missing=False, return_info=True): return it as an S3Object with .exists == False. return_info: (optional, default True) if set to True, fetch the content-type and user metadata associated with the object. - Returns: an S3Object corresponding to the object requested. """ @@ -572,48 +556,44 @@ def _download(s3, tmp): Bucket=src.netloc, Key=src.path.lstrip('/'), Range=r) - code = str(resp['ResponseMetadata']['HTTPStatusCode']) - if code[0] == '2': - with open(tmp, mode='w') as t: - t.write(resp['Body'].read()) - else: - # TODO: Better raised error - raise RuntimeError("Could not load file") else: + resp = s3.get_object( + Bucket=src.netloc, + Key=src.path.lstrip('/')) + sz = resp['ContentLength'] + if not r and sz > DOWNLOAD_FILE_THRESHOLD: + # In this case, it is more efficient to use download_file as it + # will download multiple parts in parallel (it does it after + # multipart_threshold) s3.download_file(src.netloc, src.path.lstrip('/'), tmp) - return url - - def _info(s3, tmp): - resp = s3.head_object(Bucket=src.netloc, Key=src.path.lstrip('/"')) - with open('%s' % tmp, mode='w') as f: - args = { + else: + with open(tmp, mode='wb') as t: + read_in_chunks(t, resp['Body'], sz, DOWNLOAD_MAX_CHUNK) + if return_info: + return { 'content_type': resp['ContentType'], - 'metadata': resp['Metadata']} - json.dump(args, f) + 'metadata': resp['Metadata'] + } + return None - path_info = None + addl_info = None try: - path = self._one_boto_op(_download, url) - if return_info: - path_info = self._one_boto_op(_info, url) + path, addl_info = self._one_boto_op(_download, url) except MetaflowS3NotFound: if return_missing: path = None else: raise - if path_info: - with open(path_info, 'r') as f: - info = json.load(f) + if addl_info: return S3Object( self._s3root, url, path, - content_type=info['content_type'], - metadata=info['metadata']) + content_type=addl_info['content_type'], + metadata=addl_info['metadata']) return S3Object(self._s3root, url, path) def get_many(self, keys, return_missing=False, return_info=True): """ Get many objects from S3 in parallel. - Args: keys: (required) a list of suffixes identifying the objects. Each item in the list can also be an object containing the properties @@ -624,7 +604,6 @@ def get_many(self, keys, return_missing=False, return_info=True): return it as an S3Object with .exists == False. return_info: (optional, default True) if set to True, fetch the content-type and user metadata associated with the object. - Returns: a list of S3Objects corresponding to the objects requested. """ @@ -659,13 +638,11 @@ def _get(): def get_recursive(self, keys, return_info=False): """ Get many objects from S3 recursively in parallel. - Args: keys: (required) a list of suffixes for paths to download recursively. return_info: (optional, default False) if set to True, fetch the content-type and user metadata associated with the object. - Returns: a list of S3Objects corresponding to the objects requested. """ @@ -694,11 +671,9 @@ def get_all(self, return_info=False): """ Get all objects from S3 recursively (in parallel). This request only works if S3 is initialized with a run or a s3root prefix. - Args: return_info: (optional, default False) if set to True, fetch the content-type and user metadata associated with the object. - Returns: a list of S3Objects corresponding to the objects requested. """ @@ -711,7 +686,6 @@ def get_all(self, return_info=False): def put(self, key, obj, overwrite=True, content_type=None, metadata=None): """ Put an object to S3. - Args: key: (required) suffix for the object. obj: (required) a bytes, string, or a unicode object to @@ -720,7 +694,6 @@ def put(self, key, obj, overwrite=True, content_type=None, metadata=None): content_type: (optional) string representing the MIME type of the object metadata: (optional) User metadata to store alongside the object - Returns: an S3 URL corresponding to the object stored. """ @@ -760,7 +733,7 @@ def _upload(s3, _): blob, src.netloc, src.path.lstrip('/'), ExtraArgs=extra_args) if overwrite: - self._one_boto_op(_upload, url) + self._one_boto_op(_upload, url, need_tmp_file=False) real_close() return url else: @@ -768,9 +741,9 @@ def _head(s3, _): s3.head_object(Bucket=src.netloc, Key=src.path.lstrip('/')) try: - self._one_boto_op(_head, url) + self._one_boto_op(_head, url, need_tmp_file=False) except MetaflowS3NotFound: - self._one_boto_op(_upload, url) + self._one_boto_op(_upload, url, need_tmp_file=False) finally: real_close() return url @@ -778,7 +751,6 @@ def _head(s3, _): def put_many(self, key_objs, overwrite=True): """ Put objects to S3 in parallel. - Args: key_objs: (required) an iterator of (key, value) tuples. Value must be a string, bytes, or a unicode object. Instead of @@ -787,7 +759,6 @@ def put_many(self, key_objs, overwrite=True): 'metadata' like the S3PutObject for example. 'key' and 'value' are required but others are optional. overwrite: (optional) overwrites the key with obj, if it exists - Returns: a list of (key, S3 URL) tuples corresponding to the files sent. """ @@ -832,7 +803,6 @@ def _store(): def put_files(self, key_paths, overwrite=True): """ Put files to S3 in parallel. - Args: key_paths: (required) an iterator of (key, path) tuples. Instead of (key, path) tuples, you can also pass any object that @@ -840,7 +810,6 @@ def put_files(self, key_paths, overwrite=True): 'metadata' like the S3PutObject for example. 'key' and 'path' are required but others are optional. overwrite: (optional) overwrites the key with obj, if it exists - Returns: a list of (key, S3 URL) tuples corresponding to the files sent. """ @@ -866,18 +835,20 @@ def _check(): return self._put_many_files(_check(), overwrite) - def _one_boto_op(self, op, url): - from . import s3op + def _one_boto_op(self, op, url, need_tmp_file=True): error = '' for i in range(NUM_S3OP_RETRIES + 1): - tmp = NamedTemporaryFile(dir=self._tmpdir, - prefix='metaflow.s3.one_file.', - delete=False) - self.reset_client() - try: - op(self._s3_client, tmp.name) - return tmp.name - except self._s3_client_error as err: + tmp = None + if need_tmp_file: + tmp = NamedTemporaryFile(dir=self._tmpdir, + prefix='metaflow.s3.one_file.', + delete=False) + try: + side_results = op( + self._s3_client.client, tmp.name if tmp else None) + return tmp.name if tmp else None, side_results + except self._s3_client.error as err: + from . import s3op error_code = s3op.normalize_client_error(err) if error_code == 404: raise MetaflowS3NotFound(url) @@ -889,8 +860,9 @@ def _one_boto_op(self, op, url): except Exception as ex: # TODO specific error message for out of disk space error = str(ex) - os.unlink(tmp.name) - self.reset_client(hard_reset = True) + if need_tmp_file: + os.unlink(tmp.name) + self._s3_client.reset_client() # add some jitter to make sure retries are not synchronized time.sleep(2**i + random.randint(0, 10)) raise MetaflowS3Exception("S3 operation failed.\n"\ @@ -988,4 +960,4 @@ def _s3op_with_retries(self, mode, **options): raise MetaflowS3AccessDenied(err_out) time.sleep(2**i + random.randint(0, 10)) - return None, err_out + return None, err_out \ No newline at end of file diff --git a/metaflow/datatools/s3op.py b/metaflow/datatools/s3op.py index e8865848927..d69ddfc15eb 100644 --- a/metaflow/datatools/s3op.py +++ b/metaflow/datatools/s3op.py @@ -11,6 +11,8 @@ from multiprocessing import Process, Queue from itertools import starmap, chain, islice +from boto3.s3.transfer import TransferConfig + try: # python2 from urlparse import urlparse @@ -26,14 +28,17 @@ # PYTHONPATH for the parent Metaflow explicitly. sys.path.insert(0,\ os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))) + # we use Metaflow's parallel_imap_unordered instead of # multiprocessing.Pool because https://bugs.python.org/issue31886 from metaflow.util import TempDir, url_quote, url_unquote from metaflow.multicore_utils import parallel_map -from metaflow.datastore.util.s3util import aws_retry +from metaflow.datatools.s3util import aws_retry, read_in_chunks NUM_WORKERS_DEFAULT = 64 +DOWNLOAD_FILE_THRESHOLD = 2 * TransferConfig().multipart_threshold +DOWNLOAD_MAX_CHUNK = 2 * 1024 * 1024 * 1024 - 1 class S3Url(object): def __init__(self, bucket, path, url, local, prefix, @@ -74,6 +79,8 @@ def normalize_client_error(err): except ValueError: if error_code == 'AccessDenied': return 403 + if error_code == 'NoSuchKey': + return 404 return error_code # S3 worker pool @@ -111,7 +118,7 @@ def op_info(url): with open(result_file_name, 'w') as result_file: try: - from metaflow.datastore.util.s3util import get_s3_client + from metaflow.datatools.s3util import get_s3_client s3, client_error = get_s3_client() while True: url, idx = queue.get() @@ -125,65 +132,53 @@ def op_info(url): with open(url.local, 'w') as f: json.dump(result, f) elif mode == 'download': - result_info = None - is_missing = False - if pre_op_info: - result_info = op_info(url) - if result_info['error'] == ERROR_URL_NOT_FOUND: - is_missing = True - result_file.write("%d %d\n" % (idx, -ERROR_URL_NOT_FOUND)) - elif result_info['error'] == ERROR_URL_ACCESS_DENIED: - is_missing = True - result_file.write("%d %d\n" % (idx, -ERROR_URL_ACCESS_DENIED)) - elif result_info['error'] is not None: - raise result_info['raise_error'] - if is_missing: - continue - tmp = NamedTemporaryFile(dir='.', delete=False) + tmp = NamedTemporaryFile(dir='.', mode='wb', delete=False) try: - if url.range is None: - s3.download_file(url.bucket, url.path, tmp.name) - else: - # We do get_object. We don't actually do any retries - # here because the higher levels will do the retry if - # needed + if url.range: resp = s3.get_object( Bucket=url.bucket, Key=url.path, Range=url.range) - code = str(resp['ResponseMetadata']['HTTPStatusCode']) - if code[0] == '2': - tmp.write(resp['Body'].read()) - else: - # TODO: Better raised error - raise RuntimeError("Could not load file") + else: + resp = s3.get_object( + Bucket=url.bucket, + Key=url.path) + sz = resp['ContentLength'] + if not url.range and sz > DOWNLOAD_FILE_THRESHOLD: + # In this case, it is more efficient to use download_file as it + # will download multiple parts in parallel (it does it after + # multipart_threshold) + s3.download_file(url.bucket, url.path, tmp.name) + else: + read_in_chunks(tmp, resp['Body'], sz, DOWNLOAD_MAX_CHUNK) tmp.close() os.rename(tmp.name, url.local) except client_error as err: + tmp.close() + os.unlink(tmp.name) error_code = normalize_client_error(err) if error_code == 404: - pass # We skip this + result_file.write("%d %d\n" % (idx, -ERROR_URL_NOT_FOUND)) + continue + elif error_code == 403: + result_file.write("%d %d\n" % (idx, -ERROR_URL_ACCESS_DENIED)) + continue else: raise - except: # TODO specific error message for out of disk space - tmp.close() - os.unlink(tmp.name) - raise - # If we have metadata that we retrieved, we also write it out - # to a file - if result_info: + # If we need the metadata, get it and write it out + if pre_op_info: with open('%s_meta' % url.local, mode='w') as f: - args = {'size': result_info['size']} - if result_info['content_type']: - args['content_type'] = result_info['content_type'] - if result_info['metadata'] is not None: - args['metadata'] = result_info['metadata'] + args = {'size': resp['ContentLength']} + if resp['ContentType']: + args['content_type'] = resp['ContentType'] + if resp['Metadata'] is not None: + args['metadata'] = resp['Metadata'] json.dump(args, f) # Finally, we push out the size to the result_pipe since # the size is used for verification and other purposes and # we want to avoid file operations for this simple process - result_file.write("%d %d\n" % (idx, result_info['size'])) + result_file.write("%d %d\n" % (idx, resp['ContentLength'])) else: # This is upload, if we have a pre_op, it means we do not # want to overwrite @@ -303,7 +298,7 @@ def __init__(self): self.client_error = None def reset_client(self, hard_reset=False): - from metaflow.datastore.util.s3util import get_s3_client + from metaflow.datatools.s3util import get_s3_client if hard_reset or self.s3 is None: self.s3, self.client_error = get_s3_client() @@ -769,4 +764,4 @@ def info(prefixes, print(format_triplet(url.prefix, url.url, url.local)) if __name__ == '__main__': - cli(auto_envvar_prefix='S3OP') + cli(auto_envvar_prefix='S3OP') \ No newline at end of file diff --git a/metaflow/datastore/util/s3tail.py b/metaflow/datatools/s3tail.py similarity index 100% rename from metaflow/datastore/util/s3tail.py rename to metaflow/datatools/s3tail.py diff --git a/metaflow/datatools/s3util.py b/metaflow/datatools/s3util.py index 749c97f49f3..bfb55111f38 100644 --- a/metaflow/datatools/s3util.py +++ b/metaflow/datatools/s3util.py @@ -50,3 +50,19 @@ def retry_wrapper(self, *args, **kwargs): time.sleep(2**i + random.randint(0, 5)) raise last_exc return retry_wrapper + +# Read an AWS source in a chunked manner. +# We read in chunks (at most 2GB -- here this is passed via max_chunk_size) +# because of https://bugs.python.org/issue42853 (Py3 bug); this also helps +# keep memory consumption lower +# NOTE: For some weird reason, if you pass a large value to +# read, it delays the call so we always pass it either what +# remains or 2GB, whichever is smallest. +def read_in_chunks(dst, src, src_sz, max_chunk_size): + remaining = src_sz + while remaining > 0: + buf = src.read(min(remaining, max_chunk_size)) + # Py2 doesn't return the number of bytes written so calculate size + # separately + dst.write(buf) + remaining -= len(buf) \ No newline at end of file diff --git a/metaflow/plugins/aws/batch/batch.py b/metaflow/plugins/aws/batch/batch.py index e887508c389..3ae13d1e101 100644 --- a/metaflow/plugins/aws/batch/batch.py +++ b/metaflow/plugins/aws/batch/batch.py @@ -16,7 +16,7 @@ from .batch_client import BatchClient -from metaflow.datastore.util.s3tail import S3Tail +from metaflow.datatools.s3tail import S3Tail from metaflow.mflog.mflog import refine, set_should_persist from metaflow.mflog import export_mflog_env_vars,\ bash_capture_logs,\ diff --git a/metaflow/plugins/aws/batch/batch_cli.py b/metaflow/plugins/aws/batch/batch_cli.py index 3195d9051ed..30bfa5e5581 100644 --- a/metaflow/plugins/aws/batch/batch_cli.py +++ b/metaflow/plugins/aws/batch/batch_cli.py @@ -12,7 +12,7 @@ from metaflow.datastore import MetaflowDataStore from metaflow.datastore.local import LocalDataStore -from metaflow.datastore.util.s3util import get_s3_client +from metaflow.datatools.s3util import get_s3_client from metaflow.metaflow_config import DATASTORE_LOCAL_DIR from metaflow import util from metaflow import R diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index 389ee490a2a..e4633eb7e1a 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -7,7 +7,7 @@ from metaflow.datastore import MetaflowDataStore from metaflow.datastore.datastore import TransformableObject -from metaflow.datastore.util.s3util import get_s3_client +from metaflow.datatools.s3util import get_s3_client from metaflow.decorators import StepDecorator from metaflow.metaflow_config import DATASTORE_LOCAL_DIR from metaflow.plugins import ResourcesDecorator From 50e8765d810ac968a09d82640e3302c28426732a Mon Sep 17 00:00:00 2001 From: Savin Date: Sun, 12 Sep 2021 19:25:12 -0700 Subject: [PATCH 039/176] Datatools changes from Convergence branch (#698) * datatools changes for convergence * move read_chunks up * update datatools * Datatools changes for convergence branch * fix import * fix erroneous update --- metaflow/datastore/s3.py | 2 +- metaflow/datastore/util/s3util.py | 51 ------------------------------- metaflow/datatools/s3.py | 14 ++++----- 3 files changed, 8 insertions(+), 59 deletions(-) delete mode 100644 metaflow/datastore/util/s3util.py diff --git a/metaflow/datastore/s3.py b/metaflow/datastore/s3.py index 8a4ee25c770..8dc4b27972b 100644 --- a/metaflow/datastore/s3.py +++ b/metaflow/datastore/s3.py @@ -18,7 +18,7 @@ from .. import metaflow_config from .datastore import MetaflowDataStore, only_if_not_done from ..metadata import MetaDatum -from .util.s3util import aws_retry, get_s3_client +from ..datatools.s3util import aws_retry, get_s3_client from metaflow.util import Path # We need UncloseableBytesIO for put_s3_object which may need diff --git a/metaflow/datastore/util/s3util.py b/metaflow/datastore/util/s3util.py deleted file mode 100644 index 96e9b32ac8b..00000000000 --- a/metaflow/datastore/util/s3util.py +++ /dev/null @@ -1,51 +0,0 @@ -from __future__ import print_function -import random -import time -import sys -import os - -from metaflow.exception import MetaflowException -from metaflow.metaflow_config import S3_ENDPOINT_URL, S3_VERIFY_CERTIFICATE - -S3_NUM_RETRIES = 7 - -TEST_S3_RETRY = 'TEST_S3_RETRY' in os.environ - -def get_s3_client(): - from metaflow.plugins.aws.aws_client import get_aws_client - return get_aws_client( - 's3', - with_error=True, - params={'endpoint_url': S3_ENDPOINT_URL, 'verify': S3_VERIFY_CERTIFICATE }) - -# decorator to retry functions that access S3 -def aws_retry(f): - def retry_wrapper(self, *args, **kwargs): - last_exc = None - for i in range(S3_NUM_RETRIES): - try: - ret = f(self, *args, **kwargs) - if TEST_S3_RETRY and i == 0: - raise Exception("TEST_S3_RETRY env var set. " - "Pretending that an S3 op failed. " - "This is not a real failure.") - else: - return ret - except MetaflowException as ex: - # MetaflowExceptions are not related to AWS, don't retry - raise - except Exception as ex: - try: - function_name = f.func_name - except AttributeError: - function_name = f.__name__ - sys.stderr.write("S3 datastore operation %s failed (%s). " - "Retrying %d more times..\n" - % (function_name, ex, S3_NUM_RETRIES - i)) - self.reset_client(hard_reset=True) - last_exc = ex - # exponential backoff for real failures - if not (TEST_S3_RETRY and i == 0): - time.sleep(2**i + random.randint(0, 5)) - raise last_exc - return retry_wrapper \ No newline at end of file diff --git a/metaflow/datatools/s3.py b/metaflow/datatools/s3.py index 0df4eca7836..722fb333ac5 100644 --- a/metaflow/datatools/s3.py +++ b/metaflow/datatools/s3.py @@ -472,7 +472,7 @@ def _info(s3, tmp): info_results = None try: - _, info_results = self._one_boto_op(_info, url, need_tmp_file=False) + _, info_results = self._one_boto_op(_info, url, create_tmp_file=False) except MetaflowS3NotFound: if return_missing: info_results = None @@ -733,7 +733,7 @@ def _upload(s3, _): blob, src.netloc, src.path.lstrip('/'), ExtraArgs=extra_args) if overwrite: - self._one_boto_op(_upload, url, need_tmp_file=False) + self._one_boto_op(_upload, url, create_tmp_file=False) real_close() return url else: @@ -741,9 +741,9 @@ def _head(s3, _): s3.head_object(Bucket=src.netloc, Key=src.path.lstrip('/')) try: - self._one_boto_op(_head, url, need_tmp_file=False) + self._one_boto_op(_head, url, create_tmp_file=False) except MetaflowS3NotFound: - self._one_boto_op(_upload, url, need_tmp_file=False) + self._one_boto_op(_upload, url, create_tmp_file=False) finally: real_close() return url @@ -835,11 +835,11 @@ def _check(): return self._put_many_files(_check(), overwrite) - def _one_boto_op(self, op, url, need_tmp_file=True): + def _one_boto_op(self, op, url, create_tmp_file=True): error = '' for i in range(NUM_S3OP_RETRIES + 1): tmp = None - if need_tmp_file: + if create_tmp_file: tmp = NamedTemporaryFile(dir=self._tmpdir, prefix='metaflow.s3.one_file.', delete=False) @@ -860,7 +860,7 @@ def _one_boto_op(self, op, url, need_tmp_file=True): except Exception as ex: # TODO specific error message for out of disk space error = str(ex) - if need_tmp_file: + if tmp: os.unlink(tmp.name) self._s3_client.reset_client() # add some jitter to make sure retries are not synchronized From 9f42be66f36d4038a0dce7d35150a666d6cca941 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Sun, 19 Sep 2021 07:42:32 +0200 Subject: [PATCH 040/176] Spell parameter correctly in parameter.R (#708) --- R/R/parameter.R | 2 +- R/man/parameter.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/R/parameter.R b/R/R/parameter.R index 5273e272c1a..4b00d38a3fd 100644 --- a/R/R/parameter.R +++ b/R/R/parameter.R @@ -8,7 +8,7 @@ #' @param flow metaflow object #' @param parameter name of the parameter #' @param required logical (defaults to FALSE) denoting if -#' paramter is required as an argument to \code{run} the flow +#' parameter is required as an argument to \code{run} the flow #' @param help optional help text #' @param default optional default value of the parameter #' @param type optional type of the parameter diff --git a/R/man/parameter.Rd b/R/man/parameter.Rd index 2c2e8e15bcb..455e293bc2f 100644 --- a/R/man/parameter.Rd +++ b/R/man/parameter.Rd @@ -21,7 +21,7 @@ parameter( \item{parameter}{name of the parameter} \item{required}{logical (defaults to FALSE) denoting if -paramter is required as an argument to \code{run} the flow} +parameter is required as an argument to \code{run} the flow} \item{help}{optional help text} From 8126e80004b0042a0eb148332bfac11c850ea02b Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Tue, 21 Sep 2021 04:50:13 +0200 Subject: [PATCH 041/176] Fix typos discovered by codespell (#709) --- R/inst/tutorials/04-helloaws/helloaws.Rmd | 2 +- R/inst/tutorials/05-statistics-redux/README.md | 2 +- R/inst/tutorials/07-autopilot/README.md | 2 +- docs/Environment escape.md | 2 +- docs/concurrency.md | 2 +- metaflow/datastore/datastore.py | 2 +- metaflow/datastore/local.py | 2 +- metaflow/datastore/s3.py | 2 +- metaflow/flowspec.py | 2 +- metaflow/lint.py | 4 ++-- metaflow/main_cli.py | 2 +- metaflow/metaflow_environment.py | 2 +- metaflow/metaflow_version.py | 2 +- metaflow/plugins/aws/batch/batch_decorator.py | 2 +- .../plugins/aws/step_functions/step_functions.py | 2 +- metaflow/plugins/conda/conda.py | 2 +- metaflow/plugins/env_escape/client.py | 4 ++-- metaflow/plugins/env_escape/override_decorators.py | 12 ++++++------ metaflow/plugins/env_escape/server.py | 2 +- metaflow/plugins/env_escape/stub.py | 2 +- metaflow/plugins/metadata/local.py | 2 +- metaflow/sidecar.py | 4 ++-- metaflow/tutorials/05-helloaws/helloaws.ipynb | 2 +- test/data/s3/test_s3.py | 2 +- 24 files changed, 32 insertions(+), 32 deletions(-) diff --git a/R/inst/tutorials/04-helloaws/helloaws.Rmd b/R/inst/tutorials/04-helloaws/helloaws.Rmd index 8eed9c5ddaa..5b5a4933a19 100644 --- a/R/inst/tutorials/04-helloaws/helloaws.Rmd +++ b/R/inst/tutorials/04-helloaws/helloaws.Rmd @@ -2,7 +2,7 @@ title: "Episode 04-helloaws: Look Mom, We're in the Cloud" output: html_notebook --- -In HellowAWSFlow, the 'start' and 'end' steps were run locally, while the 'hello' step was run remotely on AWS batch. Since we are using AWS, data artifacts and metdata were stored remotely. This means you can use the client to access information about any flow from anywhere. This notebook shows you how. +In HellowAWSFlow, the 'start' and 'end' steps were run locally, while the 'hello' step was run remotely on AWS batch. Since we are using AWS, data artifacts and metadata were stored remotely. This means you can use the client to access information about any flow from anywhere. This notebook shows you how. ## Import the metaflow client ```{r} diff --git a/R/inst/tutorials/05-statistics-redux/README.md b/R/inst/tutorials/05-statistics-redux/README.md index c3df17a77a1..e1873ca9754 100644 --- a/R/inst/tutorials/05-statistics-redux/README.md +++ b/R/inst/tutorials/05-statistics-redux/README.md @@ -32,5 +32,5 @@ If you are using RStudio, you can replace the last line `run()` with and run by `source("stats.R")`. ##### Inspect the results: -Open the R markdown file ```02-statistics/stats.Rmd``` in your RStudio and re-run the cells. You can acccess +Open the R markdown file ```02-statistics/stats.Rmd``` in your RStudio and re-run the cells. You can access the artifacts stored in AWS S3 from your local RStudio session. \ No newline at end of file diff --git a/R/inst/tutorials/07-autopilot/README.md b/R/inst/tutorials/07-autopilot/README.md index 73ee543d07f..0ccd1ee0f94 100644 --- a/R/inst/tutorials/07-autopilot/README.md +++ b/R/inst/tutorials/07-autopilot/README.md @@ -35,5 +35,5 @@ run(package_suffixes=".R,.csv", step_functions="trigger") for SFN trigger. You can then directly run `source("stats.R`)` in RStudio. ##### Inspect the results: -Open the R Markdown file```07-autopilot/stats.Rmd``` in your RStudio and re-run the cells. You can acccess +Open the R Markdown file```07-autopilot/stats.Rmd``` in your RStudio and re-run the cells. You can access the artifacts stored in AWS S3 from your local RStudio session. \ No newline at end of file diff --git a/docs/Environment escape.md b/docs/Environment escape.md index 9dec5a8e762..a4d76e3ecd7 100644 --- a/docs/Environment escape.md +++ b/docs/Environment escape.md @@ -143,7 +143,7 @@ and produces a JSON-able object (typically a dictionary with string keys and jsonable objects as values). The decoding is the reverse where a dictionary is taken from the channel and Python objects are returned. -Transfering exceptions requires a tiny bit more work and this logic can be found +Transferring exceptions requires a tiny bit more work and this logic can be found in ```exception_transferer.py```; this relies on ```data_transferer.py``` to do the actual encoding and decoding and ```exception_transferer.py``` merely takes care of the specificities of extracting the information needed from the diff --git a/docs/concurrency.md b/docs/concurrency.md index 841268eb9e4..53f439c9203 100644 --- a/docs/concurrency.md +++ b/docs/concurrency.md @@ -7,7 +7,7 @@ while parallelism is the simultaneous execution of (possibly related) computations* from [a talk by Rob Pike, Concurrency is not Parallelism](https://blog.golang.org/concurrency-is-not-parallelism): -**Parallelism** is a relatively straighforward and quantifiable +**Parallelism** is a relatively straightforward and quantifiable concept. However, it is not always easy to decide what constructs of **concurrency**, which can lead to parallelism, are most appropriate in each context. The choice is not easy since besides parallelism diff --git a/metaflow/datastore/datastore.py b/metaflow/datastore/datastore.py index 68b3023c454..5e8447d37a9 100644 --- a/metaflow/datastore/datastore.py +++ b/metaflow/datastore/datastore.py @@ -144,7 +144,7 @@ def done(self): def is_done(self): """ A flag indicating whether this datastore directory was closed - succesfully with done(). + successfully with done(). """ raise NotImplementedError() diff --git a/metaflow/datastore/local.py b/metaflow/datastore/local.py index 853a6fdc120..2ce2120db3a 100644 --- a/metaflow/datastore/local.py +++ b/metaflow/datastore/local.py @@ -254,7 +254,7 @@ def done(self): def is_done(self): """ A flag indicating whether this datastore directory was closed - succesfully with done(). + successfully with done(). """ filename = self.get_done_filename_for_attempt(self.attempt) path = os.path.join(self.root, filename) diff --git a/metaflow/datastore/s3.py b/metaflow/datastore/s3.py index 8dc4b27972b..ff6708daa97 100644 --- a/metaflow/datastore/s3.py +++ b/metaflow/datastore/s3.py @@ -294,7 +294,7 @@ def done(self): def is_done(self): """ A flag indicating whether this datastore directory was closed - succesfully with done(). + successfully with done(). """ filename = self.get_done_filename_for_attempt(self.attempt) path = os.path.join(self.root, filename) diff --git a/metaflow/flowspec.py b/metaflow/flowspec.py index 0757ca81dd6..fb21a1cb0aa 100644 --- a/metaflow/flowspec.py +++ b/metaflow/flowspec.py @@ -344,7 +344,7 @@ def merge_artifacts(self, inputs, exclude=[], include=[]): # We have unresolved conflicts so we do not set anything and error out msg = "Step *{step}* cannot merge the following artifacts due to them "\ "having conflicting values:\n[{artifacts}].\nTo remedy this issue, "\ - "be sure to explictly set those artifacts (using "\ + "be sure to explicitly set those artifacts (using "\ "self. = ...) prior to calling merge_artifacts."\ .format(step=self._current_step, artifacts=', '.join(unresolved)) raise UnhandledInMergeArtifactsException(msg, unresolved) diff --git a/metaflow/lint.py b/metaflow/lint.py index 25dcc6a3522..6d59926b34e 100644 --- a/metaflow/lint.py +++ b/metaflow/lint.py @@ -189,7 +189,7 @@ def check_split_join_balance(graph): msg1 = "Step *{0.name}* seems like a join step (it takes an extra input "\ "argument) but an incorrect number of steps (*{paths}*) lead to "\ "it. This join was expecting {num_roots} incoming paths, starting "\ - "from splitted step(s) *{roots}*." + "from split step(s) *{roots}*." msg2 = "Step *{0.name}* seems like a join step (it takes an extra input "\ "argument) but it is not preceded by a split. Ensure that there is "\ "a matching split for every join." @@ -240,7 +240,7 @@ def parents(n): @linter.check def check_empty_foreaches(graph): msg = "Step *{0.name}* is a foreach split that has no children: "\ - "it is followed immeditately by a join step, *{join}*. Add "\ + "it is followed immediately by a join step, *{join}*. Add "\ "at least one step between the split and the join." for node in graph: if node.type == 'foreach': diff --git a/metaflow/main_cli.py b/metaflow/main_cli.py index 6cf33548a9e..48a0ab0f82f 100644 --- a/metaflow/main_cli.py +++ b/metaflow/main_cli.py @@ -231,7 +231,7 @@ def pull(episode): continue echo('Pulling episode ', nl=False) echo('\"{0}\"'.format(episode), fg='cyan', nl=False) - # TODO: Is the following redudant? + # TODO: Is the following redundant? echo(' into your current working directory.') # Copy from (local) metaflow package dir to current. src_dir = os.path.join(tutorials_dir, episode) diff --git a/metaflow/metaflow_environment.py b/metaflow/metaflow_environment.py index 14ccca1618d..d941043c9ad 100644 --- a/metaflow/metaflow_environment.py +++ b/metaflow/metaflow_environment.py @@ -50,7 +50,7 @@ def bootstrap_commands(self, step_name): def add_to_package(self): """ A list of tuples (file, arcname) to add to the job package. - `arcname` is an alterative name for the file in the job package. + `arcname` is an alternative name for the file in the job package. """ return [] diff --git a/metaflow/metaflow_version.py b/metaflow/metaflow_version.py index 75af379e7b5..ad66b0f5553 100644 --- a/metaflow/metaflow_version.py +++ b/metaflow/metaflow_version.py @@ -57,7 +57,7 @@ def find_git_on_windows(): def call_git_describe(abbrev=7): - """return the string output of git desribe""" + """return the string output of git describe""" try: # first, make sure we are actually in a Metaflow repo, diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index e4633eb7e1a..44b830e69b3 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -265,7 +265,7 @@ def _get_registry(cls, image): [@:] - The separator must be either "@" or ":" ? - The separator is optional ((?<=[@:]).*)? - [GROUP 2] TAG / DIGEST - (?<=[@:]) - A tag / digest must be preceeded by "@" or ":" + (?<=[@:]) - A tag / digest must be preceded by "@" or ":" .* - Capture rest of tag / digest ? - A tag / digest is optional diff --git a/metaflow/plugins/aws/step_functions/step_functions.py b/metaflow/plugins/aws/step_functions/step_functions.py index 0b489c30c3d..0cd73e4e377 100644 --- a/metaflow/plugins/aws/step_functions/step_functions.py +++ b/metaflow/plugins/aws/step_functions/step_functions.py @@ -190,7 +190,7 @@ def get_existing_deployment(cls, name): return parameters.get('metaflow.owner'), \ parameters.get('metaflow.production_token') except KeyError as e: - raise StepFunctionsException("An exisiting non-metaflow " + raise StepFunctionsException("An existing non-metaflow " "workflow with the same name as " "*%s* already exists in AWS Step " "Functions. Please modify the " diff --git a/metaflow/plugins/conda/conda.py b/metaflow/plugins/conda/conda.py index 40b46b7977c..949d3b96908 100644 --- a/metaflow/plugins/conda/conda.py +++ b/metaflow/plugins/conda/conda.py @@ -216,7 +216,7 @@ def _acquire(self): 'Could not acquire lock {}'.format(self.lock)) if (time.time() - start) >= self.timeout: raise CondaException( - 'Timeout occured while acquiring lock {}'.format(self.lock)) + 'Timeout occurred while acquiring lock {}'.format(self.lock)) time.sleep(self.delay) def _release(self): diff --git a/metaflow/plugins/env_escape/client.py b/metaflow/plugins/env_escape/client.py index 0dae1a1c937..f288936d97b 100644 --- a/metaflow/plugins/env_escape/client.py +++ b/metaflow/plugins/env_escape/client.py @@ -92,7 +92,7 @@ def __init__(self, python_path, max_pickle_version, config_dir): for name in obj_funcs: if name in override_dict: raise ValueError( - "%s was already overriden for %s" % (name, obj_name) + "%s was already overridden for %s" % (name, obj_name) ) override_dict[name] = override.func self._proxied_objects = {} @@ -250,7 +250,7 @@ def get_local_class(self, name, obj_id=None): local_class = self._proxied_classes[name] if local_class is None: # We need to build up this class. To do so, we take everything that the - # remote class has and remove UNSUPPORTED things and overriden things + # remote class has and remove UNSUPPORTED things and overridden things remote_methods = self.stub_request(None, OP_GETMETHODS, name) local_class = create_class( self, name, self._overrides.get(name, {}), diff --git a/metaflow/plugins/env_escape/override_decorators.py b/metaflow/plugins/env_escape/override_decorators.py index 3bc982badac..cb9fd9cc099 100644 --- a/metaflow/plugins/env_escape/override_decorators.py +++ b/metaflow/plugins/env_escape/override_decorators.py @@ -41,7 +41,7 @@ class RemoteAttrOverride(AttrOverride): def local_override(obj_mapping): if not isinstance(obj_mapping, dict): raise ValueError( - "@local_override takes a dictionary: -> []" + "@local_override takes a dictionary: -> []" ) def _wrapped(func): @@ -53,7 +53,7 @@ def _wrapped(func): def local_getattr_override(obj_mapping): if not isinstance(obj_mapping, dict): raise ValueError( - "@local_getattr_override takes a dictionary: -> []" + "@local_getattr_override takes a dictionary: -> []" ) def _wrapped(func): @@ -65,7 +65,7 @@ def _wrapped(func): def local_setattr_override(obj_mapping): if not isinstance(obj_mapping, dict): raise ValueError( - "@local_setattr_override takes a dictionary: -> []" + "@local_setattr_override takes a dictionary: -> []" ) def _wrapped(func): @@ -77,7 +77,7 @@ def _wrapped(func): def remote_override(obj_mapping): if not isinstance(obj_mapping, dict): raise ValueError( - "@remote_override takes a dictionary: -> []" + "@remote_override takes a dictionary: -> []" ) def _wrapped(func): @@ -89,7 +89,7 @@ def _wrapped(func): def remote_getattr_override(obj_mapping): if not isinstance(obj_mapping, dict): raise ValueError( - "@remote_getattr_override takes a dictionary: -> []" + "@remote_getattr_override takes a dictionary: -> []" ) def _wrapped(func): @@ -101,7 +101,7 @@ def _wrapped(func): def remote_setattr_override(obj_mapping): if not isinstance(obj_mapping, dict): raise ValueError( - "@remote_setattr_override takes a dictionary: -> []" + "@remote_setattr_override takes a dictionary: -> []" ) def _wrapped(func): diff --git a/metaflow/plugins/env_escape/server.py b/metaflow/plugins/env_escape/server.py index c4f4b620461..1060ca71f12 100644 --- a/metaflow/plugins/env_escape/server.py +++ b/metaflow/plugins/env_escape/server.py @@ -119,7 +119,7 @@ def __init__(self, max_pickle_version, config_dir): for name in obj_funcs: if name in override_dict: raise ValueError( - "%s was already overriden for %s" % (name, obj_name) + "%s was already overridden for %s" % (name, obj_name) ) override_dict[name] = override.func elif isinstance(override, RemoteExceptionSerializer): diff --git a/metaflow/plugins/env_escape/stub.py b/metaflow/plugins/env_escape/stub.py index a53c95ffda3..1095ac14104 100644 --- a/metaflow/plugins/env_escape/stub.py +++ b/metaflow/plugins/env_escape/stub.py @@ -220,7 +220,7 @@ def class_method(connection, class_name, name, cls, *args, **kwargs): class MetaWithConnection(StubMetaClass): # The use of this metaclass is so that we can support two modes when - # instanciating a sub-class of Stub. Suppose we have a class Foo which is a stub. + # instantiating a sub-class of Stub. Suppose we have a class Foo which is a stub. # There are two ways Foo is initialized: # - when it is returned from the remote side, in which case we do # Foo(class_name, connection, identifier) diff --git a/metaflow/plugins/metadata/local.py b/metaflow/plugins/metadata/local.py index 8987cc84d56..46fbe256631 100644 --- a/metaflow/plugins/metadata/local.py +++ b/metaflow/plugins/metadata/local.py @@ -49,7 +49,7 @@ def register_run_id(self, run_id, tags=[], sys_tags=[]): try: # This metadata provider only generates integer IDs so if this is # an integer, we don't register it again (since it was "registered" - # on creation). However, some IDs are created outside the metdata + # on creation). However, some IDs are created outside the metadata # provider and need to be properly registered int(run_id) return diff --git a/metaflow/sidecar.py b/metaflow/sidecar.py index a5e8ea6b34b..afebd600cf0 100644 --- a/metaflow/sidecar.py +++ b/metaflow/sidecar.py @@ -29,12 +29,12 @@ class PipeUnavailableError(Exception): class NullSidecarError(Exception): - """raised when tyring to poll or interact with the fake subprocess in the null sidecar""" + """raised when trying to poll or interact with the fake subprocess in the null sidecar""" pass class MsgTimeoutError(Exception): - """raised when tyring unable to send message to sidecar in allocated time""" + """raised when trying unable to send message to sidecar in allocated time""" pass diff --git a/metaflow/tutorials/05-helloaws/helloaws.ipynb b/metaflow/tutorials/05-helloaws/helloaws.ipynb index 35771eb7c59..0936afe5337 100644 --- a/metaflow/tutorials/05-helloaws/helloaws.ipynb +++ b/metaflow/tutorials/05-helloaws/helloaws.ipynb @@ -6,7 +6,7 @@ "source": [ "# Episode 05-helloaws: Look Mom, We're in the Cloud\n", "\n", - "### In HellowAWSFlow, the 'start' and 'end' steps were run locally, while the 'hello' step was run remotely on AWS batch. Since we are using AWS, data artifacts and metdata were stored remotely. This means you can use the client to access information about any flow from anywhere. This notebook shows you how. " + "### In HellowAWSFlow, the 'start' and 'end' steps were run locally, while the 'hello' step was run remotely on AWS batch. Since we are using AWS, data artifacts and metadata were stored remotely. This means you can use the client to access information about any flow from anywhere. This notebook shows you how. " ] }, { diff --git a/test/data/s3/test_s3.py b/test/data/s3/test_s3.py index d30f607f0d4..0284c7d3f91 100644 --- a/test/data/s3/test_s3.py +++ b/test/data/s3/test_s3.py @@ -84,7 +84,7 @@ def assert_results(s3objs, expected, info_should_be_empty=False, info_only=False else: # Content_type is OK if content_type is None: - # Default content-type when nothing is suplied + # Default content-type when nothing is supplied assert s3obj.content_type == 'binary/octet-stream' else: assert s3obj.content_type == content_type From 5ad3742ef7a6fbf207abcc8f8de36998f32af6e4 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 22 Sep 2021 00:38:45 -0700 Subject: [PATCH 042/176] Make the number of S3 retries configurable (#700) --- metaflow/datatools/s3.py | 8 +++----- metaflow/datatools/s3util.py | 8 +++----- metaflow/metaflow_config.py | 6 ++++++ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/metaflow/datatools/s3.py b/metaflow/datatools/s3.py index 722fb333ac5..85e6dfb3ccf 100644 --- a/metaflow/datatools/s3.py +++ b/metaflow/datatools/s3.py @@ -11,7 +11,7 @@ from .. import FlowSpec from ..current import current -from ..metaflow_config import DATATOOLS_S3ROOT +from ..metaflow_config import DATATOOLS_S3ROOT, S3_RETRY_COUNT from ..util import namedtuple_with_defaults,\ is_stringish,\ to_bytes,\ @@ -54,8 +54,6 @@ def ensure_unicode(x): 'RangeInfo', 'total_size request_offset request_length', defaults=(0, -1)) -NUM_S3OP_RETRIES = 7 - class MetaflowS3InvalidObject(MetaflowException): headline = 'Not a string-like object' @@ -837,7 +835,7 @@ def _check(): def _one_boto_op(self, op, url, create_tmp_file=True): error = '' - for i in range(NUM_S3OP_RETRIES + 1): + for i in range(S3_RETRY_COUNT + 1): tmp = None if create_tmp_file: tmp = NamedTemporaryFile(dir=self._tmpdir, @@ -939,7 +937,7 @@ def _s3op_with_retries(self, mode, **options): else: cmdline.extend(('--%s' % key, value)) - for i in range(NUM_S3OP_RETRIES + 1): + for i in range(S3_RETRY_COUNT + 1): with NamedTemporaryFile(dir=self._tmpdir, mode='wb+', delete=not debug.s3client, diff --git a/metaflow/datatools/s3util.py b/metaflow/datatools/s3util.py index bfb55111f38..0574486ee13 100644 --- a/metaflow/datatools/s3util.py +++ b/metaflow/datatools/s3util.py @@ -5,9 +5,7 @@ import os from metaflow.exception import MetaflowException -from metaflow.metaflow_config import S3_ENDPOINT_URL, S3_VERIFY_CERTIFICATE - -S3_NUM_RETRIES = 7 +from metaflow.metaflow_config import S3_ENDPOINT_URL, S3_VERIFY_CERTIFICATE, S3_RETRY_COUNT TEST_S3_RETRY = 'TEST_S3_RETRY' in os.environ @@ -23,7 +21,7 @@ def get_s3_client(): def aws_retry(f): def retry_wrapper(self, *args, **kwargs): last_exc = None - for i in range(S3_NUM_RETRIES): + for i in range(S3_RETRY_COUNT): try: ret = f(self, *args, **kwargs) if TEST_S3_RETRY and i == 0: @@ -42,7 +40,7 @@ def retry_wrapper(self, *args, **kwargs): function_name = f.__name__ sys.stderr.write("S3 datastore operation %s failed (%s). " "Retrying %d more times..\n" - % (function_name, ex, S3_NUM_RETRIES - i)) + % (function_name, ex, S3_RETRY_COUNT - i)) self.reset_client(hard_reset=True) last_exc = ex # exponential backoff for real failures diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py index 8ce7738b3ce..4429f63b33a 100644 --- a/metaflow/metaflow_config.py +++ b/metaflow/metaflow_config.py @@ -75,6 +75,12 @@ def from_conf(name, default=None): S3_ENDPOINT_URL = from_conf('METAFLOW_S3_ENDPOINT_URL', None) S3_VERIFY_CERTIFICATE = from_conf('METAFLOW_S3_VERIFY_CERTIFICATE', None) +# S3 retry configuration +# This is useful if you want to "fail fast" on S3 operations; use with caution +# though as this may increase failures. Note that this is the number of *retries* +# so setting it to 0 means each operation will be tried once. +S3_RETRY_COUNT = from_conf('METAFLOW_S3_RETRY_COUNT', 7) + ### # Datastore local cache ### From bcd9ed4241355ff869805a0f8ae3685522891424 Mon Sep 17 00:00:00 2001 From: Romain Date: Thu, 23 Sep 2021 23:12:12 -0700 Subject: [PATCH 043/176] Tag metadata with attempt_id (#702) * Tag metadata with attempt_id * Remove un-needed str() conversion * Tag code-package metadata; also fix bug with double code-package metadata with SFN * Fix method definition for register_task_id --- metaflow/datastore/local.py | 3 ++- metaflow/datastore/s3.py | 3 ++- metaflow/metadata/metadata.py | 9 ++++++--- metaflow/plugins/aws/batch/batch_decorator.py | 4 +++- .../aws/step_functions/step_functions_decorator.py | 4 +++- metaflow/plugins/conda/conda_step_decorator.py | 4 +++- metaflow/plugins/metadata/local.py | 12 +++++++----- metaflow/plugins/metadata/service.py | 9 ++++++--- metaflow/runtime.py | 4 ++-- metaflow/task.py | 14 +++++++------- 10 files changed, 41 insertions(+), 25 deletions(-) diff --git a/metaflow/datastore/local.py b/metaflow/datastore/local.py index 2ce2120db3a..cf1b5ddd0cb 100644 --- a/metaflow/datastore/local.py +++ b/metaflow/datastore/local.py @@ -247,7 +247,8 @@ def done(self): raise self.metadata.register_metadata( self.run_id, self.step_name, self.task_id, - [MetaDatum(field='attempt-done', value=str(self.attempt), type='attempt-done', tags=[])]) + [MetaDatum(field='attempt-done', value=str(self.attempt), type='attempt-done', tags=[ + "attempt_id:{0}".format(self.attempt)])]) self._is_done_set = True diff --git a/metaflow/datastore/s3.py b/metaflow/datastore/s3.py index ff6708daa97..041387c7393 100644 --- a/metaflow/datastore/s3.py +++ b/metaflow/datastore/s3.py @@ -287,7 +287,8 @@ def done(self): self.metadata.register_metadata( self.run_id, self.step_name, self.task_id, - [MetaDatum(field='attempt-done', value=str(self.attempt), type='attempt-done', tags=[])]) + [MetaDatum(field='attempt-done', value=str(self.attempt), type='attempt-done', + tags=["attempt_id:{0}".format(self.attempt)])]) self._is_done_set = True diff --git a/metaflow/metadata/metadata.py b/metaflow/metadata/metadata.py index acd6803e318..55932093c79 100644 --- a/metaflow/metadata/metadata.py +++ b/metaflow/metadata/metadata.py @@ -155,7 +155,8 @@ def new_task_id(self, run_id, step_name, tags=[], sys_tags=[]): ''' raise NotImplementedError() - def register_task_id(self, run_id, step_name, task_id, tags=[], sys_tags=[]): + def register_task_id( + self, run_id, step_name, task_id, attempt=0, tags=[], sys_tags=[]): ''' No-op operation in this implementation. @@ -464,7 +465,7 @@ def _tags(self): tags.append('r_version:' + env['r_version_code']) return tags - def _register_code_package_metadata(self, run_id, step_name, task_id): + def _register_code_package_metadata(self, run_id, step_name, task_id, attempt): metadata = [] code_sha = os.environ.get('METAFLOW_CODE_SHA') code_url = os.environ.get('METAFLOW_CODE_URL') @@ -474,7 +475,9 @@ def _register_code_package_metadata(self, run_id, step_name, task_id): field='code-package', value=json.dumps({'ds_type': code_ds, 'sha': code_sha, 'location': code_url}), type='code-package', - tags=[])) + tags=["attempt_id:{0}".format(attempt)])) + # We don't tag with attempt_id here because not readily available; this + # is ok though as this doesn't change from attempt to attempt. if metadata: self.register_metadata(run_id, step_name, task_id, metadata) diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index 44b830e69b3..9596c724596 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -214,7 +214,9 @@ def task_pre_step(self, except: pass - entries = [MetaDatum(field=k, value=v, type=k, tags=[]) for k, v in meta.items()] + entries = [MetaDatum( + field=k, value=v, type=k, tags=["attempt_id:{0}".format(retry_count)]) + for k, v in meta.items()] # Register book-keeping metadata for debugging. metadata.register_metadata(run_id, step_name, task_id, entries) self._save_logs_sidecar = SidecarSubProcess('save_logs_periodically') diff --git a/metaflow/plugins/aws/step_functions/step_functions_decorator.py b/metaflow/plugins/aws/step_functions/step_functions_decorator.py index 9b31e62e37b..ce1d786e776 100644 --- a/metaflow/plugins/aws/step_functions/step_functions_decorator.py +++ b/metaflow/plugins/aws/step_functions/step_functions_decorator.py @@ -26,7 +26,9 @@ def task_pre_step(self, meta['aws-step-functions-execution'] = os.environ['METAFLOW_RUN_ID'] meta['aws-step-functions-state-machine'] =\ os.environ['SFN_STATE_MACHINE'] - entries = [MetaDatum(field=k, value=v, type=k, tags=[]) for k, v in meta.items()] + entries = [MetaDatum( + field=k, value=v, type=k, tags=["attempt_id:{0}".format(retry_count)]) + for k, v in meta.items()] # Register book-keeping metadata for debugging. metadata.register_metadata(run_id, step_name, task_id, entries) diff --git a/metaflow/plugins/conda/conda_step_decorator.py b/metaflow/plugins/conda/conda_step_decorator.py index 93e42e26dbd..28c8bf13caf 100644 --- a/metaflow/plugins/conda/conda_step_decorator.py +++ b/metaflow/plugins/conda/conda_step_decorator.py @@ -289,7 +289,9 @@ def task_pre_step(self, [MetaDatum(field='conda_env_id', value=self._env_id(), type='conda_env_id', - tags=[])]) + tags=[ + "attempt_id:{0}". + format(retry_count)])]) def runtime_step_cli(self, cli_args, diff --git a/metaflow/plugins/metadata/local.py b/metaflow/plugins/metadata/local.py index 46fbe256631..2fb8d6bc0b9 100644 --- a/metaflow/plugins/metadata/local.py +++ b/metaflow/plugins/metadata/local.py @@ -66,15 +66,17 @@ def register_task_id(self, run_id, step_name, task_id, + attempt=0, tags=[], sys_tags=[]): try: # Same logic as register_run_id int(task_id) except ValueError: - self._new_task(run_id, step_name, task_id, tags, sys_tags) - finally: - self._register_code_package_metadata(run_id, step_name, task_id) + self._new_task(run_id, step_name, task_id, attempt, tags, sys_tags) + else: + self._register_code_package_metadata( + run_id, step_name, task_id, attempt) def register_data_artifacts(self, run_id, @@ -193,10 +195,10 @@ def _new_run(self, run_id, tags=[], sys_tags=[]): self._ensure_meta('flow', None, None, None) self._ensure_meta('run', run_id, None, None, tags, sys_tags) - def _new_task(self, run_id, step_name, task_id, tags=[], sys_tags=[]): + def _new_task(self, run_id, step_name, task_id, attempt=0, tags=[], sys_tags=[]): self._ensure_meta('step', run_id, step_name, None) self._ensure_meta('task', run_id, step_name, task_id, tags, sys_tags) - self._register_code_package_metadata(run_id, step_name, task_id) + self._register_code_package_metadata(run_id, step_name, task_id, attempt) @staticmethod def _make_path( diff --git a/metaflow/plugins/metadata/service.py b/metaflow/plugins/metadata/service.py index 0bb493eddea..507b037cb55 100644 --- a/metaflow/plugins/metadata/service.py +++ b/metaflow/plugins/metadata/service.py @@ -73,6 +73,7 @@ def register_task_id(self, run_id, step_name, task_id, + attempt=0, tags=[], sys_tags=[]): try: @@ -83,10 +84,11 @@ def register_task_id(self, self._new_task(run_id, step_name, task_id, + attempt, tags=tags, sys_tags=sys_tags) - finally: - self._register_code_package_metadata(run_id, step_name, task_id) + else: + self._register_code_package_metadata(run_id, step_name, task_id, attempt) def _start_heartbeat(self, heartbeat_type, flow_id, run_id, step_name=None, task_id=None): if self._already_started(): @@ -194,12 +196,13 @@ def _new_task(self, run_id, step_name, task_id=None, + attempt=0, tags=[], sys_tags=[]): # first ensure that the step exists self._get_or_create('step', run_id, step_name) task = self._get_or_create('task', run_id, step_name, task_id, tags=tags, sys_tags=sys_tags) - self._register_code_package_metadata(run_id, step_name, task['task_id']) + self._register_code_package_metadata(run_id, step_name, task['task_id'], attempt) return task['task_id'] @staticmethod diff --git a/metaflow/runtime.py b/metaflow/runtime.py index 8b42e031519..8afb00b9ffd 100644 --- a/metaflow/runtime.py +++ b/metaflow/runtime.py @@ -571,10 +571,10 @@ def __init__(self, else: # task_id is preset only by persist_parameters() or control tasks. if ubf_context == UBF_CONTROL: - metadata.register_task_id(run_id, step, task_id, + metadata.register_task_id(run_id, step, task_id, 0, sys_tags=[CONTROL_TASK_TAG]) else: - metadata.register_task_id(run_id, step, task_id) + metadata.register_task_id(run_id, step, task_id, 0) self.step = step self.flow_name = flow.name diff --git a/metaflow/task.py b/metaflow/task.py index 0a8c5ab888c..f92aa747dbe 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -271,7 +271,7 @@ def run_step(self, if run_id and task_id: self.metadata.register_run_id(run_id) - self.metadata.register_task_id(run_id, step_name, task_id) + self.metadata.register_task_id(run_id, step_name, task_id, retry_count) else: raise MetaflowInternalError("task.run_step needs a valid run_id " "and task_id") @@ -283,25 +283,26 @@ def run_step(self, raise MetaflowInternalError("Too many task attempts (%d)! " "MAX_ATTEMPTS exceeded." % retry_count) + metadata_tags = ["attempt_id:{0}".format(retry_count)] self.metadata.register_metadata(run_id, step_name, task_id, [MetaDatum(field='attempt', value=str(retry_count), type='attempt', - tags=[]), + tags=metadata_tags), MetaDatum(field='origin-run-id', value=str(origin_run_id), type='origin-run-id', - tags=[]), + tags=metadata_tags), MetaDatum(field='ds-type', value=self.datastore.TYPE, type='ds-type', - tags=[]), + tags=metadata_tags), MetaDatum(field='ds-root', value=self.datastore.datastore_root, type='ds-root', - tags=[])]) + tags=metadata_tags)]) step_func = getattr(self.flow, step_name) node = self.flow._graph[step_name] @@ -514,8 +515,7 @@ def run_step(self, value=attempt_ok, type='internal_attempt_status', tags=["attempt_id:{0}". - format(str(retry_count)) - ]) + format(retry_count)]) ]) output.save_metadata('task_end', {}) From 4676f66adacf247de700833e3636e0b0cc57041a Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 24 Sep 2021 08:13:40 +0200 Subject: [PATCH 044/176] Undefined name: from .current import current for line 132 (#716) --- metaflow/includefile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metaflow/includefile.py b/metaflow/includefile.py index 569f028e43b..24f80e48753 100644 --- a/metaflow/includefile.py +++ b/metaflow/includefile.py @@ -12,6 +12,7 @@ import click from . import parameters +from .current import current from .datastore.datastore import TransformableObject from .exception import MetaflowException from .metaflow_config import DATATOOLS_LOCALROOT, DATATOOLS_SUFFIX From 9504c2c06e41f542aeea37c6d8f808d314521498 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 24 Sep 2021 08:15:38 +0200 Subject: [PATCH 045/176] Don't forget "self" in class methods (#715) $ `flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics` ``` ./metaflow/metaflow/util.py:36:13: F821 undefined name 'self' self.path = path ^ ``` --- metaflow/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/util.py b/metaflow/util.py index fd805b34641..71da7efdb03 100644 --- a/metaflow/util.py +++ b/metaflow/util.py @@ -32,7 +32,7 @@ def unquote_bytes(x): # this is used e.g. by datastore.save_logs to identify paths class Path(object): - def __init__(path): + def __init__(self, path): self.path = path def __str__(self): From 1572952c0bfeb19e4af7b7c4279e1b0cbf28f690 Mon Sep 17 00:00:00 2001 From: Savin Date: Fri, 24 Sep 2021 11:46:04 +0530 Subject: [PATCH 046/176] [Breaking] Modify return type of created_at/finished_at to datetime (#692) * [Breaking] Modify return type of created_at/finished_at to datetime * Update core.py * Update core.py --- metaflow/client/core.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/metaflow/client/core.py b/metaflow/client/core.py index 84ba4946a3a..046e8a44872 100644 --- a/metaflow/client/core.py +++ b/metaflow/client/core.py @@ -1,6 +1,6 @@ from __future__ import print_function +from datetime import datetime import os -import time import tarfile import json from collections import namedtuple @@ -343,8 +343,7 @@ def __init__(self, else: raise MetaflowInternalError(msg="Unknown type: %s" % self._NAME) - self._created_at = time.strftime( - '%Y-%m-%dT%H:%M:%SZ', time.gmtime(self._object['ts_epoch']//1000)) + self._created_at = datetime.fromtimestamp(self._object['ts_epoch']/1000.0) self._tags = frozenset(chain(self._object.get('system_tags') or [], self._object.get('tags') or [])) From 2410ec039e10cf5f361f331a848f18523c4a2708 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Fri, 24 Sep 2021 09:34:20 +0300 Subject: [PATCH 047/176] Conda _call_conda should print the stderr in case of error (#706) * Conda _call_conda should print the stderr in case of error and not hide the error * remove stray line --- metaflow/plugins/conda/conda.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/metaflow/plugins/conda/conda.py b/metaflow/plugins/conda/conda.py index 949d3b96908..3734b640aa3 100644 --- a/metaflow/plugins/conda/conda.py +++ b/metaflow/plugins/conda/conda.py @@ -40,7 +40,7 @@ def __init__(self): raise InvalidEnvironmentException('Conda channel \'conda-forge\' ' 'is required. Specify it with CONDA_CHANNELS ' 'environment variable.') - + def create(self, step_name, env_id, @@ -129,14 +129,14 @@ def _install(self, env_id, deps, explicit=False): def _install_order(self, env_id): cmd = ['list', '--name', env_id, '--explicit'] - response = self._call_conda(cmd).decode('utf-8') + response = self._call_conda(cmd).decode('utf-8') emit = False result = [] for line in response.splitlines(): if emit: result.append(line.split('/')[-1]) if not emit and line == '@EXPLICIT': - emit = True + emit = True return result def _deps(self, env_id): @@ -170,8 +170,8 @@ def _call_conda(self, args, architecture=None, disable_safety_checks=False): if disable_safety_checks: env['CONDA_SAFETY_CHECKS'] = 'disabled' return subprocess.check_output( - [self._bin] + args, - stderr = open(os.devnull, 'wb'), + [self._bin] + args, + stderr = subprocess.PIPE, env = dict(os.environ, **env)).strip() except subprocess.CalledProcessError as e: try: @@ -183,8 +183,8 @@ def _call_conda(self, args, architecture=None, disable_safety_checks=False): except (TypeError, ValueError) as ve: pass raise CondaException( - 'command \'{cmd}\' returned error ({code}): {output}' - .format(cmd=e.cmd, code=e.returncode, output=e.output)) + 'command \'{cmd}\' returned error ({code}): {output}, stderr={stderr}' + .format(cmd=e.cmd, code=e.returncode, output=e.output, stderr=e.stderr)) class CondaLock(object): @@ -234,4 +234,4 @@ def __exit__(self, type, value, traceback): self.__del__() def __del__(self): - self._release() \ No newline at end of file + self._release() From 4c8c470f693fd93f030b5f8cbc451efa70cd6c49 Mon Sep 17 00:00:00 2001 From: Romain Date: Fri, 24 Sep 2021 01:05:32 -0700 Subject: [PATCH 048/176] New Datastore abstraction for Metaflow (#580) * Add external_field to parameter * Fix timedate conversion from service to client * New datastore implementation This commit contains only the new datastore code and none of the backend implementations. The datastore is now split into different files: - flow_datastore.py contains the top-level FlowDataStore implementation - task_datastore.py contains the task-level datastore implementation - content_addressed_store.py contains the underlying content addressed store used by both previous data-stores. * Local backend for the datastore. The local backend is used to save and load local files. * Datatools performance improvements Datatools will now cache the s3 client it uses for single operations resulting in faster operation times. Another optimization (non advertised) is that datatools can now take an IOBase directly to avoid having an additional copy. Finally, __del__ now performs a close so the S3 datatool can be used as a regular object as opposed to just within a context. * Added S3 datastore backend. This backend allows the datastore to interface with S3. * Pure non-semantic changes (name changes and comment changes). One tiny semantic change in the way a Tar file is read (using the recommended open method instead of TarFile object) * Integrate the new datastore implementation into the current code * Remove no longer needed files for the datastore. * New features for the S3 datatools: - support for range queries (in get and get_many) - support for content-type and user metadata in put, put_many and put_files (metadata can also be retrieved using any of the get calls) - support for info and info_many to retrieve information about a file without fetching it. * Fix metadata to allow for non-string arguments * [WIP] Modify new datastore to use metadata in backend storage Instead of encoding needed information directly in the file, we now encode this information as file metadata leveraging the support for metadata for the S3 datatools and implementing support for it in the local filesystem (creating a separate file) * Properly use download_fileobj when no range is specified * Re-add intermediate compatibility mode for loading blobs from the CAS * Remove print message; fix local_backend in some cases * [WIP] Datatools: See if moving info to worker makes a performance difference * Fix issue with s3 datastore backend for multiple files * Fix issue with s3 datastore backend for multiple files * Addressed comments * Improve import of metaflow_custom to allow for top-level imports * Fix small issue with sidecar message types * Fix tags for attempt-done in task-datastore * Fix small issue with sidecar message types * Addressed comments -- minor tweaks and file refactoring * Addressed comment: removed handling of ephemeral CAS encoding directly in file * Addressed comments; refactored use of pipes Addressed the minor style comments. Removed the use of a pipe to communicate between workers and master. Communication now occurs via a file. * Forgot file in previous commit * Typo in parentheses causing issues with put of multiple files * Fix issue with S3PutObject * Fix bugs due to int/string conversion issues with new file mechanism * Fix issue with is_none and py2 vs py3 * Fix issue with metaflow listing all flows * Handle non-integer task/run IDs in local metadata * WIP: MFLog on convergence branch This is still in progress but pushing for reference. * Fixups to track master MFLog and specific convergence fixups * Fix to Batch logging * Add support for getting all artifacts from the filecache * Improve MFLog: - Save logs later in runtime to get all logs (even messages printed on error) - Allow for escaping variables in bash commands * Fix a bug when missing data is requested from the datastore * Trigger test on convergence branch * Fix failing tests * Fix issue caused by merge_artifacts now having artifacts read from datastore in write mode (#577) * Convergence master merge (#578) * UBF Implementation (#554) Unbounded Foreach Implementation Co-authored-by: Savin * Check if R step names are valid Python identifiers (#573) * Check if R step names are valid Python identifiers * Accidentally changed a docstring step_name to validator. Reverting. * get rid of _METAFLOW_RESUMED_RUN (#575) * Add Unbounded Foreach support Co-authored-by: Savin Co-authored-by: David Neuzerling Co-authored-by: Oleg Avdeev * Revert "Add external_field to parameter" This reverts commit b11868512687f739175b7871da065c23edce8f61. * Small MFlog refactor * Add a way to pass an external client to S3() * Cache the S3 boto connection in the datastore (#570) * The datastore now caches the S3 boto connection and passes it to the S3 client This allows us to use S3() using with (and therefore properly clean up the temp directory) but also benefit from the connection reuse. As part of this change, we also had to change the interface to load_bytes. This change is invisible to the user as it is 100% internal to the datastore. Cleaned up the s3tail.py and s3util.py files to co-locate them with the datatools instead of the datastore (since they were no longer used directly in the datastore). This simplified the import story. * Addressed comments and simplified the code a bit - Use a single CloseAfterUse class. - Remove the use of caching on write (in artifact and blob cache); the cache now only caches things that are read which is the way it is being used - Simplified the types of caches used in datastore_set.py * Remove no longer needed file * Clean up usage of CloseAfterUse * remove artifact_cache, simplify blob_cache * get rid of redundant LazyFile * fix FileBlobCache root * fix paths in content_addressed_store * Addressed comments * Move check on file existence in CAS to backend for better efficiency * Fix modes for datastore due to merge_artifacts functionality * Ignore error when overwrite is False and file exists Co-authored-by: Ville Tuulos * Add an option to define your own AWS client provider (#611) * Add an option to define your own AWS client provider You can now specify a function that returns an AWS client. This is useful if you want to use something other than boto3 and can be used with the metaflow_custom mechanism to provide your own authentication * Typo * Do not remove _expected_extensions since it is used in a function * Another typo * Addressed comments * Add inputs to task_pre_step (#616) * Tweaks and optimizations to datastore * Revert "Move check on file existence in CAS to backend for better efficiency" This reverts commit d0ac6fe228217b7429893f634eb7e07793aabe9b. * Change encoding type to gzip+pickle to keep forward compatibility * Reduce memory usage of artifact save and load We ensure that we do not keep around temporaries and allow the GC to remove objects that are no longer needed * Make is_file parallel We can now call is_file in parallel. It is unclear if this helps a lot given the overhead of loading the s3op and starting them all. It may help in the case of a very large number of artifacts * Set overwrite=True when uploading metadata * Improve fetching info and file at the same time * reduce duplicate copies in datastore by using generators * minimize memory consumption by handling artifacts one by one * fix docstrings and documentation * fix @batch to work with the new datastore * do not implicitly assume a correct key order in flow_datastore.load_data * Other minor consistency tweaks Co-authored-by: Romain Cledat * Merge master into convergence * Add inputs to task_pre_step (#615) * Add inputs to task_pre_step * Addressed comments * Forgot to remove an import * Refactor @resources decorator (#617) * Refactor @resources decorator @resources decorator is shared by all compute related decorators - @batch, @lambda, @k8s, @titus. This patch moves it out of batch_decorator.py so that other decorators can cleanly reference it. * Update __init__.py * Add an option to define your own AWS client provider (#620) You can now specify a function that returns an AWS client. This is useful if you want to use something other than boto3 and can be used with the metaflow_custom mechanism to provide your own authentication * Add S3 tests (#613) * Add S3 tests * Addressed comments * silence tar timestamp warnings (#627) * Handle None as default parameters properly in AWS Step Functions (#630) A parameter specification like - ``` Parameter(name="test_param", type=int, default=None) ``` will result in an error even though the default has been specified ``` Flow failed: The value of parameter test_param is ambiguous. It does not have a default and it is not required. ``` This PR fixes this issue. * IncludeFile now returns the included file in the client (#607) * DRAFT: IncludeFile now returns the included file in the client and CLI THIS IS NOT FINISHED; DO NOT MERGE AS IS. * Fix the tests * Forgot to update type check for multiple encoding * Add resource tags to AWS Batch jobs (#631) * Add resource tags to AWS Batch jobs This PR assumes that Batch:TagResource is whitelisted for the role which submits the AWS Batch job. * propagate tags * add env var * better commens * add production token * New patch release (#633) * Bug fix with s3tail exception (#635) * Revert "IncludeFile now returns the included file in the client (#607)" (#637) This reverts commit 0575f8d55c94a6969c35a41675bdb69cc8387c44. * patch release (#639) * Update Unbounded Foreach Test in the case of a Conda environment (#626) Co-authored-by: Savin Co-authored-by: Oleg Avdeev * Fix Py2 compatibility * Fix typo in s3op.py * Pin test executions to master branch * Fix names of metadata file to include ending * Renamed metaflow_custom to metaflow_extensions * Remove duplicate code * remove .json suffix from metadata * test commit * remove spurious commits * Fix merge * remove new line * Fixed batch related bug in convergence. (#710) * fix merge conflicts * Batch fixes for convergence branch * fixup aws batch * fix whitespace * fix imports * fix newline * fix merge * fix local metadata for aws batch * merge mflog * stylistic fixes * Added preliminary documentation on the datastore (#593) * Added preliminary documentation on the datastore * Update datastore.md Update with new name for the backend. * Final set of patches to convergence (#714) * convergence fixes * fixes * gzip ts changes * fix logs subcommand * typo * Address comments; add support for var_transform in bash_capture_logs * Fix local metadata sync * Forgot to remove duplicate sync_metadata Co-authored-by: Romain Cledat Co-authored-by: Romain * Typo in error message Co-authored-by: Savin Co-authored-by: David Neuzerling Co-authored-by: Oleg Avdeev Co-authored-by: Ville Tuulos Co-authored-by: Valay Dave --- docs/datastore.md | 216 +++++ metaflow/cli.py | 106 +-- metaflow/client/core.py | 92 ++- metaflow/client/filecache.py | 261 ++++-- metaflow/datastore/__init__.py | 12 +- metaflow/datastore/content_addressed_store.py | 199 +++++ metaflow/datastore/datastore.py | 631 --------------- metaflow/datastore/datastore_set.py | 83 +- metaflow/datastore/datastore_storage.py | 244 ++++++ metaflow/datastore/exceptions.py | 4 + metaflow/datastore/flow_datastore.py | 215 +++++ metaflow/datastore/inputs.py | 17 + metaflow/datastore/local.py | 262 ------- metaflow/datastore/local_storage.py | 116 +++ metaflow/datastore/s3.py | 302 ------- metaflow/datastore/s3_storage.py | 117 +++ metaflow/datastore/task_datastore.py | 741 ++++++++++++++++++ metaflow/datastore/util/__init__.py | 0 metaflow/datatools/__init__.py | 20 +- metaflow/decorators.py | 16 +- metaflow/includefile.py | 12 +- metaflow/main_cli.py | 8 +- metaflow/metadata/metadata.py | 9 +- metaflow/metadata/util.py | 33 + metaflow/mflog/__init__.py | 8 +- metaflow/mflog/save_logs.py | 27 +- metaflow/package.py | 16 +- metaflow/plugins/aws/batch/batch_cli.py | 84 +- metaflow/plugins/aws/batch/batch_decorator.py | 90 ++- .../aws/step_functions/schedule_decorator.py | 2 +- .../aws/step_functions/step_functions.py | 16 +- .../aws/step_functions/step_functions_cli.py | 23 +- .../step_functions_decorator.py | 2 +- metaflow/plugins/catch_decorator.py | 2 +- metaflow/plugins/conda/conda_environment.py | 6 +- .../plugins/conda/conda_flow_decorator.py | 2 +- .../plugins/conda/conda_step_decorator.py | 14 +- metaflow/plugins/metadata/local.py | 50 +- metaflow/plugins/package_cli.py | 3 +- metaflow/plugins/project_decorator.py | 2 +- metaflow/plugins/retry_decorator.py | 2 +- metaflow/plugins/timeout_decorator.py | 4 +- metaflow/runtime.py | 74 +- metaflow/task.py | 80 +- metaflow/util.py | 22 +- 45 files changed, 2517 insertions(+), 1728 deletions(-) create mode 100644 docs/datastore.md create mode 100644 metaflow/datastore/content_addressed_store.py delete mode 100644 metaflow/datastore/datastore.py create mode 100644 metaflow/datastore/datastore_storage.py create mode 100644 metaflow/datastore/exceptions.py create mode 100644 metaflow/datastore/flow_datastore.py create mode 100644 metaflow/datastore/inputs.py delete mode 100644 metaflow/datastore/local.py create mode 100644 metaflow/datastore/local_storage.py delete mode 100644 metaflow/datastore/s3.py create mode 100644 metaflow/datastore/s3_storage.py create mode 100644 metaflow/datastore/task_datastore.py delete mode 100644 metaflow/datastore/util/__init__.py create mode 100644 metaflow/metadata/util.py diff --git a/docs/datastore.md b/docs/datastore.md new file mode 100644 index 00000000000..ba74d85336d --- /dev/null +++ b/docs/datastore.md @@ -0,0 +1,216 @@ +# Datastore design +## Motivation + +The datastore is a crucial part of the Metaflow architecture and deals with +storing and retrieving data, be they artifacts (data produced or consumed within +user steps), logs, metadata information used by Metaflow itself to track execution +or other data like code packages. + +One of the key benefits of Metaflow is the ease with which users can access the +data; it is made available to steps of a flow that need it and users can access +it using the Metaflow client API. + +This documentation provides a brief overview of Metaflow's datastore implementation +and points out ways in which it can be extended to support, for example, other +storage systems (like GCS instead of S3). + +## High-level design + +### Design principles +A few principles were followed in designing this datastore. They are listed here +for reference and to help explain some of the choices made. + +#### Backward compatibility +The new datastore should be able to read and interact with data stored using +an older implementation of the datastore. While we do not guarantee forward +compatibility, currently, older datastores should be able to read most of the data +stored using the newer datastore. + +#### Batch operations +Where possible, APIs are batch friendly and should be used that way. In other +words, it is typically more efficient to call an API once, passing it all the +items to operate on (for example, all the keys to fetch) than to call the same +API multiple times with a single key at a time. All APIs are designed with +batch processing in mind where it makes sense. + +#### Separation of responsabilities +Each class implements few functionalities and we attempted to maximize reuse. +The idea is that this will also help in developing newer implementations going +forward and being able to surgically change a few things while keeping most of +the code the same. + +### Storage structure +Before going into the design of the datastore itself, it is worth considering +**where** Metaflow stores its information. Note that, in this section, the term +`directory` can also refer to a `prefix` in S3 for example. + +Metaflow considers a datastore to have a `datastore_root` which is the base +directory of the datastore. Within that directory, Metaflow will create multiple +sub-directories, one per flow (identified by the name of the flow). Within each +of those directories, Metaflow will create one directory per run as well as +a `data` directory which will contain all the artifacts ever produced by that +flow. + +The datastore has several components (starting at the lowest-level): +- a `DataStoreStorage` which abstracts away a storage system (like S3 or + the local filesystem). This provides very simple methods to read and write + bytes, obtain metadata about a file, list a directory as well as minor path + manipulation routines. Metaflow provides sample S3 and local filesystem + implementations. When implementing a new backend, you should only need to + implement the methods defined in `DataStoreStorage` to integrate with the + rest of the Metaflow datastore implementation. +- a `ContentAddressedStore` which implements a thin layer on top of a + `DataStoreStorage` to allow the storing of byte blobs in a content-addressable + manner. In other words, for each `ContentAddressedStore`, identical objects are + stored once and only once, thereby providing some measure of de-duplication. + This class includes the determination of what content is the same or not as well + as any additional encoding/compressing prior to storing the blob in the + `DataStoreStorage`. You can extend this class by providing alternate methods of + packing and unpacking the blob into bytes to be saved. +- a `TaskDataStore` is the main interface through which the rest of Metaflow + interfaces with the datastore. It includes functions around artifacts ( + `persisting` (saving) artifacts, loading (getting)), logs and metadata. +- a `FlowDataStore` ties everything together. A `FlowDataStore` will include + a `ContentAddressedStore` and all the `TaskDataStore`s for all the tasks that + are part of the flow. The `FlowDataStore` includes functions to find the + `TaskDataStore` for a given task as well as save and load data directly ( + this is used primarily for data that is not tied to a single task, for example + code packages which are more tied to runs). + +From the above description, you can see that there is one `ContentAddressedStore` +per flow so artifacts are de-duplicated *per flow* but not across all flows. + +## Implementation details + +In this section, we will describe each individual class mentioned above in more +detail + +### `DataStoreStorage` class + +This class implements low-level operations directly interacting with the +file-system (or other storage system such as S3). It exposes a file and +directory like abstraction (with functions such as `path_join`, `path_split`, +`basename`, `dirname` and `is_file`). + +Files manipulated at this level are byte objects; the two main functions `save_bytes` +and `load_bytes` operate at the byte level. Additional metadata to save alongside +the file can also be provided as a dictionary. The backend does not parse or +interpret this metadata in any way and simply stores and retrieves it. + +The `load_bytes` has a particularity in the sense that it returns an object +`CloseAfterUse` which must be used in a `with` statement. Any bytes loaded +will not be accessible after the `with` statement terminates and so must be +used or copied elsewhere prior to termination of the `with` scope. + +### `ContentAddressedStore` class + +The content addressed store also handles content as bytes but performs two +additional operations: + - de-duplicates data based on the content of the data (in other words, two + identical blobs of data will only be stored once + - transforms the data prior to storing; we currently only compress the data but + other operations are possible. + +Data is always de-duplicated but you can choose to skip the transformation step +by telling the content address store that the data should be stored `raw` (ie: +with no transformation). Note that the de-duplication logic happens *prior* to +any transformation (so the transformation itself will not impact the de-duplication +logic). + +Content stored by the content addressed store is addressable using a `key` which is +returned when `save_blobs` is called. `raw` objects can also directly be accessed +using a `uri` (also returned by `save_blobs`); the `uri` will point to the location +of the `raw` bytes in the underlying `DataStoreStorage` (so for exmaple a local +filesystem path or a S3 path). Objects that are not `raw` do not return a `uri` +as they should only be accessed through the content addressed store. + +The symmetrical function to `save_blobs` is `load_blobs` which takes a list of +keys (returned by `save_blobs`) and loads all the objects requested. Note that +at this level of abstraction, there is no `metadata` for the blobs; other +mechanisms exist to store, for example, task metadata or information about +artifacts. + +#### Implementation detail + +The content addressed store contains several (well currently only a pair) of +functions named `_pack_vX` and `_unpack_vX`. They effectively correspond to +the transformations (both transformation to store and reverse transformation +to load) the data undergoes prior to being stored. The `X` corresponds to the +version of the transformation allowing new transformations to be added easily. +A backward compatible `_unpack_backward_compatible` method also allows this +datastore to read any data that was stored with a previous version of the +datastore. Note that going forward, if a new datastore implements `_pack_v2` and +`_unpack_v2`, this datastore would not be able to unpack things packed with +`_pack_v2` but would throw a clear error as to what is happening. + +### `TaskDataStore` class + +This is the meatiest class and contains most of the functionality that an executing +task will use. The `TaskDataStore` is also used when accessing information and +artifacts through the Metaflow Client. + +#### Overview + +At a high level, the `TaskDataStore` is responsible for: + - storing artifacts (functions like `save_artifacts`, `persist` help with this) + - storing other metadata about the task execution; this can include logs, + general information about the task, user-level metadata and any other information + the user wishes the persist about the task. Functions for this include + `save_logs` and `save_metadata`. Internally, functions like `done` will + also store information about the task. + +Artifacts are stored using the `ContentAddressedStore` that is common to all +tasks in a flow; all other data and metadata is stored using the `DataStoreStorage` +directly at a location indicated by the `pathspec` of the task. + +#### Saving artifacts + +To save artifacts, the `TaskDataStore` will first pickle the artifacts, thereby +transforming a Python object into bytes. Those bytes will then be passed down +to the `ContentAddressedStore`. In other words, in terms of data transformation: + - Initially you have a pickle-able Python object + - `TaskDataStore` pickles it and transforms it to `bytes` + - Those `bytes` are then de-duped by the `ContentAddressedStore` + - The `ContentAddressedStore` will also gzip the `bytes` and store them + in the storage backend. + +Crucially, the `TaskDataStore` takes (and returns when loading artifacts) +Python objects whereas the `ContentAddressedStore` only operates with bytes. + +#### Saving metadata and logs + +Metadata and logs are stored directly as files using the `DataStoreStorage` to create +and write to a file. The name of the file is something that `TaskDataStore` +determines internally. + +### `FlowDataStore` class + +The `FlowDataStore` class doesn't do much except give access to `TaskDataStore` +(in effect, it creates the `TaskDataStore` objects to use) and also allows +files to be stored in the `ContentAddressedStore` directly. This is used to +store, for example, code packages. File stored using the `save_data` method +are stored in `raw` format (as in, they are not further compressed). They will, +however, still be de-duped. + +### Caching + +The datastore allows the inclusion of caching at the `ContentAddressedStore` level: + - for blobs (basically the objects returned by `load_blobs` in the + `ContentAddressedStore`). Objects in this cache have gone through: reading + from the backend storage system and the data transformations in + `ContentAddressedStore`. + +The datastore does not determine how and where to cache the data and simply +calls the functions `load_key` and `store_key` on a cache configured by the user +using `set_blob_cache`. +`load_key` is expected to return the object in the cache (if present) or None otherwise. +`store_key` takes a key (the one passed to `load`) and the object to store. The +outside cache is free to implement its own policies and/or own behavior for the +`load_key` and `store_key` functions. + +As an example, the `FileCache` uses the `blob_cache` construct to write to +a file anything passed to `store_key` and returns it by reading from the file +when `load_key` is called. The persistence of the file is controlled by the +`FileCache` so an artifact `store_key`ed may vanish from the cache and would +be re-downloaded by the datastore when needed (and then added to the cache +again). diff --git a/metaflow/cli.py b/metaflow/cli.py index 7d5e67aefde..07b6b7e06e6 100644 --- a/metaflow/cli.py +++ b/metaflow/cli.py @@ -20,7 +20,8 @@ from .task import MetaflowTask from .exception import CommandException, MetaflowException from .graph import FlowGraph -from .datastore import DATASTORES +from .datastore import DATASTORES, FlowDataStore, TaskDataStoreSet + from .runtime import NativeRuntime from .package import MetaflowPackage from .plugins import ENVIRONMENTS, LOGGING_SIDECARS, METADATA_PROVIDERS, MONITOR_SIDECARS @@ -211,14 +212,6 @@ def dump(obj, 'max_value_size': max_value_size, 'include': {t for t in include.split(',') if t}} - if obj.datastore.datastore_root is None: - obj.datastore.datastore_root = obj.datastore.get_datastore_root_from_config( - obj.echo, create_on_absent=False) - if obj.datastore.datastore_root is None: - raise CommandException( - "Could not find the location of the datastore -- did you correctly set the " - "METAFLOW_DATASTORE_SYSROOT_%s environment variable" % (obj.datastore.TYPE).upper()) - # Pathspec can either be run_id/step_name or run_id/step_name/task_id. parts = input_path.split('/') if len(parts) == 2: @@ -230,16 +223,10 @@ def dump(obj, raise CommandException("input_path should either be run_id/step_name" "or run_id/step_name/task_id") - from metaflow.datastore.datastore_set import MetaflowDatastoreSet - - datastore_set = MetaflowDatastoreSet( - obj.datastore, - obj.flow.name, + datastore_set = TaskDataStoreSet( + obj.flow_datastore, run_id, steps=[step_name], - metadata=obj.metadata, - monitor=obj.monitor, - event_logger=obj.event_logger, prefetch_data_artifacts=kwargs.get('include')) if task_id: ds_list = [datastore_set.get_with_pathspec(input_path)] @@ -317,14 +304,8 @@ def logs(obj, raise CommandException("input_path should either be run_id/step_name " "or run_id/step_name/task_id") - if obj.datastore.datastore_root is None: - obj.datastore.datastore_root = obj.datastore.get_datastore_root_from_config( - obj.echo, create_on_absent=False) - if obj.datastore.datastore_root is None: - raise CommandException( - "Could not find the location of the datastore -- did you correctly set the " - "METAFLOW_DATASTORE_SYSROOT_%s environment variable" % (obj.datastore.TYPE).upper()) - + datastore_set = TaskDataStoreSet( + obj.flow_datastore, run_id, steps=[step_name], allow_not_done=True) if task_id: ds_list = [obj.datastore(obj.flow.name, run_id=run_id, @@ -333,17 +314,7 @@ def logs(obj, mode='r', allow_unsuccessful=True)] else: - from metaflow.datastore.datastore_set import MetaflowDatastoreSet - datastore_set = MetaflowDatastoreSet( - obj.datastore, - obj.flow.name, - run_id, - steps=[step_name], - metadata=obj.metadata, - monitor=obj.monitor, - event_logger=obj.event_logger) - # get all successful tasks - ds_list = list(datastore_set) + ds_list = list(datastore_set) # get all tasks if ds_list: def echo_unicode(line, **kwargs): @@ -380,17 +351,11 @@ def echo_unicode(line, **kwargs): log = ds.load_log_legacy(stream) if log and timestamps: raise CommandException("We can't show --timestamps for " - "old runs. Sorry!") + "old runs. Sorry!") echo_unicode(log, nl=False) - - elif len(parts) == 2: - # TODO if datastore provided a way to find unsuccessful task IDs, we - # could make handle this case automatically - raise CommandException("Successful tasks were not found at the given " - "path. You can see logs for unsuccessful tasks " - "by giving an exact task ID using the " - "run_id/step_name/task_id format.") - + else: + raise CommandException("No Tasks found at the given path -- " + "either none exist or none have started yet") # TODO - move step and init under a separate 'internal' subcommand @@ -482,11 +447,6 @@ def step(ctx, if decospecs: decorators._attach_decorators_to_step(func, decospecs) - ctx.obj.datastore.datastore_root = ctx.obj.datastore_root - if ctx.obj.datastore.datastore_root is None: - ctx.obj.datastore.datastore_root = \ - ctx.obj.datastore.get_datastore_root_from_config(ctx.obj.echo) - step_kwargs = ctx.params # Remove argument `step_name` from `step_kwargs`. step_kwargs.pop('step_name', None) @@ -499,7 +459,7 @@ def step(ctx, paths = decompress_list(input_paths) if input_paths else [] task = MetaflowTask(ctx.obj.flow, - ctx.obj.datastore, + ctx.obj.flow_datastore, ctx.obj.metadata, ctx.obj.environment, ctx.obj.echo, @@ -548,15 +508,11 @@ def init(obj, run_id=None, task_id=None, tags=None, **kwargs): # user-specified parameters are often defined as environment # variables. - if obj.datastore.datastore_root is None: - obj.datastore.datastore_root = \ - obj.datastore.get_datastore_root_from_config(obj.echo) - obj.metadata.add_sticky_tags(tags=tags) runtime = NativeRuntime(obj.flow, obj.graph, - obj.datastore, + obj.flow_datastore, obj.metadata, obj.environment, obj.package, @@ -650,7 +606,7 @@ def resume(obj, runtime = NativeRuntime(obj.flow, obj.graph, - obj.datastore, + obj.flow_datastore, obj.metadata, obj.environment, obj.package, @@ -698,7 +654,7 @@ def run(obj, runtime = NativeRuntime(obj.flow, obj.graph, - obj.datastore, + obj.flow_datastore, obj.metadata, obj.environment, obj.package, @@ -739,12 +695,8 @@ def before_run(obj, tags, decospecs): obj.check(obj.graph, obj.flow, obj.environment, pylint=obj.pylint) #obj.environment.init_environment(obj.logger) - if obj.datastore.datastore_root is None: - obj.datastore.datastore_root = \ - obj.datastore.get_datastore_root_from_config(obj.echo) - decorators._init_step_decorators( - obj.flow, obj.graph, obj.environment, obj.datastore, obj.logger) + obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger) obj.metadata.add_sticky_tags(tags=tags) # Package working directory only once per run. @@ -878,19 +830,33 @@ def start(ctx, ctx.obj.flow, ctx.obj.event_logger, ctx.obj.monitor) - ctx.obj.datastore = DATASTORES[datastore] + + ctx.obj.datastore_impl = DATASTORES[datastore] if datastore_root is None: datastore_root = \ - ctx.obj.datastore.get_datastore_root_from_config(ctx.obj.echo) - ctx.obj.datastore_root = ctx.obj.datastore.datastore_root = datastore_root + ctx.obj.datastore_impl.get_datastore_root_from_config(ctx.obj.echo) + if datastore_root is None: + raise CommandException( + "Could not find the location of the datastore -- did you correctly set the " + "METAFLOW_DATASTORE_SYSROOT_%s environment variable?" % datastore.upper()) + + ctx.obj.datastore_impl.datastore_root = datastore_root + + FlowDataStore.default_storage_impl = ctx.obj.datastore_impl + ctx.obj.flow_datastore = FlowDataStore( + ctx.obj.flow.name, + ctx.obj.environment, + ctx.obj.metadata, + ctx.obj.event_logger, + ctx.obj.monitor) # It is important to initialize flow decorators early as some of the # things they provide may be used by some of the objects initialize after. decorators._init_flow_decorators(ctx.obj.flow, ctx.obj.graph, ctx.obj.environment, - ctx.obj.datastore, + ctx.obj.flow_datastore, ctx.obj.metadata, ctx.obj.logger, echo, @@ -903,7 +869,7 @@ def start(ctx, current._set_env(flow_name=ctx.obj.flow.name, is_running=False) parameters.set_parameter_context(ctx.obj.flow.name, ctx.obj.echo, - ctx.obj.datastore) + ctx.obj.flow_datastore) if ctx.invoked_subcommand not in ('run', 'resume'): # run/resume are special cases because they can add more decorators with --with, @@ -911,7 +877,7 @@ def start(ctx, decorators._attach_decorators( ctx.obj.flow, ctx.obj.environment.decospecs()) decorators._init_step_decorators( - ctx.obj.flow, ctx.obj.graph, ctx.obj.environment, ctx.obj.datastore, ctx.obj.logger) + ctx.obj.flow, ctx.obj.graph, ctx.obj.environment, ctx.obj.flow_datastore, ctx.obj.logger) #TODO (savin): Enable lazy instantiation of package ctx.obj.package = None if ctx.invoked_subcommand is None: diff --git a/metaflow/client/core.py b/metaflow/client/core.py index 046e8a44872..861b5ea2596 100644 --- a/metaflow/client/core.py +++ b/metaflow/client/core.py @@ -3,6 +3,7 @@ import os import tarfile import json +from io import BytesIO from collections import namedtuple from itertools import chain @@ -34,7 +35,7 @@ 'type', 'task']) -filecache = FileCache() +filecache = None current_namespace = False current_metadata = False @@ -246,7 +247,7 @@ def __iter__(self): # filtering on namespace on flows means finding at least one # run in this namespace. This is_in_namespace() function # does this properly in this case - all_flows = self.metadata.get_object('root', 'flow') + all_flows = self.metadata.get_object('root', 'flow', None) all_flows = all_flows if all_flows else [] for flow in all_flows: try: @@ -614,17 +615,24 @@ class MetaflowCode(object): """ def __init__(self, flow_name, code_package): + global filecache + self._flow_name = flow_name info = json.loads(code_package) self._path = info['location'] self._ds_type = info['ds_type'] self._sha = info['sha'] - with filecache.get_data(self._ds_type, self._flow_name, self._sha) as f: - self._tar = tarfile.TarFile(fileobj=f) - # The JSON module in Python3 deals with Unicode. Tar gives bytes. - info_str = self._tar.extractfile('INFO').read().decode('utf-8') - self._info = json.loads(info_str) - self._flowspec = self._tar.extractfile(self._info['script']).read() + + if filecache is None: + filecache = FileCache() + code_obj = BytesIO( + filecache.get_data( + self._ds_type, self._flow_name, self._path, self._sha)) + self._tar = tarfile.open(fileobj=code_obj, mode='r:gz') + # The JSON module in Python3 deals with Unicode. Tar gives bytes. + info_str = self._tar.extractfile('INFO').read().decode('utf-8') + self._info = json.loads(info_str) + self._flowspec = self._tar.extractfile(self._info['script']).read() @property def path(self): @@ -687,7 +695,7 @@ class DataArtifact(MetaflowObject): data : object The unpickled representation of the data contained in this artifact sha : string - SHA encoding representing the unique identity of this artifact + Encoding representing the unique identity of this artifact finished_at : datetime Alias for created_at """ @@ -706,11 +714,30 @@ def data(self): object Object contained in this artifact """ + global filecache + ds_type = self._object['ds_type'] - sha = self._object['sha'] - with filecache.get_data(ds_type, self.path_components[0], sha) as f: - obj = pickle.load(f) - return obj + location = self._object['location'] + components = self.path_components + if filecache is None: + # TODO: Pass proper environment to properly extract artifacts + filecache = FileCache() + # "create" the metadata information that the datastore needs + # to access this object. + # TODO: We can store more information in the metadata, particularly + # to determine if we need an environment to unpickle the artifact. + meta = { + 'objects': {self._object['name']: self._object['sha']}, + 'info': {self._object['name']: { + 'size': 0, 'type': None, 'encoding': self._object['content_type']}} + } + if location.startswith(':root:'): + return filecache.get_artifact( + ds_type, location[6:], meta, *components) + else: + # Older artifacts have a location information which we can use. + return filecache.get_artifact_by_location( + ds_type, location, meta, *components) # TODO add # @property @@ -725,7 +752,7 @@ def sha(self): """ Unique identifier for this artifact. - This is the SHA1 hash of the artifact. + This is a unique hash of the artifact (historically SHA1 hash) Returns ------- @@ -1072,17 +1099,15 @@ def loglines(self, stream, as_unicode=True): it is returned as a (unicode) string. """ from metaflow.mflog.mflog import merge_logs - from metaflow.mflog import LOG_SOURCES - from metaflow.datastore import DATASTORES + global filecache ds_type = self.metadata_dict.get('ds-type') ds_root = self.metadata_dict.get('ds-root') - - ds_cls = DATASTORES.get(ds_type, None) - if ds_cls is None: - raise MetaflowInternalError('Datastore %s was not found' % ds_type) - ds_cls.datastore_root = ds_root - + if ds_type is None or ds_root is None: + yield None, '' + return + if filecache is None: + filecache = FileCache() # It is possible that a task fails before any metadata has been # recorded. In this case, we assume that we are executing the # first attempt. @@ -1091,28 +1116,25 @@ def loglines(self, stream, as_unicode=True): # here. It is possible that logs exists for a newer attempt that # just failed to record metadata. We could make this logic more robust # and guarantee that we always return the latest available log. - - ds = ds_cls(self._object['flow_id'], - run_id=str(self._object['run_number']), - step_name=self._object['step_name'], - task_id=str(self._object['task_id']), - mode='r', - attempt=int(self.metadata_dict.get('attempt', 0)), - allow_unsuccessful=True) - logs = ds.load_logs(LOG_SOURCES, stream) + attempt = int(self.metadata_dict.get('attempt', 0)) + logs = filecache.get_logs_stream( + ds_type, ds_root, stream, attempt, *self.path_components) for line in merge_logs([blob for _, blob in logs]): msg = to_unicode(line.msg) if as_unicode else line.msg yield line.utc_tstamp, msg def _load_log_legacy(self, log_location, logtype, as_unicode=True): # this function is used to load pre-mflog style logfiles - ret_val = None + global filecache + log_info = json.loads(log_location) + location = log_info['location'] ds_type = log_info['ds_type'] attempt = log_info['attempt'] - components = self.path_components - with filecache.get_log_legacy(ds_type, logtype, int(attempt), *components) as f: - ret_val = f.read() + if filecache is None: + filecache = FileCache() + ret_val = filecache.get_log_legacy( + ds_type, location, logtype, int(attempt), *self.path_components) if as_unicode and (ret_val is not None): return ret_val.decode(encoding='utf8') else: diff --git a/metaflow/client/filecache.py b/metaflow/client/filecache.py index 386fa5be147..2b8935838a7 100644 --- a/metaflow/client/filecache.py +++ b/metaflow/client/filecache.py @@ -4,7 +4,8 @@ from tempfile import NamedTemporaryFile from hashlib import sha1 -from metaflow.datastore import DATASTORES +from metaflow.datastore import DATASTORES, FlowDataStore +from metaflow.datastore.content_addressed_store import BlobCache from metaflow.exception import MetaflowException from metaflow.metaflow_config import CLIENT_CACHE_PATH, CLIENT_CACHE_MAX_SIZE @@ -14,9 +15,7 @@ class FileCacheException(MetaflowException): headline = 'File cache error' - class FileCache(object): - def __init__(self, cache_dir=None, max_size=None): self._cache_dir = cache_dir self._max_size = max_size @@ -25,16 +24,161 @@ def __init__(self, cache_dir=None, max_size=None): if self._max_size is None: self._max_size = int(CLIENT_CACHE_MAX_SIZE) self._total = 0 + self._objects = None + # We have a separate blob_cache per flow and datastore type. + self._blob_caches = {} + + # We also keep a cache for FlowDataStore objects because some of them + # may have long-lived persistent connections; this is purely a + # performance optimization. We do *not* keep track of task datastores + # to refresh them as needed. Caching FlowDataStore has no adverse + # affect in terms of having to refresh the cache. + self._store_caches = {} + + @property + def cache_dir(self): + return self._cache_dir + + def get_logs_stream( + self, ds_type, ds_root, stream, attempt, flow_name, run_id, + step_name, task_id): + from metaflow.mflog import LOG_SOURCES + + ds = self._get_flow_datastore(ds_type, ds_root, flow_name) + + task_ds = ds.get_task_datastore( + run_id, step_name, task_id, + data_metadata={'objects': {}, 'info': {}}) + return task_ds.load_logs(LOG_SOURCES, stream, attempt_override=attempt) + + def get_log_legacy( + self, ds_type, location, logtype, attempt, flow_name, run_id, + step_name, task_id): + + ds_cls = self._get_datastore_storage_impl(ds_type) + ds_root = ds_cls.path_join(*ds_cls.path_split(location)[:-5]) + cache_id = self._flow_ds_type(ds_type, ds_root, flow_name) + + token = '%s.cached' % sha1(os.path.join( + run_id, step_name, task_id, '%s_log' % logtype).\ + encode('utf-8')).hexdigest() + path = os.path.join(self._cache_dir, cache_id, token[:2], token) + + cached_log = self.read_file(path) + if cached_log is not None: + return cached_log + + ds = self._get_flow_datastore(ds_type, ds_root, flow_name) + + task_ds = ds.get_task_datastore( + run_id, step_name, task_id, + data_metadata={'objects': {}, 'info': {}}) + + log = task_ds.load_log_legacy(logtype, attempt_override=attempt) + # Store this in the file cache as well + self.create_file(path, log) + return log + + def get_data(self, ds_type, flow_name, location, key): + ds_cls = self._get_datastore_storage_impl(ds_type) + ds_root = ds_cls.get_datastore_root_from_location(location, flow_name) + ds = self._get_flow_datastore(ds_type, ds_root, flow_name) + + return ds.load_data([key], force_raw=True)[0] + + def get_artifact_by_location( + self, ds_type, location, data_metadata, flow_name, run_id, + step_name, task_id, name): + ds_cls = self._get_datastore_storage_impl(ds_type) + ds_root = ds_cls.get_datastore_root_from_location(location, flow_name) + return self.get_artifact( + ds_type, ds_root, data_metadata, flow_name, run_id, step_name, + task_id, name) + + def get_artifact( + self, ds_type, ds_root, data_metadata, flow_name, run_id, step_name, + task_id, name): + _, obj = next(self.get_artifacts( + ds_type, ds_root, data_metadata, flow_name, run_id, step_name, + task_id, [name])) + return obj + + def get_all_artifacts( + self, ds_type, ds_root, data_metadata, flow_name, run_id, step_name, + task_id): + ds = self._get_flow_datastore(ds_type, ds_root, flow_name) + + # We get the task datastore for this task + task_ds = ds.get_task_datastore( + run_id, step_name, task_id, data_metadata=data_metadata) + # This will reuse the blob cache if needed. We do not have an + # artifact cache so the unpickling happens every time here. + return task_ds.load_artifacts([n for n, _ in task_ds.items()]) + + def get_artifacts( + self, ds_type, ds_root, data_metadata, flow_name, run_id, step_name, + task_id, names): + ds = self._get_flow_datastore(ds_type, ds_root, flow_name) + + # We get the task datastore for this task + task_ds = ds.get_task_datastore( + run_id, step_name, task_id, data_metadata=data_metadata) + # note that load_artifacts uses flow_datastore.castore which goes + # through one of the self._blob_cache + return task_ds.load_artifacts(names) + + def create_file(self, path, value): + if self._objects is None: + # Index objects lazily (when we first need to write to it). + # This can be an expensive operation + self._index_objects() + dirname = os.path.dirname(path) + try: + FileCache._makedirs(dirname) + except: # noqa E722 + raise FileCacheException( + 'Could not create directory: %s' % dirname) + tmpfile = NamedTemporaryFile( + dir=dirname, prefix='dlobj', delete=False) + # Now write out the file + try: + tmpfile.write(value) + tmpfile.flush() + os.rename(tmpfile.name, path) + except: # noqa E722 + os.unlink(tmpfile.name) + raise + size = os.path.getsize(path) + self._total += size + self._objects.append((int(time.time()), size, path)) + self._garbage_collect() + + def read_file(self, path): + if os.path.exists(path): + try: + with open(path, 'rb') as f: + return f.read() + except IOError: + # It may have been concurrently garbage collected by another + # process + pass + return None def _index_objects(self): objects = [] if os.path.exists(self._cache_dir): - for subdir in os.listdir(self._cache_dir): - root = os.path.join(self._cache_dir, subdir) - if os.path.isdir(root): + for flow_ds_type in os.listdir(self._cache_dir): + root = os.path.join(self._cache_dir, flow_ds_type) + if not os.path.isdir(root): + continue + for subdir in os.listdir(root): + root = os.path.join(self._cache_dir, flow_ds_type, subdir) + if not os.path.isdir(root): + continue for obj in os.listdir(root): - if obj.endswith('.cached'): + sha, ext = os.path.splitext(obj) + if ext in ['cached', 'blob']: path = os.path.join(root, obj) objects.insert(0, (os.path.getctime(path), os.path.getsize(path), @@ -43,10 +187,9 @@ def _index_objects(self): self._total = sum(size for _, size, _ in objects) self._objects = sorted(objects, reverse=False) - def _object_path(self, flow_name, run_id, step_name, task_id, name): - token = os.path.join(flow_name, run_id, step_name, task_id, name).encode('utf-8') - sha = sha1(token).hexdigest() - return os.path.join(self._cache_dir, sha[:2], sha + '.cached') + @staticmethod + def _flow_ds_type(ds_type, ds_root, flow_name): + return '.'.join([ds_type, ds_root, flow_name]) def _garbage_collect(self): now = time.time() @@ -61,7 +204,8 @@ def _garbage_collect(self): # maybe another client had already GC'ed the file away pass - def _makedirs(self, path): + @staticmethod + def _makedirs(path): # this is for python2 compatibility. # Python3 has os.makedirs(exist_ok=True). try: @@ -72,70 +216,47 @@ def _makedirs(self, path): else: raise - def get_log_legacy(self, ds_type, logtype, attempt, flow_name, run_id, step_name, task_id): - path = self._object_path(flow_name, run_id, step_name, task_id, '_log%s' % logtype) - - def load_func(ds): - return ds.load_log_legacy(logtype, attempt_override=attempt) - - return self._internal_get_data( - ds_type, flow_name, run_id, step_name, task_id, path, load_func) + @staticmethod + def _get_datastore_storage_impl(ds_type): + storage_impl = DATASTORES.get(ds_type, None) + if storage_impl is None: + raise FileCacheException('Datastore %s was not found' % ds_type) + return storage_impl - def get_data(self, ds_type, flow_name, sha): - path = self._object_path(flow_name, '_', '_', '_', sha) + def _get_flow_datastore(self, ds_type, ds_root, flow_name): + cache_id = self._flow_ds_type(ds_type, ds_root, flow_name) + cached_flow_datastore = self._store_caches.get(cache_id) - def load_func(ds): - return ds.load_data(sha) + if cached_flow_datastore: + return cached_flow_datastore + else: + storage_impl = self._get_datastore_storage_impl(ds_type) + cached_flow_datastore = FlowDataStore( + flow_name=flow_name, + environment=None, # TODO: Add environment here + storage_impl=storage_impl, + ds_root=ds_root) + blob_cache = self._blob_caches.setdefault( + cache_id, FileBlobCache(self, cache_id)) + cached_flow_datastore.ca_store.set_blob_cache(blob_cache) + self._store_caches[cache_id] = cached_flow_datastore + return cached_flow_datastore - return self._internal_get_data( - ds_type, flow_name, None, None, None, path, load_func) +class FileBlobCache(BlobCache): - def _internal_get_data(self, ds_type, flow_name, run_id, step_name, task_id, path, load_func): - ds_cls = DATASTORES.get(ds_type, None) - if ds_cls is None: - raise FileCacheException('Datastore %s was not found' % ds_type) + def __init__(self, filecache, cache_id): + self._filecache = filecache + self._cache_id = cache_id - if ds_cls.datastore_root is None: - def print_clean(line, **kwargs): - print(line) - ds_cls.datastore_root = ds_cls.get_datastore_root_from_config( - print_clean, create_on_absent=False) + def _path(self, key): + key_dir = key[:2] + return os.path.join( + self._filecache.cache_dir, self._cache_id, key_dir, '%s.blob' % key) - if ds_cls.datastore_root is None: - raise FileCacheException('Cannot locate datastore root') + def load_key(self, key): + return self._filecache.read_file(self._path(key)) - fileobj = None - if os.path.exists(path): - try: - fileobj = open(path, 'rb') - except IOError: - # maybe another client had already GC'ed the file away - fileobj = None - - if fileobj is None: - if self._objects is None: - # index objects lazily at the first request. This can be - # an expensive operation - self._index_objects() - ds = ds_cls(flow_name, run_id, step_name, task_id, mode='d') - dirname = os.path.dirname(path) - try: - self._makedirs(dirname) - except: # noqa E722 - raise FileCacheException('Could not create directory: %s' % dirname) + def store_key(self, key, blob): + self._filecache.create_file(self._path(key), blob) - tmpfile = NamedTemporaryFile(dir=dirname, prefix='s3obj', delete=False) - try: - tmpfile.write(load_func(ds)) - tmpfile.flush() - os.rename(tmpfile.name, path) - except: # noqa E722 - os.unlink(tmpfile.name) - raise - size = os.path.getsize(path) - self._total += size - self._objects.append((int(time.time()), size, path)) - self._garbage_collect() - fileobj = open(path, 'rb') - return fileobj diff --git a/metaflow/datastore/__init__.py b/metaflow/datastore/__init__.py index d48f3ad1363..f93c0b5aa86 100644 --- a/metaflow/datastore/__init__.py +++ b/metaflow/datastore/__init__.py @@ -1,7 +1,9 @@ +from .inputs import Inputs +from .flow_datastore import FlowDataStore +from .datastore_set import TaskDataStoreSet -from . import local, s3 -from .datastore import Inputs, DataException, MetaflowDataStore -from .datastore_set import MetaflowDatastoreSet +from .local_storage import LocalStorage +from .s3_storage import S3Storage -DATASTORES = {'local': local.LocalDataStore, - 's3': s3.S3DataStore} +DATASTORES = {'local': LocalStorage, + 's3': S3Storage} diff --git a/metaflow/datastore/content_addressed_store.py b/metaflow/datastore/content_addressed_store.py new file mode 100644 index 00000000000..931d33f53e9 --- /dev/null +++ b/metaflow/datastore/content_addressed_store.py @@ -0,0 +1,199 @@ +import gzip + +from collections import namedtuple +from hashlib import sha1 +from io import BytesIO + +from ..exception import MetaflowInternalError +from .exceptions import DataException + + +class ContentAddressedStore(object): + """ + This class is not meant to be overridden and is meant to be common across + different datastores. + """ + + save_blobs_result = namedtuple("save_blobs_result", "uri key") + + def __init__(self, prefix, storage_impl): + """ + Initialize a ContentAddressedStore + + A content-addressed store stores data using a name/key that is a hash + of the content. This means that duplicate content is only stored once. + + Parameters + ---------- + prefix : string + Prefix that will be prepended when storing a file + storage_impl : type + Implementation for the backing storage implementation to use + """ + self._prefix = prefix + self._storage_impl = storage_impl + self.TYPE = self._storage_impl.TYPE + self._blob_cache = None + + def set_blob_cache(self, blob_cache): + self._blob_cache = blob_cache + + def save_blobs(self, blob_iter, raw=False, len_hint=0): + """ + Saves blobs of data to the datastore + + The blobs of data are saved as is if raw is True. If raw is False, the + datastore may process the blobs and they should then only be loaded + using load_blob + + NOTE: The idea here is that there are two modes to access the file once + it is saved to the datastore: + - if raw is True, you would be able to access it directly using the + URI returned; the bytes that are passed in as 'blob' would be + returned directly by reading the object at that URI. You would also + be able to access it using load_blob passing the key returned + - if raw is False, no URI would be returned (the URI would be None) + and you would only be able to access the object using load_blob. + - The API also specifically takes a list to allow for parallel writes + if available in the datastore. We could also make a single + save_blob' API and save_blobs but this seems superfluous + + Parameters + ---------- + blob_iter : Iterator over bytes objects to save + raw : bool, optional + Whether to save the bytes directly or process them, by default False + len_hint : Hint of the number of blobs that will be produced by the + iterator, by default 0 + + Returns + ------- + List of save_blobs_result: + The list order is the same as the blobs passed in. The URI will be + None if raw is False. + """ + results = [] + + def packing_iter(): + for blob in blob_iter: + sha = sha1(blob).hexdigest() + path = self._storage_impl.path_join(self._prefix, sha[:2], sha) + results.append( + self.save_blobs_result( + uri=self._storage_impl.full_uri(path) if raw else None, + key=sha, + ) + ) + + if not self._storage_impl.is_file([path])[0]: + # only process blobs that don't exist already in the + # backing datastore + meta = {"cas_raw": raw, "cas_version": 1} + if raw: + yield path, (BytesIO(blob), meta) + else: + yield path, (self._pack_v1(blob), meta) + + # We don't actually want to overwrite but by saying =True, we avoid + # checking again saving some operations. We are already sure we are not + # sending duplicate files since we already checked. + self._storage_impl.save_bytes( + packing_iter(), overwrite=True, len_hint=len_hint + ) + return results + + def load_blobs(self, keys, force_raw=False): + """ + Mirror function of save_blobs + + This function is guaranteed to return the bytes passed to save_blob for + the keys + + Parameters + ---------- + keys : List of string + Key describing the object to load + force_raw : bool, optional + Support for backward compatibility with previous datastores. If + True, this will force the key to be loaded as is (raw). By default, + False + + Returns + ------- + Returns an iterator of (string, bytes) tuples + """ + load_paths = [] + for key in keys: + blob = None + if self._blob_cache: + blob = self._blob_cache.load_key(key) + if blob is not None: + yield key, blob + else: + path = self._storage_impl.path_join(self._prefix, key[:2], key) + load_paths.append((key, path)) + + with self._storage_impl.load_bytes( + [p for _, p in load_paths] + ) as loaded: + for (key, _), (_, file_path, meta) in zip(load_paths, loaded): + # At this point, we either return the object as is (if raw) or + # decode it according to the encoding version + with open(file_path, "rb") as f: + if force_raw or (meta and meta.get("cas_raw", False)): + blob = f.read() + else: + if meta is None: + # Previous version of the datastore had no meta + # information + unpack_code = self._unpack_backward_compatible + else: + version = meta.get("cas_version", -1) + if version == -1: + raise DataException( + "Could not extract encoding version for %s" + % path + ) + unpack_code = getattr( + self, "_unpack_v%d" % version, None + ) + if unpack_code is None: + raise DataException( + "Unknown encoding version %d for %s -- " + "the artifact is either corrupt or you " + "need to update Metaflow to the latest " + "version" % (version, path) + ) + try: + blob = unpack_code(f) + except Exception as e: + raise DataException("Could not unpack data: %s" % e) + + if self._blob_cache: + self._blob_cache.store_key(key, blob) + + yield key, blob + + def _unpack_backward_compatible(self, blob): + # This is the backward compatible unpack + # (if the blob doesn't have a version encoded) + return self._unpack_v1(blob) + + def _pack_v1(self, blob): + buf = BytesIO() + with gzip.GzipFile(fileobj=buf, mode="wb", compresslevel=3) as f: + f.write(blob) + buf.seek(0) + return buf + + def _unpack_v1(self, blob): + with gzip.GzipFile(fileobj=blob, mode="rb") as f: + return f.read() + + +class BlobCache(object): + def load_key(self, key): + pass + + def store_key(self, key, blob): + pass diff --git a/metaflow/datastore/datastore.py b/metaflow/datastore/datastore.py deleted file mode 100644 index 5e8447d37a9..00000000000 --- a/metaflow/datastore/datastore.py +++ /dev/null @@ -1,631 +0,0 @@ -import gzip -import os -import sys -import time -from hashlib import sha1 -from functools import partial - -try: - # Python 2 - import cPickle as pickle -except: - # Python 3 - import pickle - -from types import MethodType, FunctionType -from ..event_logger import NullEventLogger -from ..monitor import NullMonitor -from ..parameters import Parameter -from ..exception import MetaflowException, MetaflowInternalError -from ..metadata import DataArtifact -from .. import metaflow_config - -class DataException(MetaflowException): - headline = "Data store error" - -class Inputs(object): - """ - split-and: inputs.step_a.x inputs.step_b.x - foreach: inputs[0].x - both: (inp.x for inp in inputs) - """ - def __init__(self, flows): - # TODO sort by foreach index - self.flows = list(flows) - for flow in self.flows: - setattr(self, flow._current_step, flow) - - def __getitem__(self, idx): - return self.flows[idx] - - def __iter__(self): - return iter(self.flows) - -def only_if_not_done(f): - def method(self, *args, **kwargs): - if self._is_done_set: - raise MetaflowInternalError("Tried to write to datastore "\ - "(method %s) after it was marked "\ - ".done()" % f.func_name) - return f(self, *args, **kwargs) - return method - - -class TransformableObject(object): - # Very simple wrapper class to only keep one transform - # of an object. This is to force garbage collection - # on the transformed object if the transformation is - # successful - def __init__(self, current_object): - self._object = current_object - self._original_type = type(self._object) - - def transform(self, transformer): - # Transformer is a function taking one argument (the current object) and returning another - # object which will replace the current object if transformer does not raise an - # exception - try: - temp = transformer(self._object) - self._object = temp - except: # noqa E722 - raise - - def current(self): - return self._object - - def current_type(self): - return type(self._object) - - def original_type(self): - return self._original_type - - -class MetaflowDataStore(object): - datastore_root = None - - # Datastore needs to implement the methods below - def save_metadata(self, name, data): - """ - Save a task-specific metadata dictionary as JSON. - """ - raise NotImplementedError() - - def load_metadata(self, name): - """ - Load a task-specific metadata dictionary as JSON. - """ - raise NotImplementedError() - - def has_metadata(self, name): - """ - Return True if this metadata file exists. - """ - raise NotImplementedError() - - def save_data(self, sha, transformable_object): - """ - Save a content-addressed data blob if it doesn't exist already. - """ - raise NotImplementedError() - - def load_data(self, sha): - """ - Load a content-addressed data blob. - """ - raise NotImplementedError() - - def save_logs(self, logsource, stream_data): - """ - Save log files for multiple streams, represented as - as a list of (stream, bytes) or (stream, Path) tuples. - """ - raise NotImplementedError() - - def load_log_legacy(self, stream, attempt_override=None): - """ - Load old-style, pre-mflog, log file represented as a bytes object. - """ - raise NotImplementedError() - - def load_logs(self, logsources, stream, attempt_override=None): - """ - Given a list of logsources, return a list of (logsource, logblob) - tuples. Returns empty contents for missing log files. - """ - raise NotImplementedError() - - def done(self): - """ - Write a marker indicating that datastore has finished writing to - this path. - """ - raise NotImplementedError() - - def is_done(self): - """ - A flag indicating whether this datastore directory was closed - successfully with done(). - """ - raise NotImplementedError() - - def object_path(self, sha): - """ - Return URL of an object identified by a sha. - """ - raise NotImplementedError() - - @classmethod - def get_latest_tasks(cls, - flow_name, - run_id=None, - steps=None, - pathspecs=None): - """ - Return a list of (step, task, attempt, metadata_blob) for a subset of - the tasks (consider eventual consistency) for which the latest attempt - is done for the input `flow_name, run_id`. - We filter the list based on `steps` if non-None. - Alternatively, `pathspecs` can contain the exact list of pathspec(s) - (run_id/step_name/task_id) that should be filtered. - Note: When `pathspecs` is specified, we expect strict consistency and - not eventual consistency in contrast to other modes. - """ - raise NotImplementedError() - - @classmethod - def get_artifacts(cls, artifacts_to_prefetch): - """ - Return a list of (sha, obj_blob) for all the object_path(s) specified in - `artifacts_to_prefetch`. - """ - raise NotImplementedError() - - def artifact_path(self, artifact_name): - """ - Return the object_path() for `artifact_name`. - Pre-condition: `artifact_name` is in datastore. - """ - if self.objects: - return self.object_path(self.objects[artifact_name]) - return None - - def get_log_location(self, logsource, stream, attempt_override=None): - """ - Returns a string indicating the location of the log of the specified type. - """ - filename = self.get_log_location_for_attempt(logsource, stream, - attempt_override if attempt_override is not None - else self.attempt) - return os.path.join(self.root, filename) - - @classmethod - def get_datastore_root_from_config(cls, echo, create_on_absent=True): - """ - Returns a default choice for datastore_root from metaflow_config - depending on the datastore type. - """ - return getattr(metaflow_config, - 'DATASTORE_SYSROOT_%s' % cls.TYPE.upper()) - - @classmethod - def decode_gzip_data(cls, filename, fileobj=None): - """ - Returns the gzip data in file with `filename` or passed via `fileobj`. - """ - with gzip.GzipFile(filename, fileobj=fileobj, mode='rb') as f: - return f.read() - - @classmethod - def make_path(cls, - flow_name, - run_id=None, - step_name=None, - task_id=None, - pathspec=None): - """ - Return the path for a given flow using this datastore. - path = cls.datastore_root/flow_name/run_id/step_name/task_id - Callers are expected to invoke this function with a sequence of non-None - values depending on the nested path they want. - For example, - If run_id is None, return path = cls.datastore_root/flow_name - If step_name is None, return path = cls.datastore_root/flow_name/run_id - and so on. - If pathspec is non None, - return path = cls.datastore_root/flow_name/pathspec - """ - sysroot = cls.datastore_root - if pathspec: - return os.path.join(sysroot, flow_name, pathspec) - elif flow_name is None: - return sysroot - elif run_id is None: - return os.path.join(sysroot, flow_name) - elif step_name is None: - return os.path.join(sysroot, flow_name, run_id) - elif task_id is None: - return os.path.join(sysroot, flow_name, run_id, step_name) - else: - return os.path.join(sysroot, flow_name, run_id, step_name, task_id) - - @classmethod - def filename_with_attempt_prefix(cls, name, attempt): - """ - Return the equivalent filename for `name` depending - whether an attempt prefix must be used, if `attempt` isn't None. - """ - if attempt is None: - return name - else: - return '%d.%s' % (attempt, name) - - @classmethod - def get_metadata_filename_for_attempt(cls, attempt): - """ - Return the metadata filename (.data.json) based on `attempt`. - """ - return cls.filename_with_attempt_prefix('data.json', attempt) - - @classmethod - def get_log_location_for_attempt(cls, logprefix, stream, attempt): - """ - Return the log_location based on `logprefix`, `stream` and `attempt`. - """ - fname = '%s_%s.log' % (logprefix, stream) - return cls.filename_with_attempt_prefix(fname, attempt) - - @classmethod - def is_metadata_filename(cls, fname): - """ - Returns if the filename is a metadata filename (ends in .data.json). - """ - return fname.endswith('data.json') - - @classmethod - def get_done_filename_for_attempt(cls, attempt): - """ - Returns the done filename (.DONE.lock) based on `attempt`. - """ - return cls.filename_with_attempt_prefix('DONE.lock', attempt) - - @classmethod - def is_done_filename(cls, fname): - """ - Returns if the filename is a done filename (ends in .DONE.lock). - """ - return fname.endswith('DONE.lock') - - @classmethod - def get_filename_for_attempt(cls, attempt): - """ - Returns the attempt filename (.attempt.json) based on `attempt`. - """ - return cls.filename_with_attempt_prefix('attempt.json', attempt) - - @classmethod - def is_attempt_filename(cls, fname): - """ - Returns if the filename is an attempt filename (ends in .attempt.json). - """ - return fname.endswith('attempt.json') - - @classmethod - def parse_filename(cls, fname): - """ - Parse the filename and returns (name, attempt). - When using old style paths (pre-attempt id), returns - (name=fname, attempt=None). - This is expected to be the converse to - filename_with_attempt_prefix() method. - """ - if len(fname) >= 1 and fname[0] >= '0' and fname[0] <= '9': - # new style paths = . - attempt = int(fname[0]) - name = fname[2:] - else: - # old style paths. - attempt = None - name = fname - return name, attempt - - def __init__(self, - flow_name, - run_id=None, - step_name=None, - task_id=None, - mode='r', - metadata=None, - attempt=None, - event_logger=None, - monitor=None, - data_obj=None, - artifact_cache=None, - allow_unsuccessful=False): - if run_id == 'data': - raise DataException("Run ID 'data' is reserved. " - "Try with a different --run-id.") - if self.datastore_root is None: - raise DataException("Datastore root not found. " - "Specify with METAFLOW_DATASTORE_SYSROOT_%s " - "environment variable." % self.TYPE.upper()) - # NOTE: calling __init__(mode='w') should be a cheap operation: - # no file system accesses are allowed. It is called frequently - # e.g. to resolve log file location. - self.event_logger = event_logger if event_logger else NullEventLogger() - self.monitor = monitor if monitor else NullMonitor() - self.metadata = metadata - self.run_id = run_id - self.step_name = step_name - self.task_id = task_id - self._is_done_set = False - - self._encodings = {'gzip+pickle-v2'} - ver = sys.version_info[0] * 10 + sys.version_info[1] - if ver >= 34: - self._encodings.add('gzip+pickle-v4') - - self.artifact_cache = artifact_cache - if self.artifact_cache is None: - self.artifact_cache = {} - - self.data_root = os.path.join(self.datastore_root, flow_name, 'data') - self.root = self.make_path(flow_name, - run_id, - step_name, - task_id) - - self.attempt = attempt - if mode == 'r': - if data_obj is None: - # what is the latest attempt ID of this data store? - - # In the case of S3, the has_metadata() below makes a - # HEAD request to a non-existent object, which results - # to this object becoming eventually consistent. This - # could result to a situation that has_metadata() misses - # the latest version although it is already existing. - - # As long as nothing opens a datastore for reading before - # writing, this should not be a problem. - - # We have to make MAX_ATTEMPTS HEAD requests, which is - # very unfortunate performance-wise (TODO: parallelize this). - # On AWS Step Functions it is possible that some attempts are - # missing, so we have to check all possible attempt files to - # find the latest one. Compared to doing a LIST operation, - # these checks are guaranteed to be consistent as long as the - # task to be looked up has already finished. - self.attempt = None # backwards-compatibility for pre-attempts. - for i in range(0, metaflow_config.MAX_ATTEMPTS): - if self.has_metadata('%d.attempt' % i, with_attempt=False): - self.attempt = i - - # was the latest attempt completed successfully? - if self.is_done(): - # load the data from the latest attempt - data_obj = self.load_metadata('data') - elif allow_unsuccessful and self.attempt is not None: - # this mode can be used to load_logs, for instance - data_obj = None - else: - raise DataException("Data was not found or not finished at %s"\ - % self.root) - - if data_obj: - self.origin = data_obj.get('origin') - self.objects = data_obj['objects'] - self.info = data_obj.get('info', {}) - elif mode == 'd': - # Direct access mode used by the client. We effectively don't load any - # objects and can only access things using the load_* functions - self.origin = None - self.objects = None - self.info = None - elif mode != 'w': - raise DataException('Unknown datastore mode: %s' % mode) - - def init_task(self): - # this method should be called once after datastore has been opened - # for task-related write operations - self.save_metadata('attempt', {'time': time.time()}) - self.objects = {} - self.info = {} - - - @property - def pathspec(self): - return '%s/%s/%s' % (self.run_id, self.step_name, self.task_id) - - @property - def pathspec_index(self): - idxstr = ','.join(map(str, (f.index for f in self['_foreach_stack']))) - return '%s/%s[%s]' % (self.run_id, self.step_name, idxstr) - - def _save_object(self, transformable_obj, var, force_v4=False): - if force_v4: - blobtype = 'gzip+pickle-v4' - if blobtype not in self._encodings: - raise DataException("Artifact *%s* requires a serialization encoding that " - "requires Python 3.4 or newer." % var) - transformable_obj.transform(lambda x: pickle.dumps(x, protocol=4)) - else: - try: - # to ensure compatibility between python2 and python3, we use the - # highest protocol that works with both the versions - transformable_obj.transform(lambda x: pickle.dumps(x, protocol=2)) - blobtype = 'gzip+pickle-v2' - except (SystemError, OverflowError): - # this happens when you try to serialize an oversized - # object (2GB/4GB+) - blobtype = 'gzip+pickle-v4' - if blobtype not in self._encodings: - raise DataException("Artifact *%s* is very large (over 2GB). " - "You need to use Python 3.4 or newer if " - "you want to serialize large objects." - % var) - transformable_obj.transform(lambda x: pickle.dumps(x, protocol=4)) - sha = sha1(transformable_obj.current()).hexdigest() - sz = len(transformable_obj.current()) - self.save_data(sha, transformable_obj) - return sha, sz, blobtype - - def _load_object(self, sha): - if sha in self.artifact_cache: - blob = self.artifact_cache[sha] - else: - blob = self.load_data(sha) - return pickle.loads(blob) - - @only_if_not_done - def clone(self, origin): - self.save_metadata('data', {'datastore': self.TYPE, - 'version': '0.1', - 'origin': origin.pathspec, - 'python_version': sys.version, - 'objects': origin.objects, - 'info': origin.info}) - self._register_data_artifacts(origin.objects, origin.info) - - @only_if_not_done - def passdown_partial(self, origin, vars): - # Pass-down from datastore origin all information related to vars to - # this datastore. In other words, this adds to the current datastore all - # the variables in vars (obviously, it does not download them or anything but - # records information about them). This is used to propagate parameters between - # datastores without actually loading the parameters - for var in vars: - sha = origin.objects.get(var) - if sha: - self.objects[var] = sha - self.info[var] = origin.info[var] - - @only_if_not_done - def persist(self, flow): - - def serializable_attributes(): - for var in dir(flow): - if var.startswith('__') or var in flow._EPHEMERAL: - continue - # Skip over properties of the class (Parameters) - if hasattr(flow.__class__, var) and \ - isinstance(getattr(flow.__class__, var), property): - continue - val = getattr(flow, var) - if not (isinstance(val, MethodType) or - isinstance(val, FunctionType) or - isinstance(val, Parameter)): - yield var, TransformableObject(val), False - - # initialize with old values... - if flow._datastore: - self.objects.update(flow._datastore.objects) - self.info.update(flow._datastore.info) - - # ...overwrite with new - for var, obj, force_v4 in serializable_attributes(): - sha, size, encoding = self._save_object(obj, var, force_v4) - self.objects[var] = sha - self.info[var] = {'size': size, - 'type': str(obj.original_type()), - 'encoding': encoding} - - self.save_metadata('data', {'datastore': self.TYPE, - 'version': '1.0', - 'attempt': self.attempt, - 'python_version': sys.version, - 'objects': self.objects, - 'info': self.info}) - - self._register_data_artifacts(self.objects, self.info) - - def _register_data_artifacts(self, objects, info): - # register artifacts with the metadata service - artifacts = [DataArtifact(name=var, - ds_type=self.TYPE, - url=self.object_path(sha), - sha=sha, - type=info[var]['encoding']) - for var, sha in objects.items()] - - self.metadata.register_data_artifacts(self.run_id, - self.step_name, - self.task_id, - self.attempt, - artifacts) - - def get(self, var, default=None): - if self.objects: - return self[var] if var in self.objects else default - return default - - # Provides a fast-path to check if a given object is None. - def is_none(self, var): - if not self.info: - return True - info = self.info.get(var) - if info: - obj_type = info.get('type') - # Conservatively check if the actual object is None, in case - # the artifact is stored using a different python version. - if obj_type == str(type(None)): - return True - # Slow path since this has to get the object from S3. - return self.get(var) is None - - def __contains__(self, var): - if self.objects: - return var in self.objects - return False - - def __getitem__(self, var): - # backwards compatibility: we might not have info for all objects - if not self.info: - return None - info = self.info.get(var) - if info: - encoding = info.get('encoding', 'gzip+pickle-v2') - else: - encoding = 'gzip+pickle-v2' - if encoding in self._encodings: - return self._load_object(self.objects[var]) - raise DataException("Artifact *%s* requires a newer version " - "of Python. Try with Python 3.4 or newer." % - var) - - def __iter__(self): - if self.objects: - return iter(self.objects) - return iter([]) - - def __str__(self): - return self.format(show_private=True, max_value_size=1000) - - def items(self): - if self.objects: - return self.objects.items() - return {} - - def to_dict(self, show_private=False, max_value_size=None, include=None): - d = {} - for k, v in self.items(): - if include and k not in include: - continue - if k[0] == '_' and not show_private: - continue - if max_value_size is not None and\ - self.info[k]['size'] > max_value_size: - d[k] = ArtifactTooLarge() - else: - d[k] = self[k] - return d - - def format(self, **kwargs): - def lines(): - for k, v in self.to_dict(**kwargs).items(): - yield k, '*{key}* [size: {size} type: {type}] = {value}'\ - .format(key=k, value=v, **self.info[k]) - return '\n'.join(line for k, line in sorted(lines())) - -class ArtifactTooLarge(object): - def __str__(self): - return '< artifact too large >' diff --git a/metaflow/datastore/datastore_set.py b/metaflow/datastore/datastore_set.py index bb8e79d5d66..c4685e4b8a9 100644 --- a/metaflow/datastore/datastore_set.py +++ b/metaflow/datastore/datastore_set.py @@ -1,53 +1,44 @@ import json -from metaflow.util import to_unicode +from io import BytesIO + +from .exceptions import DataException +from .content_addressed_store import BlobCache """ -MetaflowDatastoreSet allows you to prefetch multiple (read) datastores into a -cache and lets you access them. -As a performance optimization it also lets you prefetch select data artifacts -leveraging a shared cache. +TaskDataStoreSet allows you to prefetch multiple (read) datastores into a +cache and lets you access them. As a performance optimization it also lets you +prefetch select data artifacts leveraging a shared cache. """ -class MetaflowDatastoreSet(object): +class TaskDataStoreSet(object): def __init__(self, - ds_class, - flow_name, + flow_datastore, run_id, steps=None, pathspecs=None, - metadata=None, - event_logger=None, - monitor=None, - prefetch_data_artifacts=None): - data_blobs = ds_class.get_latest_tasks(flow_name, - run_id, - steps=steps, - pathspecs=pathspecs) - artifact_cache = {} - datastores = [ds_class(flow_name, - run_id=run_id, - step_name=step_name, - task_id=task_id, - metadata=metadata, - attempt=attempt, - event_logger=event_logger, - monitor=monitor, - data_obj=json.loads(to_unicode(data_blob)), - artifact_cache=artifact_cache) - for step_name, task_id, attempt, data_blob in data_blobs] + prefetch_data_artifacts=None, + allow_not_done=False): + + task_datastores = flow_datastore.get_latest_task_datastores( + run_id, steps=steps, pathspecs=pathspecs, allow_not_done=allow_not_done) + if prefetch_data_artifacts: - artifacts_to_prefetch = set( - [ds.artifact_path(artifact_name) - for ds in datastores - for artifact_name in prefetch_data_artifacts - if artifact_name in ds]) - - # Update (and not re-assign) the artifact_cache since each datastore - # created above has a reference to this object. - artifact_cache.update(ds_class.get_artifacts(artifacts_to_prefetch)) + # produce a set of SHA keys to prefetch based on artifact names + prefetch = set() + for ds in task_datastores: + prefetch.update(ds.keys_for_artifacts(prefetch_data_artifacts)) + # ignore missing keys + prefetch.discard(None) + + # prefetch artifacts and share them with all datastores + # in this DatastoreSet + preloaded = dict(flow_datastore.ca_store.load_blobs(prefetch)) + cache = ImmutableBlobCache(preloaded) + flow_datastore.ca_store.set_blob_cache(cache) + self.pathspec_index_cache = {} self.pathspec_cache = {} - for ds in datastores: + for ds in task_datastores: self.pathspec_index_cache[ds.pathspec_index] = ds self.pathspec_cache[ds.pathspec] = ds @@ -60,3 +51,19 @@ def get_with_pathspec_index(self, pathspec_index): def __iter__(self): for v in self.pathspec_cache.values(): yield v + +""" +This class ensures that blobs that correspond to artifacts that +are common to all datastores in this set are only loaded once +""" +class ImmutableBlobCache(BlobCache): + + def __init__(self, preloaded): + self._preloaded = preloaded + + def load_key(self, key): + return self._preloaded.get(key) + + def store_key(self, key, blob): + # we cache only preloaded keys, so no need to store anything + pass diff --git a/metaflow/datastore/datastore_storage.py b/metaflow/datastore/datastore_storage.py new file mode 100644 index 00000000000..e9f501efdf6 --- /dev/null +++ b/metaflow/datastore/datastore_storage.py @@ -0,0 +1,244 @@ +from collections import namedtuple +import re + +from .exceptions import DataException + + +class CloseAfterUse(object): + """ + Class that can be used to wrap data and a closer (cleanup code). + This class should be used in a with statement and, when the with + scope exits, `close` will be called on the closer object + """ + def __init__(self, data, closer=None): + self.data = data + self._closer = closer + + def __enter__(self): + return self.data + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._closer: + self._closer.close() + + +class DataStoreStorage(object): + """ + A DataStoreStorage defines the interface of communication between the + higher-level datastores and the actual storage system. + + Both the ContentAddressedStore and the TaskDataStore use these methods to + read/write/list from the actual storage system. These methods are meant to + be low-level; they are in a class to provide better abstraction but this + class itself is not meant to be initialized. + """ + TYPE = None + datastore_root = None + path_rexp = None + + list_content_result = namedtuple('list_content_result', 'path is_file') + + def __init__(self, root=None): + self.datastore_root = root if root else self.datastore_root + + @classmethod + def get_datastore_root_from_config(cls, echo, create_on_absent=True): + """Returns a default choice for datastore_root from metaflow_config + + Parameters + ---------- + echo : function + Function to use to print out messages + create_on_absent : bool, optional + Create the datastore root if it doesn't exist, by default True + """ + raise NotImplementedError + + @classmethod + def get_datastore_root_from_location(cls, path, flow_name): + """Extracts the datastore_root location from a path using + a content-addressed store. + + NOTE: This leaks some detail of the content-addressed store so not ideal + + This method will raise an exception if the flow_name is not as expected + + Parameters + ---------- + path : str + Location from which to extract the datastore root value + flow_name : str + Flow name (for verification purposes) + + Returns + ------- + str + The datastore_root value that can be used to initialize an instance + of this datastore storage. + + Raises + ------ + DataException + Raised if the path is not a valid path from this datastore. + """ + if cls.path_rexp is None: + cls.path_rexp = re.compile(cls.path_join( + '(?P.*)', + '(?P[_a-zA-Z][_a-zA-Z0-9]+)', + 'data', + '(?P[0-9a-f]{2})', + '(?:r_)?(?P=init)[0-9a-f]{38}')) + m = cls.path_rexp.match(path) + if not m or m.group('flow_name') != flow_name: + raise DataException( + "Location %s does not correspond to a valid location for " + "flow %s." % (path, flow_name)) + return m.group('root') + + @classmethod + def path_join(cls, *components): + if len(components) == 0: + return '' + component = components[0].rstrip('/') + components = [component] + [c.strip('/') for c in components[1:]] + return '/'.join(components) + + @classmethod + def path_split(cls, path): + return path.split('/') + + @classmethod + def basename(cls, path): + return path.split('/')[-1] + + @classmethod + def dirname(cls, path): + return path.rsplit('/', 1)[0] + + def full_uri(self, path): + return self.path_join(self.datastore_root, path) + + def is_file(self, paths): + """ + Returns True or False depending on whether path refers to a valid + file-like object + + This method returns False if path points to a directory + + Parameters + ---------- + path : List[string] + Path to the object + + Returns + ------- + List[bool] + """ + raise NotImplementedError + + def info_file(self, path): + """ + Returns a tuple where the first element is True or False depending on + whether path refers to a valid file-like object (like is_file) and the + second element is a dictionary of metadata associated with the file or + None if the file does not exist or there is no metadata. + + Parameters + ---------- + path : string + Path to the object + + Returns + ------- + tuple + (bool, dict) + """ + raise NotImplementedError + + def list_content(self, paths): + """ + Lists the content of the datastore in the directory indicated by 'paths'. + + This is similar to executing a 'ls'; it will only list the content one + level down and simply returns the paths to the elements present as well + as whether or not those elements are files (if not, they are further + directories that can be traversed) + + The path returned always include the path passed in. As an example, + if your filesystem contains the files: A/b.txt A/c.txt and the directory + A/D, on return, you would get, for an input of ['A']: + [('A/b.txt', True), ('A/c.txt', True), ('A/D', False)] + + Parameters + ---------- + paths : List[string] + Directories to list + + Returns + ------- + List[list_content_result] + Content of the directory + """ + raise NotImplementedError + + def save_bytes(self, path_and_bytes_iter, overwrite=False, len_hint=0): + """ + Creates objects and stores them in the datastore. + + If overwrite is False, any existing object will not be overwritten and + an error will be returned. + + The objects are specified in an iterator over (path, obj) tuples where + the path is the path to store the object and the value is a file-like + object from which bytes can be read. + + Parameters + ---------- + path_and_bytes_iter : Iterator[(string, (RawIOBase|BufferedIOBase, metadata))] + Iterator over objects to store; the first element in the outermost + tuple is the path to store the bytes at. The second element in the + outermost tuple is either a RawIOBase or BufferedIOBase or a tuple + where the first element is a RawIOBase or BufferedIOBase and the + second element is a dictionary of metadata to associate with the + object. + Keys for the metadata must be ascii only string and elements + can be anything that can be converted to a string using json.dumps. + If you have no metadata, you can simply pass a RawIOBase or + BufferedIOBase. + overwrite : bool + True if the objects can be overwritten. Defaults to False. + len_hint : int + Estimated number of items produced by the iterator + + Returns + ------- + None + """ + raise NotImplementedError + + def load_bytes(self, paths): + """ + Gets objects from the datastore + + Note that objects may be fetched in parallel so if order is important + for your consistency model, the caller is responsible for calling this + multiple times in the proper order. + + Parameters + ---------- + paths : List[string] + Paths to fetch + + Returns + ------- + CloseAfterUse : + A CloseAfterUse which should be used in a with statement. The data + in the CloseAfterUse will be an iterator over (key, path, metadata) + tuples. Path and metadata will be None if the key was missing. + Metadata will be None if no metadata is present; otherwise it is + a dictionary of metadata associated with the object. + + Note that the file at `path` may no longer be accessible outside of + the scope of the returned object. + """ + raise NotImplementedError diff --git a/metaflow/datastore/exceptions.py b/metaflow/datastore/exceptions.py new file mode 100644 index 00000000000..139a2c25f43 --- /dev/null +++ b/metaflow/datastore/exceptions.py @@ -0,0 +1,4 @@ +from ..exception import MetaflowException + +class DataException(MetaflowException): + headline = "Data store error" \ No newline at end of file diff --git a/metaflow/datastore/flow_datastore.py b/metaflow/datastore/flow_datastore.py new file mode 100644 index 00000000000..71e1f9b2e38 --- /dev/null +++ b/metaflow/datastore/flow_datastore.py @@ -0,0 +1,215 @@ +import itertools +import json + +from .. import metaflow_config + +from .content_addressed_store import ContentAddressedStore +from .task_datastore import TaskDataStore + +class FlowDataStore(object): + default_storage_impl = None + + def __init__(self, + flow_name, + environment, + metadata=None, + event_logger=None, + monitor=None, + storage_impl=None, + ds_root=None): + """ + Initialize a Flow level datastore. + + This datastore can then be used to get TaskDataStore to store artifacts + and metadata about a task as well as a ContentAddressedStore to store + things like packages, etc. + + Parameters + ---------- + flow_name : str + The name of the flow + environment : MetaflowEnvironment + Environment this datastore is operating in + metadata : MetadataProvider, optional + The metadata provider to use and update if needed, by default None + event_logger : EventLogger, optional + EventLogger to use to report events, by default None + monitor : Monitor, optional + Monitor to use to measure/monitor events, by default None + storage_impl : type + Class for the backing DataStoreStorage to use; if not provided use + default_storage_impl, optional + ds_root : str + The optional root for this datastore; if not provided, use the + default for the DataStoreStorage, optional + """ + storage_impl = storage_impl if storage_impl else \ + self.default_storage_impl + if storage_impl is None: + raise RuntimeError("No datastore storage implementation specified") + self._storage_impl = storage_impl(ds_root) + self.TYPE = self._storage_impl.TYPE + + # Public attributes + self.flow_name = flow_name + self.environment = environment + self.metadata = metadata + self.logger = event_logger + self.monitor = monitor + + self.ca_store = ContentAddressedStore( + self._storage_impl.path_join(self.flow_name, 'data'), + self._storage_impl) + + @property + def datastore_root(self): + return self._storage_impl.datastore_root + + def get_latest_task_datastores( + self, run_id=None, steps=None, pathspecs=None, allow_not_done=False): + """ + Return a list of TaskDataStore for a subset of the tasks. + + We filter the list based on `steps` if non-None. + Alternatively, `pathspecs` can contain the exact list of pathspec(s) + (run_id/step_name/task_id) that should be filtered. + Note: When `pathspecs` is specified, we expect strict consistency and + not eventual consistency in contrast to other modes. + + Parameters + ---------- + run_id : str, optional + Run ID to get the tasks from. If not specified, use pathspecs, + by default None + steps : List[str] , optional + Steps to get the tasks from. If run_id is specified, this + must also be specified, by default None + pathspecs : List[str], optional + Full task specs (run_id/step_name/task_id). Can be used instead of + specifiying run_id and steps, by default None + allow_not_done : bool, optional + If True, returns the latest attempt of a task even if that attempt + wasn't marked as done, by default False + + Returns + ------- + List[TaskDataStore] + Task datastores for all the tasks specified. + """ + task_urls = [] + # Note: When `pathspecs` is specified, we avoid the potentially + # eventually consistent `list_content` operation, and directly construct + # the task_urls list. + if pathspecs: + task_urls = [self._storage_impl.path_join(self.flow_name, pathspec) + for pathspec in pathspecs] + else: + run_prefix = self._storage_impl.path_join(self.flow_name, run_id) + if steps: + step_urls = [self._storage_impl.path_join(run_prefix, step) + for step in steps] + else: + step_urls = [step.path for step in self._storage_impl.list_content( + [run_prefix]) if step.is_file is False] + task_urls = [task.path for task in self._storage_impl.list_content( + step_urls) if task.is_file is False] + urls = [] + for task_url in task_urls: + for attempt in range(metaflow_config.MAX_ATTEMPTS): + for suffix in [TaskDataStore.METADATA_DATA_SUFFIX, + TaskDataStore.METADATA_ATTEMPT_SUFFIX, + TaskDataStore.METADATA_DONE_SUFFIX]: + urls.append(self._storage_impl.path_join( + task_url, + TaskDataStore.metadata_name_for_attempt(suffix, attempt) + )) + + latest_started_attempts = {} + done_attempts = set() + data_objs = {} + with self._storage_impl.load_bytes(urls) as get_results: + for key, path, meta in get_results: + if path is not None: + _, run, step, task, fname = self._storage_impl.path_split(key) + attempt, fname = TaskDataStore.parse_attempt_metadata(fname) + attempt = int(attempt) + if fname == TaskDataStore.METADATA_DONE_SUFFIX: + done_attempts.add((run, step, task, attempt)) + elif fname == TaskDataStore.METADATA_ATTEMPT_SUFFIX: + latest_started_attempts[(run, step, task)] = \ + max(latest_started_attempts.get((run, step, task), 0), + attempt) + elif fname == TaskDataStore.METADATA_DATA_SUFFIX: + # This somewhat breaks the abstraction since we are using + # load_bytes directly instead of load_metadata + with open(path, 'rb') as f: + data_objs[(run, step, task, attempt)] = json.load(f) + # We now figure out the latest attempt that started *and* finished. + # Note that if an attempt started but didn't finish, we do *NOT* return + # the previous attempt + latest_started_attempts = set( + (run, step, task, attempt) + for (run, step, task), attempt in latest_started_attempts.items()) + if allow_not_done: + latest_to_fetch = latest_started_attempts + else: + latest_to_fetch = latest_started_attempts & done_attempts + latest_to_fetch = [(v[0], v[1], v[2], v[3], data_objs[v], 'r', allow_not_done) + for v in latest_to_fetch] + return list(itertools.starmap(self.get_task_datastore, latest_to_fetch)) + + def get_task_datastore( + self, run_id, step_name, task_id, attempt=None, + data_metadata=None, mode='r', allow_not_done=False): + + return TaskDataStore( + self, + run_id, + step_name, + task_id, + attempt=attempt, + data_metadata=data_metadata, + mode=mode, + allow_not_done=allow_not_done) + + def save_data(self, data_iter, len_hint=0): + """Saves data to the underlying content-addressed store + + Parameters + ---------- + data : Iterator[bytes] + Iterator over blobs to save; each item in the list will be saved individually. + len_hint : int + Estimate of the number of items that will be produced by the iterator, + by default 0. + + Returns + ------- + (str, str) + Tuple containing the URI to access the saved resource as well as + the key needed to retrieve it using load_data. This is returned in + the same order as the input. + """ + save_results = self.ca_store.save_blobs(data_iter, raw=True, len_hint=len_hint) + return [(r.uri, r.key) for r in save_results] + + def load_data(self, keys, force_raw=False): + """Retrieves data from the underlying content-addressed store + + Parameters + ---------- + keys : List[str] + Keys to retrieve + force_raw : bool, optional + Backward compatible mode. Raw data will be properly identified with + metadata information but older datastores did not do this. If you + know the data should be handled as raw data, set this to True, + by default False + + Returns + ------- + Iterator[bytes] + Iterator over (key, blob) tuples + """ + for key, blob in self.ca_store.load_blobs(keys, force_raw=force_raw): + yield key, blob diff --git a/metaflow/datastore/inputs.py b/metaflow/datastore/inputs.py new file mode 100644 index 00000000000..cea6ea020b8 --- /dev/null +++ b/metaflow/datastore/inputs.py @@ -0,0 +1,17 @@ +class Inputs(object): + """ + split-and: inputs.step_a.x inputs.step_b.x + foreach: inputs[0].x + both: (inp.x for inp in inputs) + """ + def __init__(self, flows): + # TODO sort by foreach index + self.flows = list(flows) + for flow in self.flows: + setattr(self, flow._current_step, flow) + + def __getitem__(self, idx): + return self.flows[idx] + + def __iter__(self): + return iter(self.flows) diff --git a/metaflow/datastore/local.py b/metaflow/datastore/local.py deleted file mode 100644 index cf1b5ddd0cb..00000000000 --- a/metaflow/datastore/local.py +++ /dev/null @@ -1,262 +0,0 @@ -""" -Local storage - -Store data under .metaflow/ in the cwd -""" -import os -import json -import gzip -from tempfile import NamedTemporaryFile - -from metaflow.util import Path -from metaflow.metaflow_config import DATASTORE_LOCAL_DIR, DATASTORE_SYSROOT_LOCAL -from .datastore import MetaflowDataStore, DataException, only_if_not_done -from ..metadata import MetaDatum - - -class LocalDataStore(MetaflowDataStore): - TYPE = 'local' - - METADATA_DIR = '_meta' - - def _makedirs(self, path): - try: - os.makedirs(path) - except OSError as x: - if x.errno == 17: - return - else: - raise - - def object_path(self, sha): - root = os.path.join(self.data_root, sha[:2]) - return os.path.join(root, sha) - - @classmethod - def get_datastore_root_from_config(cls, echo, create_on_absent=True): - # Compute path for DATASTORE_SYSROOT_LOCAL - result = DATASTORE_SYSROOT_LOCAL - if result is None: - try: - # Python2 - current_path = os.getcwdu() - except: # noqa E722 - current_path = os.getcwd() - check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR) - check_dir = os.path.realpath(check_dir) - orig_path = check_dir - top_level_reached = False - while not os.path.isdir(check_dir): - new_path = os.path.dirname(current_path) - if new_path == current_path: - top_level_reached = True - break # We are no longer making upward progress - current_path = new_path - check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR) - if top_level_reached: - if create_on_absent: - # Could not find any directory to use so create a new one - echo('Creating local datastore in current directory (%s)' % orig_path, - fg='magenta', bold=True) - os.mkdir(orig_path) - result = orig_path - else: - return None - else: - result = check_dir - else: - result = os.path.join(result, DATASTORE_LOCAL_DIR) - return result - - @classmethod - def get_latest_tasks(cls, - flow_name, - run_id=None, - steps=None, - pathspecs=None): - run_prefix = cls.make_path(flow_name, run_id) - data_blobs = [] - - if os.path.exists(run_prefix): - if steps is None: - steps = [s for s in os.listdir(run_prefix) if s != cls.METADATA_DIR] - if pathspecs is None: - task_prefixes = [] - for step in steps: - step_prefix = cls.make_path(flow_name, run_id, step) - for task in os.listdir(step_prefix): - if task == cls.METADATA_DIR: - continue - task_prefixes.append( - cls.make_path(flow_name, run_id, step, task)) - else: - task_prefixes = [cls.make_path(flow_name, pathspec) - for pathspec in pathspecs] - for task_prefix in task_prefixes: - step, task = task_prefix.split('/')[-2:] - # Sort the file listing to iterate in increasing order of - # attempts. - latest_data_path = None - latest_attempt = None - latest_done_attempt = None - for fname in sorted(os.listdir(task_prefix)): - if cls.is_done_filename(fname): - _, attempt = cls.parse_filename(fname) - latest_done_attempt = attempt - # Read the corresponding metadata file. - meta_fname = \ - cls.get_metadata_filename_for_attempt(attempt) - latest_data_path = os.path.join(task_prefix, meta_fname) - elif cls.is_attempt_filename(fname): - _, attempt = cls.parse_filename(fname) - latest_attempt = attempt - # Only read the metadata if the latest attempt is also done. - if latest_done_attempt is not None and\ - latest_done_attempt == latest_attempt: - with open(latest_data_path) as f: - data_blobs.append((step, task, attempt, f.read())) - return data_blobs - else: - raise DataException("Couldn't find data at %s" % run_prefix) - - @classmethod - def get_artifacts(cls, artifacts_to_prefetch): - artifact_list = [] - for path in artifacts_to_prefetch: - sha = path.split('/')[-1] - artifact_list.append((sha, - cls.decode_gzip_data(path))) - return artifact_list - - @only_if_not_done - def save_logs(self, logsource, stream_data): - """ - Save log files for multiple streams, represented as - as a list of (stream, bytes) or (stream, Path) tuples. - """ - for stream, data in stream_data: - if isinstance(data, Path): - with open(str(data), 'rb') as f: - data = f.read() - path = self.get_log_location(logsource, stream) - with open(path + '.tmp', 'wb') as f: - f.write(data) - os.rename(path + '.tmp', path) - - def _read_file_or_empty(self, path): - if os.path.exists(path): - with open(path, 'rb') as f: - return f.read() - else: - return b'' - - def load_log_legacy(self, stream, attempt_override=None): - """ - Load old-style, pre-mflog, log file represented as a bytes object. - """ - f = self.filename_with_attempt_prefix('%s.log' % stream, - attempt_override if attempt_override is not None - else self.attempt) - return self._read_file_or_empty(os.path.join(self.root, f)) - - def load_logs(self, logsources, stream, attempt_override=None): - paths = [self.get_log_location(source, stream, attempt_override) - for source in logsources] - return list(zip(logsources, map(self._read_file_or_empty, paths))) - - @only_if_not_done - def save_metadata(self, name, metadata): - """ - Save a task-specific metadata dictionary as JSON. - """ - self._makedirs(self.root) - filename = self.filename_with_attempt_prefix('%s.json' % name, - self.attempt) - path = os.path.join(self.root, filename) - with open(path + '.tmp', 'w') as f: - json.dump(metadata, f) - os.rename(path + '.tmp', path) - - def load_metadata(self, name): - """ - Load a task-specific metadata dictionary as JSON. - """ - filename = self.filename_with_attempt_prefix('%s.json' % name, - self.attempt) - path = os.path.join(self.root, filename) - with open(path) as f: - return json.load(f) - - def has_metadata(self, name, with_attempt=True): - attempt = self.attempt if with_attempt else None - filename = self.filename_with_attempt_prefix('%s.json' % name, attempt) - path = os.path.join(self.root, filename) - return os.path.exists(path) - - @only_if_not_done - def save_data(self, sha, transformable_object): - """ - Save a content-addressed data blob if it doesn't exist already. - """ - path = self.object_path(sha) - if not os.path.exists(path): - self._makedirs(os.path.dirname(path)) - # NOTE multiple tasks may try to save an object with the - # same sha concurrently, hence we need to use a proper tmp - # file - with NamedTemporaryFile(dir=os.path.dirname(path), - prefix='blobtmp.', - delete=False) as tmp: - # NOTE compresslevel makes a huge difference. The default - # level of 9 can be impossibly slow. - with gzip.GzipFile(fileobj=tmp, - mode='wb', - compresslevel=3) as f: - f.write(transformable_object.current()) - os.rename(tmp.name, path) - return path - - def load_data(self, sha): - """ - Load a content-addressed data blob. - """ - with gzip.open(self.object_path(sha), 'rb') as f: - return f.read() - - @only_if_not_done - def done(self): - """ - Write a marker indicating that datastore has finished writing to - this path. - """ - filename = self.get_done_filename_for_attempt(self.attempt) - path = os.path.join(self.root, filename) - self._makedirs(self.root) - try: - # this is for python2 compatibility. - # Python3 has open(mode='x'). - fd = os.fdopen(os.open(path, - os.O_EXCL | os.O_WRONLY | os.O_CREAT), - 'wb') - fd.close() - except OSError as x: - if x.errno == 17: - raise DataException('Path %s already exists. Try with a ' - 'different --run-id.' % path) - else: - raise - self.metadata.register_metadata( - self.run_id, self.step_name, self.task_id, - [MetaDatum(field='attempt-done', value=str(self.attempt), type='attempt-done', tags=[ - "attempt_id:{0}".format(self.attempt)])]) - - self._is_done_set = True - - def is_done(self): - """ - A flag indicating whether this datastore directory was closed - successfully with done(). - """ - filename = self.get_done_filename_for_attempt(self.attempt) - path = os.path.join(self.root, filename) - return os.path.exists(path) diff --git a/metaflow/datastore/local_storage.py b/metaflow/datastore/local_storage.py new file mode 100644 index 00000000000..6b8a21ed196 --- /dev/null +++ b/metaflow/datastore/local_storage.py @@ -0,0 +1,116 @@ +import json +import os + +from ..metaflow_config import DATASTORE_LOCAL_DIR, DATASTORE_SYSROOT_LOCAL +from .datastore_storage import CloseAfterUse, DataStoreStorage +from .exceptions import DataException + +class LocalStorage(DataStoreStorage): + TYPE = 'local' + METADATA_DIR = '_meta' + + @classmethod + def get_datastore_root_from_config(cls, echo, create_on_absent=True): + result = DATASTORE_SYSROOT_LOCAL + if result is None: + try: + # Python2 + current_path = os.getcwdu() + except: # noqa E722 + current_path = os.getcwd() + check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR) + check_dir = os.path.realpath(check_dir) + orig_path = check_dir + top_level_reached = False + while not os.path.isdir(check_dir): + new_path = os.path.dirname(current_path) + if new_path == current_path: + top_level_reached = True + break # We are no longer making upward progress + current_path = new_path + check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR) + if top_level_reached: + if create_on_absent: + # Could not find any directory to use so create a new one + echo('Creating local datastore in current directory (%s)' + % orig_path) + os.mkdir(orig_path) + result = orig_path + else: + return None + else: + result = check_dir + else: + result = os.path.join(result, DATASTORE_LOCAL_DIR) + return result + + @staticmethod + def _makedirs(path): + try: + os.makedirs(path) + except OSError as x: + if x.errno == 17: + return + else: + raise + + def is_file(self, paths): + results = [] + for path in paths: + full_path = self.full_uri(path) + results.append(os.path.isfile(full_path)) + return results + + def info_file(self, path): + file_exists = self.is_file([path])[0] + if file_exists: + full_meta_path = "%s_meta" % self.full_uri(path) + try: + with open(full_meta_path, 'r') as f: + return True, json.load(f) + except OSError: + return True, None + return False, None + + def list_content(self, paths): + results = [] + for path in paths: + if path == self.METADATA_DIR: + continue + full_path = self.full_uri(path) + results.extend([self.list_content_result( + path=self.path_join(path, f), + is_file=self.is_file( + [self.path_join(path, f)])[0]) for f in os.listdir(full_path) + if f != self.METADATA_DIR]) + return results + + def save_bytes(self, path_and_bytes_iter, overwrite=False, len_hint=0): + for path, obj in path_and_bytes_iter: + if isinstance(obj, tuple): + byte_obj, metadata = obj + else: + byte_obj, metadata = obj, None + full_path = self.full_uri(path) + if not overwrite and os.path.exists(full_path): + continue + LocalStorage._makedirs(os.path.dirname(full_path)) + with open(full_path, mode='wb') as f: + f.write(byte_obj.read()) + if metadata: + with open("%s_meta" % full_path, mode='w') as f: + json.dump(metadata, f) + + def load_bytes(self, paths): + def iter_results(): + for path in paths: + full_path = self.full_uri(path) + metadata = None + if os.path.exists(full_path): + if os.path.exists("%s_meta" % full_path): + with open("%s_meta" % full_path, mode='r') as f: + metadata = json.load(f) + yield path, full_path, metadata + else: + yield path, None, None + return CloseAfterUse(iter_results()) diff --git a/metaflow/datastore/s3.py b/metaflow/datastore/s3.py deleted file mode 100644 index 041387c7393..00000000000 --- a/metaflow/datastore/s3.py +++ /dev/null @@ -1,302 +0,0 @@ -""" -S3 storage - -Store data in S3 -""" -import os -import json -import gzip -from io import BytesIO - -try: - # python2 - from urlparse import urlparse -except: - # python3 - from urllib.parse import urlparse - -from .. import metaflow_config -from .datastore import MetaflowDataStore, only_if_not_done -from ..metadata import MetaDatum -from ..datatools.s3util import aws_retry, get_s3_client -from metaflow.util import Path - -# We need UncloseableBytesIO for put_s3_object which may need -# to consume a BytesIO buffer multiple times. Blocking close() -# is cheaper than creating a new BytesIO object every time -# which would create duplicate copies of data. -class UncloseableBytesIO(BytesIO): - def close(self): - pass - -class S3DataStore(MetaflowDataStore): - TYPE='s3' - - s3 = None - ClientError = None - - def __init__(self, *args, **kwargs): - from .. import S3 - self.S3 = S3 - self.reset_client() - super(S3DataStore, self).__init__(*args, **kwargs) - - @classmethod - def reset_client(cls, hard_reset=False): - # the s3 client is shared across all S3DataStores - # so we don't open N connections to S3 unnecessarily - if cls.s3 is None or hard_reset: - cls.s3, cls.ClientError = get_s3_client() - - @aws_retry - def _get_s3_object(self, path, return_buf=False): - url = urlparse(path) - buf = BytesIO() - if self.monitor: - with self.monitor.measure("metaflow.s3.get_object"): - self.s3.download_fileobj(url.netloc, url.path.lstrip('/'), buf) - else: - self.s3.download_fileobj(url.netloc, url.path.lstrip('/'), buf) - if return_buf: - buf.seek(0) - return buf - else: - return buf.getvalue() - - @aws_retry - def _put_s3_object(self, path, blob=None, buf=None): - url = urlparse(path) - # @aws_retry may cause this function to be called multiple times with the same arguments. - # Make sure that the buffer state is reset for every iteration - if buf is None: - buf = BytesIO(blob) - else: - buf.seek(0) - if self.monitor: - with self.monitor.measure("metaflow.s3.put_object"): - self.s3.upload_fileobj(buf, url.netloc, url.path.lstrip('/')) - else: - self.s3.upload_fileobj(buf, url.netloc, url.path.lstrip('/')) - - @aws_retry - def _head_s3_object(self, path): - url = urlparse(path) - try: - return self.s3.head_object(Bucket=url.netloc, Key=url.path.lstrip('/')) - except self.ClientError as err: - error_code = int(err.response['Error']['Code']) - if error_code == 404: - return None - else: - raise - - @classmethod - def get_latest_tasks(cls, - flow_name, - run_id=None, - steps=None, - pathspecs=None): - run_prefix = cls.make_path(flow_name, run_id) - - from metaflow import S3 - with S3() as s3: - task_urls = [] - # Note: When `pathspecs` is specified, we avoid the eventually - # consistent `s3.list_paths` operation, and directly construct the - # task_urls list. - if pathspecs: - task_urls = [cls.make_path(flow_name, pathspec=pathspec) - for pathspec in pathspecs] - elif steps: - task_objs = s3.list_paths( - [cls.make_path(flow_name, run_id, step) for step in steps]) - task_urls = [task.url for task in task_objs] - else: - step_objs = s3.list_paths([run_prefix]) - task_objs = s3.list_paths([step.url for step in step_objs]) - task_urls = [task.url for task in task_objs] - urls = [] - for task_url in task_urls: - for attempt in range(metaflow_config.MAX_ATTEMPTS): - metadata_filename = \ - cls.get_metadata_filename_for_attempt(attempt) - urls.append(os.path.join(task_url, metadata_filename)) - # Note for potential future optimization: - # Find the list of latest attempt for each task first, and - # follow up with a call to get done and metadata. - attempt_filename = \ - cls.get_filename_for_attempt(attempt) - urls.append(os.path.join(task_url, attempt_filename)) - done_filename = cls.get_done_filename_for_attempt(attempt) - urls.append(os.path.join(task_url, done_filename)) - - results = s3.get_many(urls, return_missing=True) - - all_data_blobs = {} - latest_attempt = {} - done_attempts = set() - - for result in results: - if result.exists: - path = result.url - step_name, task_id, fname = path.split('/')[-3:] - _, attempt = cls.parse_filename(fname) - if cls.is_done_filename(fname): - done_attempts.add((step_name, task_id, attempt)) - elif cls.is_attempt_filename(fname): - # files are in sorted order, so overwrite is ok. - latest_attempt[(step_name, task_id)] = attempt - else: - # is_metadata_filename(fname) == True. - all_data_blobs[(step_name, task_id, attempt)] = \ - result.blob - latest_attempts = set((step_name, task_id, attempt) - for (step_name, task_id), attempt - in latest_attempt.items()) - latest_and_done = latest_attempts & done_attempts - - return [(step_name, task_id, attempt, - all_data_blobs[(step_name, task_id, attempt)]) - for step_name, task_id, attempt in latest_and_done] - - @classmethod - def get_artifacts(cls, artifacts_to_prefetch): - artifact_list = [] - from metaflow import S3 - with S3() as s3: - for obj in s3.get_many(artifacts_to_prefetch): - sha = obj.key.split('/')[-1] - artifact_list.append((sha, cls.decode_gzip_data(obj.path))) - return artifact_list - - @only_if_not_done - def save_metadata(self, name, data): - """ - Save a task-specific metadata dictionary as JSON. - """ - filename = self.filename_with_attempt_prefix('%s.json' % name, - self.attempt) - path = os.path.join(self.root, filename) - self._put_s3_object(path, json.dumps(data).encode('utf-8')) - - def has_metadata(self, name, with_attempt=True): - attempt = self.attempt if with_attempt else None - filename = self.filename_with_attempt_prefix('%s.json' % name, - attempt) - path = os.path.join(self.root, filename) - return bool(self._head_s3_object(path)) - - def load_metadata(self, name): - """ - Load a task-specific metadata dictionary as JSON. - """ - filename = self.filename_with_attempt_prefix('%s.json' % name, - self.attempt) - path = os.path.join(self.root, filename) - return json.loads(self._get_s3_object(path).decode('utf-8')) - - def object_path(self, sha): - root = os.path.join(self.data_root, sha[:2]) - return os.path.join(root, sha) - - @only_if_not_done - def save_data(self, sha, transformable_object): - """ - Save a content-addressed data blob if it doesn't exist already. - """ - path = self.object_path(sha) - if not self._head_s3_object(path): - # we need UncloseableBytesIO for put_s3_object which may need - # to consume the buffer multiple times - buf = UncloseableBytesIO() - # NOTE compresslevel makes a huge difference. The default - # level of 9 can be impossibly slow. - with gzip.GzipFile(fileobj=buf, - mode='wb', - compresslevel=3) as f: - f.write(transformable_object.current()) - transformable_object.transform(lambda _: buf) - buf.seek(0) - self._put_s3_object(path, buf=buf) - return path - - def load_data(self, sha): - """ - Load a content-addressed data blob. - """ - path = self.object_path(sha) - buf = self._get_s3_object(path, return_buf=True) - return self.decode_gzip_data(None, buf) # filename=None - - @only_if_not_done - def save_logs(self, logsource, stream_data): - """ - Save log files for multiple streams, represented as - as a list of (stream, bytes) or (stream, Path) tuples. - """ - locs = [self.get_log_location(logsource, stream) - for stream, _ in stream_data] - - if stream_data and isinstance(stream_data[0][1], Path): - keys = list(zip(locs, (str(path) for _, path in stream_data))) - with self.S3() as s3: - s3.put_files(keys) - else: - with self.S3() as s3: - for url, data in zip(locs, (data for _, data in stream_data)): - s3.put(url, data) - - def load_log_legacy(self, stream, attempt_override=None): - """ - Load old-style, pre-mflog, log file represented as a bytes object. - """ - f = self.filename_with_attempt_prefix('%s.log' % stream, - attempt_override if attempt_override is not None - else self.attempt) - path = os.path.join(self.root, f) - if self._head_s3_object(path): - return self._get_s3_object(path) - else: - return b'' - - def load_logs(self, logsources, stream, attempt_override=None): - urls = [self.get_log_location(source, stream, attempt_override) - for source in logsources] - with self.S3() as s3: - results = s3.get_many(urls, return_missing=True) - blobs = [log.blob if log.exists else b'' for log in results] - return list(zip(logsources, blobs)) - - - def load_log(self, logtype, attempt_override=None): - """ - Load a task-specific log file represented as a bytes object. - """ - path = self.get_log_location(logtype, attempt_override) - return self._get_s3_object(path) - - @only_if_not_done - def done(self): - """ - Write a marker indicating that datastore has finished writing to - this path. - """ - filename = self.get_done_filename_for_attempt(self.attempt) - path = os.path.join(self.root, filename) - self._put_s3_object(path, b'') - - self.metadata.register_metadata( - self.run_id, self.step_name, self.task_id, - [MetaDatum(field='attempt-done', value=str(self.attempt), type='attempt-done', - tags=["attempt_id:{0}".format(self.attempt)])]) - - self._is_done_set = True - - def is_done(self): - """ - A flag indicating whether this datastore directory was closed - successfully with done(). - """ - filename = self.get_done_filename_for_attempt(self.attempt) - path = os.path.join(self.root, filename) - return bool(self._head_s3_object(path)) diff --git a/metaflow/datastore/s3_storage.py b/metaflow/datastore/s3_storage.py new file mode 100644 index 00000000000..a4ee7fe1bf4 --- /dev/null +++ b/metaflow/datastore/s3_storage.py @@ -0,0 +1,117 @@ + +import os + +from itertools import starmap + +from ..datatools.s3 import S3, S3Client, S3PutObject +from ..metaflow_config import DATASTORE_SYSROOT_S3 +from .datastore_storage import CloseAfterUse, DataStoreStorage + + +try: + # python2 + from urlparse import urlparse +except: + # python3 + from urllib.parse import urlparse + + +class S3Storage(DataStoreStorage): + TYPE = 's3' + + def __init__(self, root=None): + super(S3Storage, self).__init__(root) + self.s3_client = S3Client() + + @classmethod + def get_datastore_root_from_config(cls, echo, create_on_absent=True): + return DATASTORE_SYSROOT_S3 + + def is_file(self, paths): + with S3(s3root=self.datastore_root, + tmproot=os.getcwd(), external_client=self.s3_client) as s3: + if len(paths) > 10: + s3objs = s3.info_many(paths, return_missing=True) + return [s3obj.exists for s3obj in s3objs] + else: + result = [] + for path in paths: + result.append(s3.info(path, return_missing=True).exists) + return result + + def info_file(self, path): + with S3(s3root=self.datastore_root, + tmproot=os.getcwd(), external_client=self.s3_client) as s3: + s3obj = s3.info(path, return_missing=True) + return s3obj.exists, s3obj.metadata + + def list_content(self, paths): + strip_prefix_len = len(self.datastore_root.rstrip('/')) + 1 + with S3(s3root=self.datastore_root, + tmproot=os.getcwd(), external_client=self.s3_client) as s3: + results = s3.list_paths(paths) + return [self.list_content_result( + path=o.url[strip_prefix_len:], is_file=o.exists) for o in results] + + def save_bytes(self, path_and_bytes_iter, overwrite=False, len_hint=0): + def _convert(): + # Output format is the same as what is needed for S3PutObject: + # key, value, path, content_type, metadata + for path, obj in path_and_bytes_iter: + if isinstance(obj, tuple): + yield path, obj[0], None, None, obj[1] + else: + yield path, obj, None, None, None + + with S3(s3root=self.datastore_root, + tmproot=os.getcwd(), external_client=self.s3_client) as s3: + # HACK: The S3 datatools we rely on does not currently do a good job + # determining if uploading things in parallel is more efficient than + # serially. We use a heuristic for now where if we have a lot of + # files, we will go in parallel and if we have few files, we will + # serially upload them. This is not ideal because there is also a size + # factor and one very large file with a few other small files, for + # example, would benefit from a parallel upload. + # + # In the case of save_artifacts, currently len_hint is based on the + # total number of artifacts, not taking into account how many of them + # already exist in the CAS, i.e. it can be a gross overestimate. As a + # result, it is possible we take a latency hit by using put_many only + # for a few artifacts. + # + # A better approach would be to e.g. write all blobs to temp files + # and based on the total size and number of files use either put_files + # (which avoids re-writing the files) or s3.put sequentially. + if len_hint > 10: + # Use put_many + s3.put_many(starmap(S3PutObject, _convert()), overwrite) + else: + # Sequential upload + for key, obj, _, _, metadata in _convert(): + s3.put(key, obj, overwrite=overwrite, metadata=metadata) + + def load_bytes(self, paths): + if len(paths) == 0: + return CloseAfterUse(iter([])) + + s3 = S3(s3root=self.datastore_root, + tmproot=os.getcwd(), external_client=self.s3_client) + + def iter_results(): + # We similarly do things in parallel for many files. This is again + # a hack. + if len(paths) > 10: + results = s3.get_many(paths, return_missing=True, return_info=True) + for r in results: + if r.exists: + yield r.key, r.path, r.metadata + else: + yield r.key, None, None + else: + for p in paths: + r = s3.get(p, return_missing=True, return_info=True) + if r.exists: + yield r.key, r.path, r.metadata + else: + yield r.key, None, None + return CloseAfterUse(iter_results(), closer=s3) diff --git a/metaflow/datastore/task_datastore.py b/metaflow/datastore/task_datastore.py new file mode 100644 index 00000000000..880da8048c4 --- /dev/null +++ b/metaflow/datastore/task_datastore.py @@ -0,0 +1,741 @@ +import json +import pickle +import sys +import time + +from functools import wraps +from io import BufferedIOBase, FileIO, RawIOBase +from types import MethodType, FunctionType + +from .. import metaflow_config +from ..exception import MetaflowInternalError +from ..metadata import DataArtifact, MetaDatum +from ..parameters import Parameter +from ..util import Path, is_stringish, to_fileobj + +from .exceptions import DataException + +def only_if_not_done(f): + @wraps(f) + def method(self, *args, **kwargs): + if self._is_done_set: + raise MetaflowInternalError("Tried to write to datastore "\ + "(method %s) after it was marked "\ + ".done()" % f.__name__) + return f(self, *args, **kwargs) + return method + +def require_mode(mode): + def wrapper(f): + @wraps(f) + def method(self, *args, **kwargs): + if mode is not None and self._mode != mode: + raise MetaflowInternalError( + "Attempting a datastore operation '%s' requiring mode '%s' " + "but have mode '%s'" % (f.__name__, mode, self._mode)) + return f(self, *args, **kwargs) + return method + return wrapper + +class ArtifactTooLarge(object): + def __str__(self): + return '< artifact too large >' + +class TaskDataStore(object): + """ + TaskDataStore is obtained through FlowDataStore.get_datastore_for_task and + is used to store three things: + - Task artifacts (using save_artifacts and load_artifacts) which will + ultimately be stored using ContentAddressedStore's save_blobs and + load_blobs. This is basically the content indexed portion of the + storage (identical objects are stored only once). + - Metadata information (using save_metadata and load_metadata) which + stores JSON encoded metadata about a task in a non-content indexed + way in a hierarchical manner (ie: the files are stored + in a path indicated by the pathspec (run_id/step_name/task_id)). + This portion of the store can be viewed as name indexed (storing + two metadata items with the same name will overwrite the previous item + so the condition of equality is the name as + opposed to the content). + - Logs which are a special sort of task metadata but are handled + differently (they are not JSON-encodable dictionaries). + """ + + METADATA_ATTEMPT_SUFFIX = 'attempt.json' + METADATA_DONE_SUFFIX = 'DONE.lock' + METADATA_DATA_SUFFIX = 'data.json' + + @staticmethod + def metadata_name_for_attempt(name, attempt): + if attempt is None: + return name + return '%d.%s' % (attempt, name) + + @staticmethod + def parse_attempt_metadata(name): + return name.split('.', 1) + + def __init__(self, + flow_datastore, + run_id, + step_name, + task_id, + attempt=None, + data_metadata=None, + mode='r', + allow_not_done=False): + + self._storage_impl = flow_datastore._storage_impl + self.TYPE = self._storage_impl.TYPE + self._ca_store = flow_datastore.ca_store + self._environment = flow_datastore.environment + self._run_id = run_id + self._step_name = step_name + self._task_id = task_id + self._path = self._storage_impl.path_join( + flow_datastore.flow_name, run_id, step_name, task_id) + self._mode = mode + self._attempt = attempt + self._metadata = flow_datastore.metadata + self._parent = flow_datastore + + # The GZIP encodings are for backward compatibility + self._encodings = {'pickle-v2', 'gzip+pickle-v2'} + ver = sys.version_info[0] * 10 + sys.version_info[1] + if ver >= 34: + self._encodings.add('pickle-v4') + self._encodings.add('gzip+pickle-v4') + + self._is_done_set = False + + # If the mode is 'write', we initialize things to empty + if self._mode == 'w': + self._objects = {} + self._info = {} + elif self._mode == 'r': + if data_metadata is not None: + # We already loaded the data metadata so just use that + self._objects = data_metadata.get('objects', {}) + self._info = data_metadata.get('info', {}) + else: + # What is the latest attempt ID for this task store. + # NOTE: We *only* access to the data if the attempt that + # produced it is done. In particular, we do not allow access to + # a past attempt if a new attempt has started to avoid + # inconsistencies (depending on when the user accesses the + # datastore, the data may change). We make an exception to that + # rule when allow_not_done is True which allows access to things + # like logs even for tasks that did not write a done marker + self._attempt = None + for i in range(metaflow_config.MAX_ATTEMPTS): + check_meta = self._metadata_name_for_attempt( + self.METADATA_ATTEMPT_SUFFIX, i) + if self.has_metadata(check_meta, add_attempt=False): + self._attempt = i + # Check if the latest attempt was completed successfully except + # if we have allow_not_done + data_obj = None + if self.has_metadata(self.METADATA_DONE_SUFFIX): + data_obj = self.load_metadata([self.METADATA_DATA_SUFFIX]) + data_obj = data_obj[self.METADATA_DATA_SUFFIX] + elif self._attempt is None or not allow_not_done: + raise DataException( + "Data was not found or not finished at %s" % self._path) + + if data_obj is not None: + self._objects = data_obj.get('objects', {}) + self._info = data_obj.get('info', {}) + else: + raise DataException("Unknown datastore mode: '%s'" % self._mode) + + @property + def pathspec(self): + return '/'.join([self.run_id, self.step_name, self.task_id]) + + @property + def run_id(self): + return self._run_id + + @property + def step_name(self): + return self._step_name + + @property + def task_id(self): + return self._task_id + + @property + def pathspec_index(self): + idxstr = ','.join(map(str, (f.index for f in self['_foreach_stack']))) + return '%s/%s[%s]' % (self._run_id, self._step_name, idxstr) + + @property + def parent_datastore(self): + return self._parent + + @require_mode(None) + def get_log_location(self, logprefix, stream): + log_name = self._get_log_location(logprefix, stream) + path = self._storage_impl.path_join( + self._path, self._metadata_name_for_attempt(log_name)) + return self._storage_impl.full_uri(path) + + @require_mode('r') + def keys_for_artifacts(self, names): + return [self._objects.get(name) for name in names] + + @only_if_not_done + @require_mode('w') + def init_task(self): + """ + Call this to initialize the datastore with a new attempt. + + This method requires mode 'w'. + """ + self.save_metadata( + {self.METADATA_ATTEMPT_SUFFIX: {'time': time.time()}}) + + @only_if_not_done + @require_mode('w') + def save_artifacts(self, artifacts_iter, force_v4=False, len_hint=0): + """ + Saves Metaflow Artifacts (Python objects) to the datastore and stores + any relevant metadata needed to retrieve them. + + Typically, objects are pickled but the datastore may perform any + operation that it deems necessary. You should only access artifacts + using load_artifacts + + This method requires mode 'w'. + + Parameters + ---------- + artifacts : Iterator[(string, object)] + Iterator over the human-readable name of the object to save + and the object itself + force_v4 : boolean or Dict[string -> boolean] + Indicates whether the artifact should be pickled using the v4 + version of pickle. If a single boolean, applies to all artifacts. + If a dictionary, applies to the object named only. Defaults to False + if not present or not specified + len_hint: integer + Estimated number of items in artifacts_iter + """ + artifact_names = [] + + def pickle_iter(): + for name, obj in artifacts_iter: + do_v4 = force_v4 and \ + force_v4 if isinstance(force_v4, bool) else \ + force_v4.get(name, False) + if do_v4: + encode_type = 'gzip+pickle-v4' + if encode_type not in self._encodings: + raise DataException( + "Artifact *%s* requires a serialization encoding that " + "requires Python 3.4 or newer." % name) + blob = pickle.dumps(obj, protocol=4) + else: + try: + blob = pickle.dumps(obj, protocol=2) + encode_type = 'gzip+pickle-v2' + except (SystemError, OverflowError): + encode_type = 'gzip+pickle-v4' + if encode_type not in self._encodings: + raise DataException( + "Artifact *%s* is very large (over 2GB). " + "You need to use Python 3.4 or newer if you want to " + "serialize large objects." % name) + blob = pickle.dumps(obj, protocol=4) + self._info[name] = { + 'size': len(blob), + 'type': str(type(obj)), + 'encoding': encode_type + } + artifact_names.append(name) + yield blob + # Use the content-addressed store to store all artifacts + save_result = self._ca_store.save_blobs(pickle_iter(), + len_hint=len_hint) + for name, result in zip(artifact_names, save_result): + self._objects[name] = result.key + + @require_mode(None) + def load_artifacts(self, names): + """ + Mirror function to save_artifacts + + This function will retrieve the objects referenced by 'name'. Each + object will be fetched and returned if found. Note that this function + will return objects that may not be the same as the ones saved using + saved_objects (taking into account possible environment changes, for + example different conda environments) but it will return objects that + can be used as the objects passed in to save_objects. + + This method can be used in both 'r' and 'w' mode. For the latter use + case, this can happen when `passdown_partial` is called and an artifact + passed down that way is then loaded. + + Parameters + ---------- + names : List[string] + List of artifacts to retrieve + + Returns + ------- + Iterator[(string, object)] : + An iterator over objects retrieved. + """ + if not self._info: + raise DataException( + "No info object available to retrieve artifacts") + to_load = [] + sha_to_names = {} + for name in names: + info = self._info.get(name) + # We use gzip+pickle-v2 as this is the oldest/most compatible. + # This datastore will always include the proper encoding version so + # this is just to be able to read very old artifacts + if info: + encode_type = info.get('encoding', 'gzip+pickle-v2') + else: + encode_type = 'gzip+pickle-v2' + if encode_type not in self._encodings: + raise DataException( + "Python 3.4 or later is required to load %s" % name) + else: + sha = self._objects[name] + sha_to_names[sha] = name + to_load.append(sha) + # At this point, we load what we don't have from the CAS + # We assume that if we have one "old" style artifact, all of them are + # like that which is an easy assumption to make since artifacts are all + # stored by the same implementation of the datastore for a given task. + for sha, blob in self._ca_store.load_blobs(to_load): + yield sha_to_names[sha], pickle.loads(blob) + + @only_if_not_done + @require_mode('w') + def save_metadata(self, contents, allow_overwrite=True, add_attempt=True): + """ + Save task metadata. This is very similar to save_artifacts; this + function takes a dictionary with the key being the name of the metadata + to save and the value being the metadata. + The metadata, however, will not be stored in the CAS but rather directly + in the TaskDataStore. + + This method requires mode 'w' + + Parameters + ---------- + contents : Dict[string -> JSON-ifiable objects] + Dictionary of metadata to store + allow_overwrite : boolean, optional + If True, allows the overwriting of the metadata, defaults to True + add_attempt : boolean, optional + If True, adds the attempt identifier to the metadata. defaults to + True + """ + return self._save_file( + {k: json.dumps(v).encode('utf-8') for k, v in contents.items()}, + allow_overwrite, add_attempt) + + @require_mode('r') + def load_metadata(self, names, add_attempt=True): + """ + Loads metadata saved with `save_metadata` + + Parameters + ---------- + names : List[string] + The name of the metadata elements to load + add_attempt : bool, optional + Adds the attempt identifier to the metadata name if True, + by default True + + Returns + ------- + Dict: string -> JSON decoded object + Results indexed by the name of the metadata loaded + """ + return {k: json.loads(v) for k, v \ + in self._load_file(names, add_attempt).items()} + + @require_mode(None) + def has_metadata(self, name, add_attempt=True): + """ + Checks if this TaskDataStore has the metadata requested + + TODO: Should we make this take multiple names like the other calls? + + This method operates like load_metadata in both 'w' and 'r' modes. + + Parameters + ---------- + names : string + Metadata name to fetch + add_attempt : bool, optional + Adds the attempt identifier to the metadata name if True, + by default True + + Returns + ------- + boolean + True if the metadata exists or False otherwise + """ + if add_attempt: + path = self._storage_impl.path_join( + self._path, self._metadata_name_for_attempt(name)) + else: + path = self._storage_impl.path_join(self._path, name) + return self._storage_impl.is_file([path])[0] + + @require_mode(None) + def get(self, name, default=None): + """ + Convenience method around load_artifacts for a given name and with a + provided default. + + This method requires mode 'r'. + + Parameters + ---------- + name : str + Name of the object to get + default : object, optional + Returns this value if object not found, by default None + """ + if self._objects: + try: + return self[name] if name in self._objects else default + except DataException: + return default + return default + + @require_mode('r') + def is_none(self, name): + """ + Convenience method to test if an artifact is None + + This method requires mode 'r'. + + Parameters + ---------- + name : string + Name of the artifact + """ + if not self._info: + return True + info = self._info.get(name) + if info: + obj_type = info.get('type') + # Conservatively check if the actual object is None, + # in case the artifact is stored using a different python version. + # Note that if an object is None and stored in Py2 and accessed in + # Py3, this test will fail and we will fallback to the slow path. This + # is intended (being conservative) + if obj_type == str(type(None)): + return True + # Slow path since this has to get the object from the datastore + return self.get(name) is None + + @only_if_not_done + @require_mode('w') + def done(self): + """ + Mark this task-datastore as 'done' for the current attempt + + Will throw an exception if mode != 'w' + """ + self.save_metadata({ + self.METADATA_DATA_SUFFIX: + {'datastore': self.TYPE, + 'version': '1.0', + 'attempt': self._attempt, + 'python_version': sys.version, + 'objects': self._objects, + 'info': self._info}, + self.METADATA_DONE_SUFFIX: "" + }) + + if self._metadata: + self._metadata.register_metadata( + self._run_id, self._step_name, self._task_id, + [MetaDatum(field='attempt-done', value=str(self._attempt), + type='attempt-done', tags=[])]) + artifacts = [DataArtifact( + name=var, + ds_type=self.TYPE, + ds_root=self._storage_impl.datastore_root, + url=None, + sha=sha, + type=self._info[var]['encoding']) + for var, sha in self._objects.items()] + + self._metadata.register_data_artifacts( + self.run_id, self.step_name, self.task_id, self._attempt, + artifacts) + + self._is_done_set = True + + @only_if_not_done + @require_mode('w') + def clone(self, origin): + """ + Clone the information located in the TaskDataStore origin into this + datastore + + Parameters + ---------- + origin : TaskDataStore + TaskDataStore to clone + """ + self._objects = origin._objects + self._info = origin._info + + @only_if_not_done + @require_mode('w') + def passdown_partial(self, origin, variables): + # Pass-down from datastore origin all information related to vars to + # this datastore. In other words, this adds to the current datastore all + # the variables in vars (obviously, it does not download them or + # anything but records information about them). This is used to + # propagate parameters between datastores without actually loading the + # parameters as well as for merge_artifacts + for var in variables: + sha = origin._objects.get(var) + if sha: + self._objects[var] = sha + self._info[var] = origin._info[var] + + @only_if_not_done + @require_mode('w') + def persist(self, flow): + """ + Persist any new artifacts that were produced when running flow + + NOTE: This is a DESTRUCTIVE operation that deletes artifacts from + the given flow to conserve memory. Don't rely on artifact attributes + of the flow object after calling this function. + + Parameters + ---------- + flow : FlowSpec + Flow to persist + """ + + if flow._datastore: + self._objects.update(flow._datastore._objects) + self._info.update(flow._datastore._info) + + # we create a list of valid_artifacts in advance, outside of + # artifacts_iter so we can provide a len_hint below + valid_artifacts = [] + for var in dir(flow): + if var.startswith('__') or var in flow._EPHEMERAL: + continue + # Skip over properties of the class (Parameters) + if hasattr(flow.__class__, var) and \ + isinstance(getattr(flow.__class__, var), property): + continue + + val = getattr(flow, var) + if not (isinstance(val, MethodType) or + isinstance(val, FunctionType) or + isinstance(val, Parameter)): + valid_artifacts.append((var, val)) + + def artifacts_iter(): + # we consume the valid_artifacts list destructively to + # make sure we don't keep references to artifacts. We + # want to avoid keeping original artifacts and encoded + # artifacts in memory simultaneously + while valid_artifacts: + var, val = valid_artifacts.pop() + if not var.startswith('_') and var != 'name': + # NOTE: Destructive mutation of the flow object. We keep + # around artifacts called 'name' and anything starting with + # '_' as they are used by the Metaflow runtime. + delattr(flow, var) + yield var, val + + self.save_artifacts(artifacts_iter(), len_hint=len(valid_artifacts)) + + @only_if_not_done + @require_mode('w') + def save_logs(self, logsource, stream_data): + """ + Save log files for multiple streams, represented as + a dictionary of streams. Each stream is identified by a type (a string) + and is either a stringish or a BytesIO object or a Path object. + + Parameters + ---------- + logsource : string + Identifies the source of the stream (runtime, task, etc) + + stream_data : Dict[string -> bytes or Path] + Each entry should have a string as the key indicating the type + of the stream ('stderr', 'stdout') and as value should be bytes or + a Path from which to stream the log. + """ + to_store_dict = {} + for stream, data in stream_data.items(): + n = self._get_log_location(logsource, stream) + if isinstance(data, Path): + to_store_dict[n] = FileIO(str(data), mode='r') + else: + to_store_dict[n] = data + self._save_file(to_store_dict) + + @require_mode('r') + def load_log_legacy(self, stream, attempt_override=None): + """ + Load old-style, pre-mflog, log file represented as a bytes object. + """ + name = self._metadata_name_for_attempt( + '%s.log' % stream, attempt_override) + r = self._load_file([name], add_attempt=False)[name] + return r if r is not None else b'' + + @require_mode('r') + def load_logs(self, logsources, stream, attempt_override=None): + paths = dict(map( + lambda s: (self._metadata_name_for_attempt( + self._get_log_location(s, stream), + attempt_override=attempt_override), s), logsources)) + r = self._load_file(paths.keys(), add_attempt=False) + return [(paths[k], v if v is not None else b'') for k, v in r.items()] + + @require_mode(None) + def items(self): + if self._objects: + return self._objects.items() + return {} + + @require_mode(None) + def to_dict(self, show_private=False, max_value_size=None, include=None): + d = {} + for k, _ in self.items(): + if include and k not in include: + continue + if k[0] == '_' and not show_private: + continue + if max_value_size is not None and\ + self._info[k]['size'] > max_value_size: + d[k] = ArtifactTooLarge() + else: + d[k] = self[k] + return d + + @require_mode('r') + def format(self, **kwargs): + def lines(): + for k, v in self.to_dict(**kwargs).items(): + yield k, '*{key}* [size: {size} type: {type}] = {value}'\ + .format(key=k, value=v, **self._info[k]) + return '\n'.join(line for k, line in sorted(lines())) + + @require_mode(None) + def __contains__(self, name): + if self._objects: + return name in self._objects + return False + + @require_mode(None) + def __getitem__(self, name): + _, obj = next(self.load_artifacts([name])) + return obj + + @require_mode('r') + def __iter__(self): + if self._objects: + return iter(self._objects) + return iter([]) + + @require_mode('r') + def __str__(self): + return self.format(show_private=True, max_value_size=1000) + + def _metadata_name_for_attempt(self, name, attempt_override=None): + return self.metadata_name_for_attempt( + name, self._attempt if attempt_override is None else + attempt_override) + + @staticmethod + def _get_log_location(logprefix, stream): + return '%s_%s.log' % (logprefix, stream) + + def _save_file(self, contents, allow_overwrite=True, add_attempt=True): + """ + Saves files in the directory for this TaskDataStore. This can be + metadata, a log file or any other data that doesn't need to (or + shouldn't) be stored in the Content Addressed Store. + + Parameters + ---------- + contents : Dict[string -> stringish or RawIOBase or BufferedIOBase] + Dictionary of file to store + allow_overwrite : boolean, optional + If True, allows the overwriting of the metadata, defaults to True + add_attempt : boolean, optional + If True, adds the attempt identifier to the metadata, + defaults to True + """ + def blob_iter(): + for name, value in contents.items(): + if add_attempt: + path = self._storage_impl.path_join( + self._path, self._metadata_name_for_attempt(name)) + else: + path = self._storage_impl.path_join(self._path, name) + if isinstance(value, (RawIOBase, BufferedIOBase)) and \ + value.readable(): + yield path, value + elif is_stringish(value): + yield path, to_fileobj(value) + else: + raise DataException("Metadata '%s' has an invalid type: %s" % + (name, type(value))) + self._storage_impl.save_bytes(blob_iter(), overwrite=allow_overwrite) + + def _load_file(self, names, add_attempt=True): + """ + Loads files from the TaskDataStore directory. These can be metadata, + logs or any other files + + Parameters + ---------- + names : List[string] + The names of the files to load + add_attempt : bool, optional + Adds the attempt identifier to the metadata name if True, + by default True + + Returns + ------- + Dict: string -> bytes + Results indexed by the name of the metadata loaded + """ + to_load = [] + for name in names: + if add_attempt: + path = self._storage_impl.path_join( + self._path, self._metadata_name_for_attempt(name)) + else: + path = self._storage_impl.path_join(self._path, name) + to_load.append(path) + results = {} + with self._storage_impl.load_bytes(to_load) as load_results: + for key, path, meta in load_results: + if add_attempt: + _, name = self.parse_attempt_metadata( + self._storage_impl.basename(key)) + else: + name = self._storage_impl.basename(key) + if path is None: + results[name] = None + else: + with open(path, 'rb') as f: + results[name] = f.read() + return results diff --git a/metaflow/datastore/util/__init__.py b/metaflow/datastore/util/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/metaflow/datatools/__init__.py b/metaflow/datatools/__init__.py index 5fb290219fe..d65626be034 100644 --- a/metaflow/datatools/__init__.py +++ b/metaflow/datatools/__init__.py @@ -1,13 +1,31 @@ import sys import types +ver = sys.version_info[0] * 10 + sys.version_info[1] + +# Read an AWS source in a chunked manner. +# We read in chunks (at most 2GB -- here this is passed via max_chunk_size) +# because of https://bugs.python.org/issue42853 (Py3 bug); this also helps +# keep memory consumption lower +# NOTE: For some weird reason, if you pass a large value to +# read, it delays the call so we always pass it either what +# remains or 2GB, whichever is smallest. +def read_in_chunks(dst, src, src_sz, max_chunk_size): + remaining = src_sz + while remaining > 0: + buf = src.read(min(remaining, max_chunk_size)) + # Py2 doesn't return the number of bytes written so calculate size + # separately + dst.write(buf) + remaining -= len(buf) + + from .s3 import MetaflowS3Exception, S3 # Import any additional datatools defined by a Metaflow extensions package try: import metaflow_extensions.datatools as extension_module except ImportError as e: - ver = sys.version_info[0] * 10 + sys.version_info[1] if ver >= 36: # e.name is set to the name of the package that fails to load # so don't error ONLY IF the error is importing this module (but do diff --git a/metaflow/decorators.py b/metaflow/decorators.py index 398f44b18eb..19ccaf2e97c 100644 --- a/metaflow/decorators.py +++ b/metaflow/decorators.py @@ -139,7 +139,7 @@ def flow_init(self, flow, graph, environment, - datastore, + flow_datastore, metadata, logger, echo, @@ -211,7 +211,7 @@ class MyDecorator(StepDecorator): pass them around with every lifecycle call. """ - def step_init(self, flow, graph, step_name, decorators, environment, datastore, logger): + def step_init(self, flow, graph, step_name, decorators, environment, flow_datastore, logger): """ Called when all decorators have been created for this step """ @@ -244,7 +244,7 @@ def runtime_init(self, flow, graph, package, run_id): pass def runtime_task_created(self, - datastore, + task_datastore, task_id, split_index, input_paths, @@ -273,7 +273,7 @@ def runtime_step_cli(self, def task_pre_step(self, step_name, - datastore, + task_datastore, metadata, run_id, task_id, @@ -442,7 +442,7 @@ def _attach_decorators_to_step(step, decospecs): def _init_flow_decorators(flow, graph, environment, - datastore, + flow_datastore, metadata, logger, echo, @@ -450,14 +450,14 @@ def _init_flow_decorators(flow, for deco in flow._flow_decorators.values(): opts = {option: deco_options[option] for option in deco.options} deco.flow_init(flow, graph, environment, - datastore, metadata, logger, echo, opts) + flow_datastore, metadata, logger, echo, opts) -def _init_step_decorators(flow, graph, environment, datastore, logger): +def _init_step_decorators(flow, graph, environment, flow_datastore, logger): for step in flow: for deco in step.decorators: deco.step_init(flow, graph, step.__name__, - step.decorators, environment, datastore, logger) + step.decorators, environment, flow_datastore, logger) def step(f): diff --git a/metaflow/includefile.py b/metaflow/includefile.py index 24f80e48753..524b2ebc78a 100644 --- a/metaflow/includefile.py +++ b/metaflow/includefile.py @@ -13,7 +13,6 @@ from . import parameters from .current import current -from .datastore.datastore import TransformableObject from .exception import MetaflowException from .metaflow_config import DATATOOLS_LOCALROOT, DATATOOLS_SUFFIX from .parameters import DeployTimeField, Parameter @@ -104,8 +103,8 @@ def _makedirs(path): def get_root_from_config(cls, echo, create_on_absent=True): result = DATATOOLS_LOCALROOT if result is None: - from .datastore.local import LocalDataStore - result = LocalDataStore.get_datastore_root_from_config(echo, create_on_absent) + from .datastore.local_storage import LocalStorage + result = LocalStorage.get_datastore_root_from_config(echo, create_on_absent) result = os.path.join(result, DATATOOLS_SUFFIX) if create_on_absent and not os.path.exists(result): os.mkdir(result) @@ -343,21 +342,20 @@ def store(self, flow_name, path, is_text, encoding, echo): echo( 'Including file %s of size %d%s %s' % (path, sz, unit[pos], extra)) try: - cur_obj = TransformableObject(io.open(path, mode='rb').read()) + input_file = io.open(path, mode='rb').read() except IOError: # If we get an error here, since we know that the file exists already, # it means that read failed which happens with Python 2.7 for large files raise MetaflowException('Cannot read file at %s -- this is likely because it is too ' 'large to be properly handled by Python 2.7' % path) - sha = sha1(cur_obj.current()).hexdigest() + sha = sha1(input_file).hexdigest() path = os.path.join(self._client_class.get_root_from_config(echo, True), flow_name, sha) buf = io.BytesIO() with gzip.GzipFile( fileobj=buf, mode='wb', compresslevel=3) as f: - f.write(cur_obj.current()) - cur_obj.transform(lambda _: buf) + f.write(input_file) buf.seek(0) with self._client_class() as client: url = client.put(path, buf.getvalue(), overwrite=False) diff --git a/metaflow/main_cli.py b/metaflow/main_cli.py index 48a0ab0f82f..8229ff8cbb8 100644 --- a/metaflow/main_cli.py +++ b/metaflow/main_cli.py @@ -5,8 +5,8 @@ from os.path import expanduser -from metaflow.datastore.local import LocalDataStore -from metaflow.metaflow_config import DATASTORE_LOCAL_DIR, DEFAULT_METADATA +from metaflow.datastore.local_storage import LocalStorage +from metaflow.metaflow_config import DATASTORE_LOCAL_DIR from metaflow.util import to_unicode @@ -106,8 +106,8 @@ def status(): from metaflow.client import namespace, metadata, Metaflow # Get the local data store path - path = LocalDataStore.get_datastore_root_from_config(echo, - create_on_absent=False) + path = LocalStorage.get_datastore_root_from_config( + echo, create_on_absent=False) # Throw an exception if path is None: raise click.ClickException("Could not find " +\ diff --git a/metaflow/metadata/metadata.py b/metaflow/metadata/metadata.py index 55932093c79..d5818e8e8b9 100644 --- a/metaflow/metadata/metadata.py +++ b/metaflow/metadata/metadata.py @@ -4,13 +4,12 @@ from collections import namedtuple from datetime import datetime -from metaflow.current import current from metaflow.exception import MetaflowInternalError from metaflow.util import get_username, resolve_identity DataArtifact = namedtuple('DataArtifact', - 'name ds_type url type sha') + 'name ds_type ds_root url type sha') MetaDatum = namedtuple('MetaDatum', 'field value type tags') @@ -251,7 +250,7 @@ def stop_heartbeat(self): pass @classmethod - def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters=None, *args): + def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters, *args): ''' Return objects for the implementation of this class @@ -299,7 +298,7 @@ def add_sticky_tags(self, tags=[], sys_tags=[]): self.sticky_sys_tags.extend(sys_tags) @classmethod - def get_object(cls, obj_type, sub_type, filters=None, *args): + def get_object(cls, obj_type, sub_type, filters, *args): '''Returns the requested object depending on obj_type and sub_type obj_type can be one of 'root', 'flow', 'run', 'step', 'task', @@ -431,7 +430,7 @@ def _artifacts_to_json(self, run_id, step_name, task_id, attempt_id, artifacts): 'type': 'metaflow.artifact', 'sha': art.sha, 'ds_type': art.ds_type, - 'location': art.url} + 'location': art.url if art.url else ':root:%s' % art.ds_root} d.update(self._all_obj_elements(self.sticky_tags, self.sticky_sys_tags)) result.append(d) return result diff --git a/metaflow/metadata/util.py b/metaflow/metadata/util.py new file mode 100644 index 00000000000..e7ec022f282 --- /dev/null +++ b/metaflow/metadata/util.py @@ -0,0 +1,33 @@ +from io import BytesIO +import os +import tarfile + +from distutils.dir_util import copy_tree + +from metaflow import util +from metaflow.datastore.local_storage import LocalStorage + + +def sync_local_metadata_to_datastore(metadata_local_dir, task_ds): + with util.TempDir() as td: + tar_file_path = os.path.join(td, 'metadata.tgz') + buf = BytesIO() + with tarfile.open(name=tar_file_path, mode='w:gz', fileobj=buf) as tar: + tar.add(metadata_local_dir) + blob = buf.getvalue() + _, key = task_ds.parent_datastore.save_data([blob], len_hint=1)[0] + task_ds.save_metadata({'local_metadata': key}) + + +def sync_local_metadata_from_datastore(metadata_local_dir, task_ds): + def echo_none(*args, **kwargs): + pass + key_to_load = task_ds.load_metadata(['local_metadata'])['local_metadata'] + _, tarball = next(task_ds.parent_datastore.load_data([key_to_load])) + with util.TempDir() as td: + with tarfile.open(fileobj=BytesIO(tarball), mode='r:gz') as tar: + tar.extractall(td) + copy_tree( + os.path.join(td, metadata_local_dir), + LocalStorage.get_datastore_root_from_config(echo_none), + update=True) \ No newline at end of file diff --git a/metaflow/mflog/__init__.py b/metaflow/mflog/__init__.py index 8c84a3ced72..85902e9dea4 100644 --- a/metaflow/mflog/__init__.py +++ b/metaflow/mflog/__init__.py @@ -42,11 +42,13 @@ # this function returns a bash expression that redirects stdout # and stderr of the given bash expression to mflog.tee -def bash_capture_logs(bash_expr): +def bash_capture_logs(bash_expr, var_transform=None): + if var_transform is None: + var_transform = lambda s: '$%s' % s cmd = 'python -m metaflow.mflog.tee %s %s' parts = (bash_expr, - cmd % (TASK_LOG_SOURCE, '$MFLOG_STDOUT'), - cmd % (TASK_LOG_SOURCE, '$MFLOG_STDERR')) + cmd % (TASK_LOG_SOURCE, var_transform('MFLOG_STDOUT')), + cmd % (TASK_LOG_SOURCE, var_transform('MFLOG_STDERR'))) return '(%s) 1>> >(%s) 2>> >(%s >&2)' % parts # update_delay determines how often logs should be uploaded to S3 diff --git a/metaflow/mflog/save_logs.py b/metaflow/mflog/save_logs.py index 5968878db4b..4e87053ce97 100644 --- a/metaflow/mflog/save_logs.py +++ b/metaflow/mflog/save_logs.py @@ -3,7 +3,7 @@ # This script is used to upload logs during task bootstrapping, so # it shouldn't have external dependencies besides Metaflow itself # (e.g. no click for parsing CLI args). -from metaflow.datastore import DATASTORES +from metaflow.datastore import DATASTORES, FlowDataStore from metaflow.util import Path from . import TASK_LOG_SOURCE @@ -23,19 +23,20 @@ def _read_file(path): os.environ['MFLOG_STDERR']) flow_name, run_id, step_name, task_id = pathspec.split('/') - Datastore = DATASTORES[ds_type] + storage_impl = DATASTORES[ds_type] if ds_root is None: def print_clean(line, **kwargs): pass - ds_root = Datastore.get_datastore_root_from_config(print_clean) - Datastore.datastore_root = ds_root - - ds = Datastore(flow_name, - run_id=run_id, - step_name=step_name, - task_id=task_id, - attempt=int(attempt), - mode='w') + ds_root = storage_impl.get_datastore_root_from_config(print_clean) + flow_datastore = FlowDataStore(flow_name, + None, + storage_impl=storage_impl, + ds_root=ds_root) + task_datastore = flow_datastore.get_task_datastore(run_id, + step_name, + task_id, + int(attempt), + mode='w') try: streams = ('stdout', 'stderr') @@ -48,8 +49,8 @@ def print_clean(line, **kwargs): else: op = Path - data = [(stream, op(path)) for stream, path, _ in sizes] - ds.save_logs(TASK_LOG_SOURCE, data) + data = {stream: op(path) for stream, path, _ in sizes} + task_datastore.save_logs(TASK_LOG_SOURCE, data) except: # Upload failing is not considered a fatal error. # This script shouldn't return non-zero exit codes diff --git a/metaflow/package.py b/metaflow/package.py index 105d7c35a10..dc3d18d8339 100644 --- a/metaflow/package.py +++ b/metaflow/package.py @@ -1,8 +1,8 @@ import os import sys import tarfile +import time import json -from hashlib import sha1 from io import BytesIO from itertools import chain @@ -30,13 +30,15 @@ def __init__(self, flow, environment, echo, suffixes=DEFAULT_SUFFIXES_LIST): 'METAFLOW_EXTENSIONS_PACKAGE_SUFFIXES', None) + self.flow_name = flow.name + self.create_time = time.time() environment.init_environment(echo) for step in flow: for deco in step.decorators: deco.package_init(flow, step.__name__, environment) - self.blob, self.sha = self._make() + self.blob = self._make() def _walk(self, root, exclude_hidden=True, addl_suffixes=None): if addl_suffixes is None: @@ -105,14 +107,16 @@ def no_mtime(tarinfo): return tarinfo buf = BytesIO() - with tarfile.TarFile(fileobj=buf, mode='w') as tar: + with tarfile.open(fileobj=buf, mode='w:gz', compresslevel=3) as tar: self._add_info(tar) for path, arcname in self.path_tuples(): tar.add(path, arcname=arcname, recursive=False, filter=no_mtime) - blob = buf.getvalue() - return blob, sha1(blob).hexdigest() + blob = bytearray(buf.getvalue()) + blob[4:8] = [0] * 4 # Reset 4 bytes from offset 4 to account for ts + return blob def __str__(self): - return '' % self.sha + return '' % \ + (self.flow_name, time.strftime("%a, %d %b %Y %H:%M:%S", self.create_time)) diff --git a/metaflow/plugins/aws/batch/batch_cli.py b/metaflow/plugins/aws/batch/batch_cli.py index 30bfa5e5581..a45319b1a26 100644 --- a/metaflow/plugins/aws/batch/batch_cli.py +++ b/metaflow/plugins/aws/batch/batch_cli.py @@ -1,34 +1,19 @@ +import click import os import sys -import tarfile import time import traceback -import click - from distutils.dir_util import copy_tree -from .batch import Batch, BatchKilledException, STDOUT_PATH, STDERR_PATH - -from metaflow.datastore import MetaflowDataStore -from metaflow.datastore.local import LocalDataStore -from metaflow.datatools.s3util import get_s3_client -from metaflow.metaflow_config import DATASTORE_LOCAL_DIR from metaflow import util from metaflow import R -from metaflow.exception import ( - CommandException, - METAFLOW_EXIT_DISALLOW_RETRY, -) +from metaflow.exception import CommandException, METAFLOW_EXIT_DISALLOW_RETRY +from metaflow.metadata.util import sync_local_metadata_from_datastore +from metaflow.metaflow_config import DATASTORE_LOCAL_DIR from metaflow.mflog import TASK_LOG_SOURCE - -try: - # python2 - from urlparse import urlparse -except: # noqa E722 - # python3 - from urllib.parse import urlparse +from .batch import Batch, BatchKilledException @click.group() def cli(): @@ -63,34 +48,6 @@ def _execute_cmd(func, flow_name, run_id, user, my_runs, echo): func(flow_name, run_id, user, echo) -def _sync_metadata(echo, metadata, datastore_root, attempt): - if metadata.TYPE == 'local': - def echo_none(*args, **kwargs): - pass - path = os.path.join( - datastore_root, - MetaflowDataStore.filename_with_attempt_prefix('metadata.tgz', attempt)) - url = urlparse(path) - bucket = url.netloc - key = url.path.lstrip('/') - s3, err = get_s3_client() - try: - s3.head_object(Bucket=bucket, Key=key) - # If we are here, we can download the object - with util.TempDir() as td: - tar_file_path = os.path.join(td, 'metadata.tgz') - with open(tar_file_path, 'wb') as f: - s3.download_fileobj(bucket, key, f) - with tarfile.open(tar_file_path, 'r:gz') as tar: - tar.extractall(td) - copy_tree( - os.path.join(td, DATASTORE_LOCAL_DIR), - LocalDataStore.get_datastore_root_from_config(echo_none), - update=True) - except err as e: # noqa F841 - pass - - @batch.command(help="List unfinished AWS Batch tasks of this flow") @click.option("--my-runs", default=False, is_flag=True, help="List all my unfinished tasks.") @@ -197,9 +154,6 @@ def echo(msg, stream='stderr', batch_id=None): msg = '[%s] %s' % (batch_id, msg) ctx.obj.echo_always(msg, err=(stream == sys.stderr)) - if ctx.obj.datastore.datastore_root is None: - ctx.obj.datastore.datastore_root = ctx.obj.datastore.get_datastore_root_from_config(echo) - if R.use_r(): entrypoint = R.entrypoint() else: @@ -255,8 +209,6 @@ def echo(msg, stream='stderr', batch_id=None): else: env = {} - datastore_root = os.path.join(ctx.obj.datastore.make_path( - ctx.obj.flow.name, kwargs['run_id'], step_name, kwargs['task_id'])) # Add the environment variables related to the input-paths argument if split_vars: env.update(split_vars) @@ -268,12 +220,24 @@ def echo(msg, stream='stderr', batch_id=None): time.sleep(minutes_between_retries * 60) # this information is needed for log tailing - spec = task_spec.copy() - spec['attempt'] = int(spec.pop('retry_count')) - ds = ctx.obj.datastore(mode='w', **spec) + ds = ctx.obj.flow_datastore.get_task_datastore( + mode='w', + run_id=kwargs['run_id'], + step_name=step_name, + task_id=kwargs['task_id'], + attempt=int(retry_count) + ) stdout_location = ds.get_log_location(TASK_LOG_SOURCE, 'stdout') stderr_location = ds.get_log_location(TASK_LOG_SOURCE, 'stderr') + def _sync_metadata(): + if ctx.obj.metadata.TYPE == 'local': + sync_local_metadata_from_datastore( + DATASTORE_LOCAL_DIR, + ctx.obj.flow_datastore.get_task_datastore(kwargs['run_id'], + step_name, + kwargs['task_id'])) + batch = Batch(ctx.obj.metadata, ctx.obj.environment) try: with ctx.obj.monitor.measure("metaflow.batch.launch"): @@ -283,7 +247,7 @@ def echo(msg, stream='stderr', batch_id=None): task_spec, code_package_sha, code_package_url, - ctx.obj.datastore.TYPE, + ctx.obj.flow_datastore.TYPE, image=image, queue=queue, iam_role=iam_role, @@ -301,13 +265,13 @@ def echo(msg, stream='stderr', batch_id=None): ) except Exception as e: print(e) - _sync_metadata(echo, ctx.obj.metadata, datastore_root, retry_count) + _sync_metadata() sys.exit(METAFLOW_EXIT_DISALLOW_RETRY) try: batch.wait(stdout_location, stderr_location, echo=echo) except BatchKilledException: # don't retry killed tasks traceback.print_exc() - _sync_metadata(echo, ctx.obj.metadata, datastore_root, retry_count) sys.exit(METAFLOW_EXIT_DISALLOW_RETRY) - _sync_metadata(echo, ctx.obj.metadata, datastore_root, retry_count) + finally: + _sync_metadata() diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index 9596c724596..40d76e0c699 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -5,31 +5,22 @@ import tarfile import requests -from metaflow.datastore import MetaflowDataStore -from metaflow.datastore.datastore import TransformableObject -from metaflow.datatools.s3util import get_s3_client from metaflow.decorators import StepDecorator from metaflow.metaflow_config import DATASTORE_LOCAL_DIR from metaflow.plugins import ResourcesDecorator from metaflow.plugins.timeout_decorator import get_run_time_limit_for_task from metaflow.metadata import MetaDatum +from metaflow.metadata.util import sync_local_metadata_to_datastore from metaflow import util from metaflow import R -from .batch import Batch, BatchException +from .batch import BatchException from metaflow.metaflow_config import ECS_S3_ACCESS_IAM_ROLE, BATCH_JOB_QUEUE, \ BATCH_CONTAINER_IMAGE, BATCH_CONTAINER_REGISTRY, \ ECS_FARGATE_EXECUTION_ROLE from metaflow.sidecar import SidecarSubProcess -try: - # python2 - from urlparse import urlparse -except: # noqa E722 - # python3 - from urllib.parse import urlparse - class BatchDecorator(StepDecorator): """ @@ -123,13 +114,21 @@ def __init__(self, attributes=None, statically_defined=False): self.attributes['image'] = '%s/%s' % (BATCH_CONTAINER_REGISTRY.rstrip('/'), self.attributes['image']) - def step_init(self, flow, graph, step, decos, environment, datastore, logger): - if datastore.TYPE != 's3': + def step_init(self, + flow, + graph, + step, + decos, + environment, + flow_datastore, + logger): + if flow_datastore.TYPE != 's3': raise BatchException('The *@batch* decorator requires --datastore=s3.') self.logger = logger self.environment = environment self.step = step + self.flow_datastore = flow_datastore for deco in decos: if isinstance(deco, ResourcesDecorator): for k, v in deco.attributes.items(): @@ -149,14 +148,14 @@ def runtime_init(self, flow, graph, package, run_id): self.run_id = run_id def runtime_task_created(self, - datastore, + task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context): if not is_cloned: - self._save_package_once(datastore, self.package) + self._save_package_once(self.flow_datastore, self.package) def runtime_step_cli(self, cli_args, @@ -176,7 +175,7 @@ def runtime_step_cli(self, def task_pre_step(self, step_name, - ds, + task_datastore, metadata, run_id, task_id, @@ -187,9 +186,9 @@ def task_pre_step(self, ubf_context, inputs): if metadata.TYPE == 'local': - self.ds_root = ds.root + self.task_datastore = task_datastore else: - self.ds_root = None + self.task_datastore = None meta = {} meta['aws-batch-job-id'] = os.environ['AWS_BATCH_JOB_ID'] meta['aws-batch-job-attempt'] = os.environ['AWS_BATCH_JOB_ATTEMPT'] @@ -221,35 +220,44 @@ def task_pre_step(self, metadata.register_metadata(run_id, step_name, task_id, entries) self._save_logs_sidecar = SidecarSubProcess('save_logs_periodically') - def task_finished(self, step_name, flow, graph, is_task_ok, retry_count, max_retries): - if self.ds_root: - # We have a local metadata service so we need to persist it to the datastore. - # Note that the datastore is *always* s3 (see runtime_task_created function) - with util.TempDir() as td: - tar_file_path = os.path.join(td, 'metadata.tgz') - with tarfile.open(tar_file_path, 'w:gz') as tar: - # The local metadata is stored in the local datastore - # which, for batch jobs, is always the DATASTORE_LOCAL_DIR - tar.add(DATASTORE_LOCAL_DIR) - # At this point we upload what need to s3 - s3, _ = get_s3_client() - with open(tar_file_path, 'rb') as f: - path = os.path.join( - self.ds_root, - MetaflowDataStore.filename_with_attempt_prefix( - 'metadata.tgz', retry_count)) - url = urlparse(path) - s3.upload_fileobj(f, url.netloc, url.path.lstrip('/')) + def task_post_step(self, + step_name, + flow, + graph, + retry_count, + max_user_code_retries): + if self.task_datastore: + sync_local_metadata_to_datastore(DATASTORE_LOCAL_DIR, + self.task_datastore) + + def task_exception(self, + exception, + step_name, + flow, + graph, + retry_count, + max_user_code_retries): + if self.task_datastore: + sync_local_metadata_to_datastore(DATASTORE_LOCAL_DIR, + self.task_datastore) + + def task_finished(self, + step_name, + flow, + graph, + is_task_ok, + retry_count, + max_retries): try: self._save_logs_sidecar.kill() except: pass @classmethod - def _save_package_once(cls, datastore, package): + def _save_package_once(cls, flow_datastore, package): if cls.package_url is None: - cls.package_url = datastore.save_data(package.sha, TransformableObject(package.blob)) - cls.package_sha = package.sha + cls.package_url, cls.package_sha = flow_datastore.save_data( + [package.blob], len_hint=1)[0] @classmethod def _get_registry(cls, image): @@ -303,4 +311,4 @@ def _get_registry(cls, image): registry, repository, tag = pattern.match(image).groups() if registry is not None: registry = registry.rstrip("/") - return registry + return registry \ No newline at end of file diff --git a/metaflow/plugins/aws/step_functions/schedule_decorator.py b/metaflow/plugins/aws/step_functions/schedule_decorator.py index 2dd96ef852b..d3e789620ec 100644 --- a/metaflow/plugins/aws/step_functions/schedule_decorator.py +++ b/metaflow/plugins/aws/step_functions/schedule_decorator.py @@ -12,7 +12,7 @@ def flow_init(self, flow, graph, environment, - datastore, + flow_datastore, metadata, logger, echo, diff --git a/metaflow/plugins/aws/step_functions/step_functions.py b/metaflow/plugins/aws/step_functions/step_functions.py index 0cd73e4e377..8e306e0c3d0 100644 --- a/metaflow/plugins/aws/step_functions/step_functions.py +++ b/metaflow/plugins/aws/step_functions/step_functions.py @@ -34,11 +34,11 @@ def __init__(self, name, graph, flow, - code_package, + code_package_sha, code_package_url, production_token, metadata, - datastore, + flow_datastore, environment, event_logger, monitor, @@ -51,11 +51,11 @@ def __init__(self, self.name = name self.graph = graph self.flow = flow - self.code_package = code_package + self.code_package_sha = code_package_sha self.code_package_url = code_package_url self.production_token = production_token self.metadata = metadata - self.datastore = datastore + self.flow_datastore = flow_datastore self.environment = environment self.event_logger = event_logger self.monitor = monitor @@ -598,9 +598,9 @@ def _batch(self, node): self.code_package_url, user_code_retries), task_spec=task_spec, - code_package_sha=self.code_package.sha, + code_package_sha=self.code_package_sha, code_package_url=self.code_package_url, - code_package_ds=self.datastore.TYPE, + code_package_ds=self.flow_datastore.TYPE, image=resources['image'], queue=resources['queue'], iam_role=resources['iam_role'], @@ -710,8 +710,8 @@ def _step_cli(self, '--quiet', '--metadata=%s' % self.metadata.TYPE, '--environment=%s' % self.environment.TYPE, - '--datastore=%s' % self.datastore.TYPE, - '--datastore-root=%s' % self.datastore.datastore_root, + '--datastore=%s' % self.flow_datastore.TYPE, + '--datastore-root=%s' % self.flow_datastore.datastore_root, '--event-logger=%s' % self.event_logger.logger_type, '--monitor=%s' % self.monitor.monitor_type, '--no-pylint', diff --git a/metaflow/plugins/aws/step_functions/step_functions_cli.py b/metaflow/plugins/aws/step_functions/step_functions_cli.py index 1bbe2ac8a90..3192cc593d2 100644 --- a/metaflow/plugins/aws/step_functions/step_functions_cli.py +++ b/metaflow/plugins/aws/step_functions/step_functions_cli.py @@ -8,7 +8,6 @@ from metaflow import current, decorators, parameters, JSONType from metaflow.metaflow_config import SFN_STATE_MACHINE_PREFIX from metaflow.exception import MetaflowException, MetaflowInternalError -from metaflow.datastore.datastore import TransformableObject from metaflow.package import MetaflowPackage from metaflow.plugins import BatchDecorator from metaflow.util import get_username, to_bytes, to_unicode @@ -221,32 +220,30 @@ def make_flow(obj, max_workers, workflow_timeout, is_project): - datastore = obj.datastore(obj.flow.name, - mode='w', - metadata=obj.metadata, - event_logger=obj.event_logger, - monitor=obj.monitor) - if datastore.TYPE != 's3': + if obj.flow_datastore.TYPE != 's3': raise MetaflowException("AWS Step Functions requires --datastore=s3.") # Attach AWS Batch decorator to the flow decorators._attach_decorators(obj.flow, [BatchDecorator.name]) - decorators._init_step_decorators( - obj.flow, obj.graph, obj.environment, obj.datastore, obj.logger) + decorators._init_step_decorators(obj.flow, + obj.graph, + obj.environment, + obj.flow_datastore, + obj.logger) obj.package = MetaflowPackage( obj.flow, obj.environment, obj.echo, obj.package_suffixes) - package_url = datastore.save_data( - obj.package.sha, TransformableObject(obj.package.blob)) + package_url, package_sha = obj.flow_datastore.save_data( + [obj.package.blob], len_hint=1)[0] return StepFunctions(name, obj.graph, obj.flow, - obj.package, + package_sha, package_url, token, obj.metadata, - obj.datastore, + obj.flow_datastore, obj.environment, obj.event_logger, obj.monitor, diff --git a/metaflow/plugins/aws/step_functions/step_functions_decorator.py b/metaflow/plugins/aws/step_functions/step_functions_decorator.py index ce1d786e776..c26a3ebca07 100644 --- a/metaflow/plugins/aws/step_functions/step_functions_decorator.py +++ b/metaflow/plugins/aws/step_functions/step_functions_decorator.py @@ -12,7 +12,7 @@ class StepFunctionsInternalDecorator(StepDecorator): def task_pre_step(self, step_name, - datastore, + task_datastore, metadata, run_id, task_id, diff --git a/metaflow/plugins/catch_decorator.py b/metaflow/plugins/catch_decorator.py index 545fd27f6ff..fc70d715d39 100644 --- a/metaflow/plugins/catch_decorator.py +++ b/metaflow/plugins/catch_decorator.py @@ -49,7 +49,7 @@ def myStep(self): defaults = {'var': None, 'print_exception': True} - def step_init(self, flow, graph, step, decos, environment, datastore, logger): + def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger): # handling _foreach_var and _foreach_num_splits requires some # deeper thinking, so let's not support that use case for now self.logger = logger diff --git a/metaflow/plugins/conda/conda_environment.py b/metaflow/plugins/conda/conda_environment.py index faa15d08c89..c34c902dc0e 100644 --- a/metaflow/plugins/conda/conda_environment.py +++ b/metaflow/plugins/conda/conda_environment.py @@ -109,9 +109,9 @@ def get_client_info(cls, flow_name, metadata): if info is None or env_id is None: return {'type': 'conda'} info = json.loads(info) - with cls._filecache.get_data(info['ds_type'], flow_name, info['sha']) as f: - tar = tarfile.TarFile(fileobj=f) - conda_file = tar.extractfile(CONDA_MAGIC_FILE) + with cls._filecache.get_data(info['ds_type'], flow_name, info['location'], info['sha']) as f: + with tarfile.open(fileobj=f, mode='r:gz') as tar: + conda_file = tar.extractfile(CONDA_MAGIC_FILE) if conda_file is None: return {'type': 'conda'} info = json.loads(conda_file.read().decode('utf-8')) diff --git a/metaflow/plugins/conda/conda_flow_decorator.py b/metaflow/plugins/conda/conda_flow_decorator.py index 552bd0a9b7e..18b0d07cae9 100644 --- a/metaflow/plugins/conda/conda_flow_decorator.py +++ b/metaflow/plugins/conda/conda_flow_decorator.py @@ -41,7 +41,7 @@ def flow_init(self, flow, graph, environment, - datastore, + flow_datastore, metadata, logger, echo, diff --git a/metaflow/plugins/conda/conda_step_decorator.py b/metaflow/plugins/conda/conda_step_decorator.py index 28c8bf13caf..c73f84c1e3e 100644 --- a/metaflow/plugins/conda/conda_step_decorator.py +++ b/metaflow/plugins/conda/conda_step_decorator.py @@ -12,12 +12,12 @@ from urllib.parse import urlparse -from metaflow.datastore.local import LocalDataStore from metaflow.decorators import StepDecorator from metaflow.metaflow_environment import InvalidEnvironmentException from metaflow.metadata import MetaDatum from metaflow.metaflow_config import get_pinned_conda_libs, CONDA_PACKAGE_S3ROOT from metaflow.util import get_metaflow_root +from metaflow.datastore import LocalStorage from metaflow.datatools import S3 from metaflow.unbounded_foreach import UBF_CONTROL @@ -133,7 +133,7 @@ def _resolve_step_environment(self, ds_root, force=False): } else: payload = cached_deps[env_id] - if self.datastore.TYPE == 's3' and 'cache_urls' not in payload: + if self.flow_datastore.TYPE == 's3' and 'cache_urls' not in payload: payload['cache_urls'] = self._cache_env() write_to_conda_manifest(ds_root, self.flow.name, env_id, payload) CondaStepDecorator.environments =\ @@ -241,19 +241,19 @@ def runtime_init(self, flow, graph, package, run_id): generate_trampolines(self.metaflow_home) - def step_init(self, flow, graph, step, decos, environment, datastore, logger): + def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger): if environment.TYPE != 'conda': raise InvalidEnvironmentException('The *@conda* decorator requires ' '--environment=conda') def _logger(line, **kwargs): logger(line) - self.local_root = LocalDataStore.get_datastore_root_from_config(_logger) + self.local_root = LocalStorage.get_datastore_root_from_config(_logger) environment.set_local_root(self.local_root) self.architecture = self._architecture(decos) self.disable_safety_checks = self._disable_safety_checks(decos) self.step = step self.flow = flow - self.datastore = datastore + self.flow_datastore = flow_datastore self.base_attributes = self._get_base_attributes() os.environ['PYTHONNOUSERSITE'] = '1' @@ -262,7 +262,7 @@ def package_init(self, flow, step, environment): self._prepare_step_environment(step, self.local_root) def runtime_task_created(self, - datastore, + task_datastore, task_id, split_index, input_paths, @@ -274,7 +274,7 @@ def runtime_task_created(self, def task_pre_step(self, step_name, - ds, + task_datastore, meta, run_id, task_id, diff --git a/metaflow/plugins/metadata/local.py b/metaflow/plugins/metadata/local.py index 2fb8d6bc0b9..ab3b0c36a53 100644 --- a/metaflow/plugins/metadata/local.py +++ b/metaflow/plugins/metadata/local.py @@ -15,21 +15,21 @@ def __init__(self, environment, flow, event_logger, monitor): @classmethod def compute_info(cls, val): - from metaflow.datastore.local import LocalDataStore + from metaflow.datastore.local_storage import LocalStorage v = os.path.realpath(os.path.join(val, DATASTORE_LOCAL_DIR)) if os.path.isdir(v): - LocalDataStore.datastore_root = v + LocalStorage.datastore_root = v return val raise ValueError( 'Could not find directory %s in directory %s' % (DATASTORE_LOCAL_DIR, val)) @classmethod def default_info(cls): - from metaflow.datastore.local import LocalDataStore + from metaflow.datastore.local_storage import LocalStorage def print_clean(line, **kwargs): print(line) - v = LocalDataStore.get_datastore_root_from_config(print_clean, create_on_absent=False) + v = LocalStorage.get_datastore_root_from_config(print_clean, create_on_absent=False) if v is None: return '' % DATASTORE_LOCAL_DIR return os.path.dirname(v) @@ -97,8 +97,8 @@ def register_metadata(self, run_id, step_name, task_id, metadata): self._save_meta(meta_dir, metadict) @classmethod - def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters=None, *args): - from metaflow.datastore.local import LocalDataStore + def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters, *args): + from metaflow.datastore.local_storage import LocalStorage if obj_type == 'artifact': # Artifacts are actually part of the tasks in the filesystem obj_type = 'task' @@ -154,7 +154,7 @@ def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters= if obj_path is None: return result skip_dirs = '*/'*(sub_order - obj_order) - all_meta = os.path.join(obj_path, skip_dirs, LocalDataStore.METADATA_DIR) + all_meta = os.path.join(obj_path, skip_dirs, LocalStorage.METADATA_DIR) for meta_path in glob.iglob(all_meta): self_file = os.path.join(meta_path, '_self.json') if os.path.isfile(self_file): @@ -202,37 +202,49 @@ def _new_task(self, run_id, step_name, task_id, attempt=0, tags=[], sys_tags=[]) @staticmethod def _make_path( - flow_name=None, run_id=None, step_name=None, task_id=None, pathspec=None, - create_on_absent=True): + flow_name=None, run_id=None, step_name=None, task_id=None, + create_on_absent=True): - from metaflow.datastore.local import LocalDataStore - if LocalDataStore.datastore_root is None: + from metaflow.datastore.local_storage import LocalStorage + if LocalStorage.datastore_root is None: def print_clean(line, **kwargs): print(line) - LocalDataStore.datastore_root = LocalDataStore.get_datastore_root_from_config( + LocalStorage.datastore_root = LocalStorage.get_datastore_root_from_config( print_clean, create_on_absent=create_on_absent) - if LocalDataStore.datastore_root is None: + if LocalStorage.datastore_root is None: return None - return LocalDataStore.make_path(flow_name, run_id, step_name, task_id, pathspec) + if flow_name is None: + return LocalStorage.datastore_root + components = [] + if flow_name: + components.append(flow_name) + if run_id: + components.append(run_id) + if step_name: + components.append(step_name) + if task_id: + components.append(task_id) + return LocalStorage().full_uri( + LocalStorage.path_join(*components)) @staticmethod def _create_and_get_metadir( - flow_name=None, run_id=None, step_name=None, task_id=None): - from metaflow.datastore.local import LocalDataStore + flow_name=None, run_id=None, step_name=None, task_id=None): + from metaflow.datastore.local_storage import LocalStorage root_path = LocalMetadataProvider._make_path(flow_name, run_id, step_name, task_id) - subpath = os.path.join(root_path, LocalDataStore.METADATA_DIR) + subpath = os.path.join(root_path, LocalStorage.METADATA_DIR) LocalMetadataProvider._makedirs(subpath) return subpath @staticmethod def _get_metadir(flow_name=None, run_id=None, step_name=None, task_id=None): - from metaflow.datastore.local import LocalDataStore + from metaflow.datastore.local_storage import LocalStorage root_path = LocalMetadataProvider._make_path( flow_name, run_id, step_name, task_id, create_on_absent=False) if root_path is None: return None - subpath = os.path.join(root_path, LocalDataStore.METADATA_DIR) + subpath = os.path.join(root_path, LocalStorage.METADATA_DIR) if os.path.isdir(subpath): return subpath return None diff --git a/metaflow/plugins/package_cli.py b/metaflow/plugins/package_cli.py index 1b1e544ec7d..0d42004cc83 100644 --- a/metaflow/plugins/package_cli.py +++ b/metaflow/plugins/package_cli.py @@ -1,4 +1,5 @@ import click +from hashlib import sha1 from metaflow.package import MetaflowPackage @click.group() @@ -18,7 +19,7 @@ def package(obj): @click.pass_obj def info(obj): obj.echo('Status of the current working directory:', fg='magenta', bold=False) - obj.echo_always('Hash: *%s*' % obj.package.sha, + obj.echo_always('Hash: *%s*' % sha1(obj.package.blob).hexdigest(), highlight='green', highlight_bold=False) obj.echo_always('Package size: *%d* KB' % (len(obj.package.blob) / 1024), diff --git a/metaflow/plugins/project_decorator.py b/metaflow/plugins/project_decorator.py index 96fbc9d6ae1..89f7929966d 100644 --- a/metaflow/plugins/project_decorator.py +++ b/metaflow/plugins/project_decorator.py @@ -34,7 +34,7 @@ def flow_init(self, flow, graph, environment, - datastore, + flow_datastore, metadata, logger, echo, diff --git a/metaflow/plugins/retry_decorator.py b/metaflow/plugins/retry_decorator.py index 3778eae8cec..68e52375907 100644 --- a/metaflow/plugins/retry_decorator.py +++ b/metaflow/plugins/retry_decorator.py @@ -33,7 +33,7 @@ def myStep(self): defaults = {'times': '3', 'minutes_between_retries': '2'} - def step_init(self, flow, graph, step, decos, environment, datastore, logger): + def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger): # The total number of attempts must not exceed MAX_ATTEMPTS. # attempts = normal task (1) + retries (N) + @catch fallback (1) if int(self.attributes['times']) + 2 > MAX_ATTEMPTS: diff --git a/metaflow/plugins/timeout_decorator.py b/metaflow/plugins/timeout_decorator.py index 4af34223836..f5278177fbb 100644 --- a/metaflow/plugins/timeout_decorator.py +++ b/metaflow/plugins/timeout_decorator.py @@ -56,14 +56,14 @@ def __init__(self, *args, **kwargs): int(self.attributes['minutes']) * 60 +\ int(self.attributes['seconds']) - def step_init(self, flow, graph, step, decos, environment, datastore, logger): + def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger): self.logger = logger if not self.secs: raise MetaflowException('Specify a duration for @timeout.') def task_pre_step(self, step_name, - datastore, + task_datastore, metadata, run_id, task_id, diff --git a/metaflow/runtime.py b/metaflow/runtime.py index 8afb00b9ffd..f02e1cbf9cf 100644 --- a/metaflow/runtime.py +++ b/metaflow/runtime.py @@ -22,7 +22,8 @@ MetaflowInternalError,\ METAFLOW_EXIT_DISALLOW_RETRY from . import procpoll -from .datastore import DataException, MetaflowDatastoreSet +from .datastore import TaskDataStoreSet +from .datastore.exceptions import DataException from .metadata import MetaDatum from .debug import debug from .decorators import flow_decorators @@ -36,7 +37,7 @@ PROGRESS_INTERVAL = 1000 #ms # The following is a list of the (data) artifacts used by the runtime while # executing a flow. These are prefetched during the resume operation by -# leveraging the MetaflowDatastoreSet. +# leveraging the TaskDataStoreSet. PREFETCH_DATA_ARTIFACTS = ['_foreach_stack', '_task_ok', '_transition'] # Runtime must use logsource=RUNTIME_LOG_SOURCE for all loglines that it @@ -50,7 +51,7 @@ class NativeRuntime(object): def __init__(self, flow, graph, - datastore, + flow_datastore, metadata, environment, package, @@ -73,7 +74,7 @@ def __init__(self, self._flow = flow self._graph = graph - self._datastore = datastore + self._flow_datastore = flow_datastore self._metadata = metadata self._environment = environment self._logger = logger @@ -105,7 +106,7 @@ def __init__(self, # 3. All steps that couldn't be cloned (either unsuccessful or not # run) are run as regular tasks. # Lastly, to improve the performance of the cloning process, we - # leverage the MetaflowDatastoreSet abstraction to prefetch the + # leverage the TaskDataStoreSet abstraction to prefetch the # entire DAG of `clone_run_id` and relevant data artifacts # (see PREFETCH_DATA_ARTIFACTS) so that the entire runtime can # access the relevant data from cache (instead of going to the datastore @@ -113,13 +114,9 @@ def __init__(self, logger( 'Gathering required information to resume run (this may take a bit of time)...') self._origin_ds_set = \ - MetaflowDatastoreSet( - datastore, - flow.name, + TaskDataStoreSet( + flow_datastore, clone_run_id, - metadata=metadata, - event_logger=event_logger, - monitor=monitor, prefetch_data_artifacts=PREFETCH_DATA_ARTIFACTS) self._run_queue = [] self._poll = procpoll.make_poll() @@ -154,7 +151,7 @@ def _new_task(self, step, input_paths=None, **kwargs): else: decos = getattr(self._flow, step).decorators - return Task(self._datastore, + return Task(self._flow_datastore, self._flow, step, self._run_id, @@ -536,7 +533,7 @@ class Task(object): clone_pathspec_mapping = {} def __init__(self, - datastore, + flow_datastore, flow, step, run_id, @@ -606,9 +603,9 @@ def __init__(self, self.monitor_type = monitor.monitor_type self.metadata_type = metadata.TYPE - self.datastore_type = datastore.TYPE - self._datastore = datastore - self.datastore_sysroot = datastore.datastore_root + self.datastore_type = flow_datastore.TYPE + self._flow_datastore = flow_datastore + self.datastore_sysroot = flow_datastore.datastore_root self._results_ds = None if clone_run_id and may_clone: @@ -650,18 +647,10 @@ def __init__(self, self.error_retries = 0 def new_attempt(self): - self._ds = self._datastore(self.flow_name, - run_id=self.run_id, - step_name=self.step, - task_id=self.task_id, - mode='w', - metadata=self.metadata, - attempt=self.retries, - event_logger=self.event_logger, - monitor=self.monitor) + self._ds = self._flow_datastore.get_task_datastore( + self.run_id, self.step, self.task_id, attempt=self.retries, mode='w') self._ds.init_task() - def log(self, msg, system_msg=False, pid=None, timestamp=True): if pid: prefix = '[%s (pid %s)] ' % (self._path, pid) @@ -743,14 +732,8 @@ def results(self): if self._results_ds: return self._results_ds else: - self._results_ds = self._datastore(self.flow_name, - run_id=self.run_id, - step_name=self.step, - task_id=self.task_id, - mode='r', - metadata=self.metadata, - event_logger=self.event_logger, - monitor=self.monitor) + self._results_ds = self._flow_datastore.get_task_datastore( + self.run_id, self.step, self.task_id) return self._results_ds @property @@ -769,14 +752,11 @@ def persist(self, flow): self._ds.persist(flow) self._ds.done() - def save_logs(self, stdout_buffer, stderr_buffer): - self._ds.save_logs(RUNTIME_LOG_SOURCE, [ - ('stdout', stdout_buffer), - ('stderr', stderr_buffer) - ]) + def save_logs(self, logtype_to_logs): + self._ds.save_logs(RUNTIME_LOG_SOURCE, logtype_to_logs) def save_metadata(self, name, metadata): - self._ds.save_metadata(name, metadata) + self._ds.save_metadata({name: metadata}) def __str__(self): return ' '.join(self._args) @@ -819,6 +799,10 @@ def write(self, bytedata, system_msg=False): def get_bytes(self): return self._buffer.getvalue() + def get_buffer(self): + self._buffer.seek(0) + return self._buffer + class CLIArgs(object): """ @@ -1052,8 +1036,8 @@ def terminate(self): # perform any log collection. if not self.task.is_cloned: self.task.save_metadata('runtime', {'return_code': returncode, - 'killed': self.killed, - 'success': returncode == 0}) + 'killed': self.killed, + 'success': returncode == 0}) if returncode: if not self.killed: if returncode == -11: @@ -1073,8 +1057,10 @@ def terminate(self): self.task.log('Task finished successfully.', system_msg=True, pid=self._proc.pid) - self.task.save_logs(self._stdout.get_bytes(), - self._stderr.get_bytes()) + self.task.save_logs({ + 'stdout': self._stdout.get_buffer(), + 'stderr': self._stderr.get_buffer()}) + return returncode def __str__(self): diff --git a/metaflow/task.py b/metaflow/task.py index f92aa747dbe..4f23793ee2a 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -5,7 +5,7 @@ from .metaflow_config import MAX_ATTEMPTS from .metadata import MetaDatum -from .datastore import Inputs, MetaflowDatastoreSet +from .datastore import Inputs, TaskDataStoreSet from .exception import MetaflowInternalError,\ MetaflowDataMissing,\ MetaflowExceptionWrapper @@ -29,7 +29,7 @@ class MetaflowTask(object): def __init__(self, flow, - datastore, + flow_datastore, metadata, environment, console_logger, @@ -37,7 +37,7 @@ def __init__(self, monitor, ubf_context): self.flow = flow - self.datastore = datastore + self.flow_datastore = flow_datastore self.metadata = metadata self.environment = environment self.console_logger = console_logger @@ -59,7 +59,8 @@ def _init_parameters(self, parameter_ds, passdown=True): # make the parameter a read-only property # note x=x binds the current value of x to the closure def property_setter( - _, cls=self.flow.__class__, param=param, var=var, parameter_ds=parameter_ds): + _, cls=self.flow.__class__, param=param, var=var, + parameter_ds=parameter_ds): v = param.load_parameter(parameter_ds[var]) setattr(cls, var, property(fget=lambda _, val=v: val)) return v @@ -74,7 +75,7 @@ def property_setter( def _init_data(self, run_id, join_type, input_paths): # We prefer to use the parallelized version to initialize datastores - # (via MetaflowDatastoreSet) only with more than 4 datastores, because + # (via TaskDataStoreSet) only with more than 4 datastores, because # the baseline overhead of using the set is ~1.5s and each datastore # init takes ~200-300ms when run sequentially. if len(input_paths) > 4: @@ -87,13 +88,9 @@ def _init_data(self, run_id, join_type, input_paths): # Note: Specify `pathspecs` while creating the datastore set to # guarantee strong consistency and guard against missing input. datastore_set = \ - MetaflowDatastoreSet(self.datastore, - self.flow.name, + TaskDataStoreSet(self.flow_datastore, run_id, pathspecs=input_paths, - metadata=self.metadata, - event_logger=self.event_logger, - monitor=self.monitor, prefetch_data_artifacts=prefetch_data_artifacts) ds_list = [ds for ds in datastore_set] if len(ds_list) != len(input_paths): @@ -106,13 +103,7 @@ def _init_data(self, run_id, join_type, input_paths): for input_path in input_paths: run_id, step_name, task_id = input_path.split('/') ds_list.append( - self.datastore(self.flow.name, - run_id=run_id, - step_name=step_name, - task_id=task_id, - metadata=self.metadata, - event_logger=self.event_logger, - monitor=self.monitor)) + self.flow_datastore.get_task_datastore(run_id, step_name, task_id)) if not ds_list: # this guards against errors in input paths raise MetaflowDataMissing("Input paths *%s* resolved to zero " @@ -212,26 +203,17 @@ def clone_only(self, step_name, run_id, task_id, clone_origin_task): raise MetaflowInternalError("task.clone_only needs a valid " "clone_origin_task value.") # 1. initialize output datastore - output = self.datastore(self.flow.name, - run_id=run_id, - step_name=step_name, - task_id=task_id, - mode='w', - metadata=self.metadata, - attempt=0, - event_logger=self.event_logger, - monitor=self.monitor) + output = self.flow_datastore.get_task_datastore( + run_id, step_name, task_id, attempt=0, mode='w') + output.init_task() + origin_run_id, origin_step_name, origin_task_id =\ clone_origin_task.split('/') # 2. initialize origin datastore - origin = self.datastore(self.flow.name, - run_id=origin_run_id, - step_name=origin_step_name, - task_id=origin_task_id, - metadata=self.metadata, - event_logger=self.event_logger, - monitor=self.monitor) + origin = self.flow_datastore.get_task_datastore( + origin_run_id, origin_step_name, origin_task_id) + output.clone(origin) output.done() @@ -296,11 +278,11 @@ def run_step(self, type='origin-run-id', tags=metadata_tags), MetaDatum(field='ds-type', - value=self.datastore.TYPE, + value=self.flow_datastore.TYPE, type='ds-type', tags=metadata_tags), MetaDatum(field='ds-root', - value=self.datastore.datastore_root, + value=self.flow_datastore.datastore_root, type='ds-root', tags=metadata_tags)]) @@ -311,15 +293,9 @@ def run_step(self, join_type = self.flow._graph[node.split_parents[-1]].type # 1. initialize output datastore - output = self.datastore(self.flow.name, - run_id=run_id, - step_name=step_name, - task_id=task_id, - mode='w', - metadata=self.metadata, - attempt=retry_count, - event_logger=self.event_logger, - monitor=self.monitor) + output = self.flow_datastore.get_task_datastore( + run_id, step_name, task_id, attempt=retry_count, mode='w') + output.init_task() if input_paths: @@ -346,12 +322,13 @@ def run_step(self, is_running=True) # 5. run task - output.save_metadata('task_begin', { - 'code_package_sha': os.environ.get('METAFLOW_CODE_SHA'), - 'code_package_ds': os.environ.get('METAFLOW_CODE_DS'), - 'code_package_url': os.environ.get('METAFLOW_CODE_URL'), - 'retry_count': retry_count - }) + output.save_metadata({'task_begin': + { + 'code_package_sha': os.environ.get('METAFLOW_CODE_SHA'), + 'code_package_ds': os.environ.get('METAFLOW_CODE_DS'), + 'code_package_url': os.environ.get('METAFLOW_CODE_URL'), + 'retry_count': retry_count + }}) logger = self.event_logger start = time.time() self.metadata.start_task_heartbeat(self.flow.name, run_id, step_name, @@ -461,7 +438,6 @@ def run_step(self, self.flow._success = True except Exception as ex: - tsk_msg = { "task_id": task_id, "exception_msg": str(ex), @@ -518,7 +494,7 @@ def run_step(self, format(retry_count)]) ]) - output.save_metadata('task_end', {}) + output.save_metadata({'task_end': {}}) output.persist(self.flow) # this writes a success marker indicating that the diff --git a/metaflow/util.py b/metaflow/util.py index 71da7efdb03..a577f7f6d09 100644 --- a/metaflow/util.py +++ b/metaflow/util.py @@ -29,7 +29,7 @@ def unquote_bytes(x): return to_unicode(unquote(to_bytes(x))) - # this is used e.g. by datastore.save_logs to identify paths + # this is used e.g. by mflog/save_logs.py to identify paths class Path(object): def __init__(self, path): @@ -63,7 +63,6 @@ def namedtuple_with_defaults(typename, field_names, defaults=()): T.__new__.__defaults__ = tuple(prototype) return T - class TempDir(object): # Provide a temporary directory since Python 2.7 does not have it inbuilt def __enter__(self): @@ -182,11 +181,10 @@ def resolve_identity(): def get_latest_run_id(echo, flow_name): - from metaflow.datastore.local import LocalDataStore - local_root = LocalDataStore.datastore_root + from metaflow.datastore.local_storage import LocalStorage + local_root = LocalStorage.datastore_root if local_root is None: - v = LocalDataStore.get_datastore_root_from_config(echo, create_on_absent=False) - LocalDataStore.datastore_root = local_root = v + local_root = LocalStorage.get_datastore_root_from_config(echo, create_on_absent=False) if local_root: path = os.path.join(local_root, flow_name, 'latest_run') if os.path.exists(path): @@ -196,15 +194,15 @@ def get_latest_run_id(echo, flow_name): def write_latest_run_id(obj, run_id): - from metaflow.datastore.local import LocalDataStore - if LocalDataStore.datastore_root is None: - LocalDataStore.datastore_root = LocalDataStore.get_datastore_root_from_config(obj.echo) - path = os.path.join(LocalDataStore.datastore_root, obj.flow.name) + from metaflow.datastore.local_storage import LocalStorage + if LocalStorage.datastore_root is None: + LocalStorage.datastore_root = LocalStorage.get_datastore_root_from_config(obj.echo) + path = LocalStorage.path_join(LocalStorage.datastore_root, obj.flow.name) try: os.makedirs(path) except OSError as x: if x.errno != 17: - # Directories exists in other casewhich is fine + # Directories exists in other case which is fine raise with open(os.path.join(path, 'latest_run'), 'w') as f: f.write(str(run_id)) @@ -314,7 +312,7 @@ def dict_to_cli_options(params): # Of the value starts with $, assume the caller wants shell variable # expansion to happen, so we pass it as is. - # NOTE: We strip '\' to allow for various backends to use escaped + # NOTE: We strip '\' to allow for various storages to use escaped # shell variables as well. if value.lstrip("\\").startswith("$"): yield value From 5ece6adc712e7a4c850f4822d29077bdaa1fe3f1 Mon Sep 17 00:00:00 2001 From: Romain Date: Tue, 28 Sep 2021 21:02:38 -0700 Subject: [PATCH 049/176] Missed attempt_id on attempt_done (#724) --- metaflow/datastore/task_datastore.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metaflow/datastore/task_datastore.py b/metaflow/datastore/task_datastore.py index 880da8048c4..f2d352f099b 100644 --- a/metaflow/datastore/task_datastore.py +++ b/metaflow/datastore/task_datastore.py @@ -462,7 +462,8 @@ def done(self): self._metadata.register_metadata( self._run_id, self._step_name, self._task_id, [MetaDatum(field='attempt-done', value=str(self._attempt), - type='attempt-done', tags=[])]) + type='attempt-done', + tags=['attempt_id:{0}'.format(self._attempt)])]) artifacts = [DataArtifact( name=var, ds_type=self.TYPE, From 0f322705864fc38aa0bc35214a04d7581f0c40f6 Mon Sep 17 00:00:00 2001 From: David Neuzerling Date: Wed, 29 Sep 2021 17:28:46 +1000 Subject: [PATCH 050/176] Retrieve flows from global environment (#723) --- R/R/package.R | 2 +- R/tests/testthat/test-flow.R | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/R/R/package.R b/R/R/package.R index d70a3b1bfc0..413505315d8 100644 --- a/R/R/package.R +++ b/R/R/package.R @@ -22,5 +22,5 @@ set_global_variable <- function(key, val, pos = 1) { #' @export metaflow <- function(cls, ...) { set_global_variable(cls, Flow$new(cls, list(...))) - get(cls) + get(cls, pos = 1) } diff --git a/R/tests/testthat/test-flow.R b/R/tests/testthat/test-flow.R index a3e25a3b82e..ce7e5acd376 100644 --- a/R/tests/testthat/test-flow.R +++ b/R/tests/testthat/test-flow.R @@ -1,5 +1,7 @@ context("test-flow.R") +teardown(if ("sqrt" %in% names(.GlobalEnv)) rm("sqrt", envir = .GlobalEnv)) + test_that("header() formatted correctly", { skip_if_no_metaflow() actual <- header("TestFlow") @@ -88,3 +90,11 @@ test_that("get_functions() works", { ) expect_equal(actual, expected) }) + +test_that("flow names are assigned to global environment", { + expect_false("sqrt" %in% names(.GlobalEnv)) + step(metaflow("sqrt"), step = "start") + expect_true("sqrt" %in% names(.GlobalEnv)) + expect_s3_class(get("sqrt", envir = .GlobalEnv), "Flow") + expect_equal(base::sqrt(4), 2) +}) From 8e379bb656cf4a0ec6935779836761b2243dc308 Mon Sep 17 00:00:00 2001 From: Sakari Ikonen <64256562+saikonen@users.noreply.github.com> Date: Wed, 29 Sep 2021 14:40:25 +0300 Subject: [PATCH 051/176] cast s3_retry_count to integer so it can be used with range() (#727) --- metaflow/metaflow_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py index 4429f63b33a..663a678e194 100644 --- a/metaflow/metaflow_config.py +++ b/metaflow/metaflow_config.py @@ -79,7 +79,7 @@ def from_conf(name, default=None): # This is useful if you want to "fail fast" on S3 operations; use with caution # though as this may increase failures. Note that this is the number of *retries* # so setting it to 0 means each operation will be tried once. -S3_RETRY_COUNT = from_conf('METAFLOW_S3_RETRY_COUNT', 7) +S3_RETRY_COUNT = int(from_conf('METAFLOW_S3_RETRY_COUNT', 7)) ### # Datastore local cache From 18988134f2e4d728d244ad9225ffcc5193a188e8 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 30 Sep 2021 21:13:43 +0200 Subject: [PATCH 052/176] from metaflow.sidecar_messages import Message for line 13 (#718) As done at https://github.com/Netflix/metaflow/blob/master/metaflow/plugins/debug_monitor.py#L5 $ `BUILTINS=assert_equals,assert_exception,inputs,is_resumed,NUM_FOREACH,NUM_LINES,ResumeFromHere,TestRetry` $ `flake8 . --builtins=$BUILTINS --count --select=E9,F63,F7,F82 --show-source --statistics` ``` ./metaflow/metaflow/plugins/debug_logger.py:11:9: F821 undefined name 'Message' # type: (Message) -> None ^ ``` --- metaflow/plugins/debug_logger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metaflow/plugins/debug_logger.py b/metaflow/plugins/debug_logger.py index ba23c34bae7..b1e31fea658 100644 --- a/metaflow/plugins/debug_logger.py +++ b/metaflow/plugins/debug_logger.py @@ -1,5 +1,7 @@ import sys +from metaflow.sidecar_messages import Message + class DebugEventLogger(object): TYPE = 'debugLogger' From 1abc5163beedd9965f5c59d0c92b8d5954b11e52 Mon Sep 17 00:00:00 2001 From: Matias Savela Date: Fri, 1 Oct 2021 09:46:37 +0300 Subject: [PATCH 053/176] Fix two issues with MetaflowCode and get_data (#728) * Fixes two issues with codepackage datastore loading. * Fix filecache.get_data tuple problem with conda. * Update conda_environment.py Co-authored-by: Romain --- metaflow/client/core.py | 6 +++--- metaflow/client/filecache.py | 2 +- metaflow/plugins/conda/conda_environment.py | 14 ++++++++------ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/metaflow/client/core.py b/metaflow/client/core.py index 861b5ea2596..07c1ce903e1 100644 --- a/metaflow/client/core.py +++ b/metaflow/client/core.py @@ -625,9 +625,9 @@ def __init__(self, flow_name, code_package): if filecache is None: filecache = FileCache() - code_obj = BytesIO( - filecache.get_data( - self._ds_type, self._flow_name, self._path, self._sha)) + _, blobdata = filecache.get_data( + self._ds_type, self._flow_name, self._path, self._sha) + code_obj = BytesIO(blobdata) self._tar = tarfile.open(fileobj=code_obj, mode='r:gz') # The JSON module in Python3 deals with Unicode. Tar gives bytes. info_str = self._tar.extractfile('INFO').read().decode('utf-8') diff --git a/metaflow/client/filecache.py b/metaflow/client/filecache.py index 2b8935838a7..8d538b82e9f 100644 --- a/metaflow/client/filecache.py +++ b/metaflow/client/filecache.py @@ -85,7 +85,7 @@ def get_data(self, ds_type, flow_name, location, key): ds_root = ds_cls.get_datastore_root_from_location(location, flow_name) ds = self._get_flow_datastore(ds_type, ds_root, flow_name) - return ds.load_data([key], force_raw=True)[0] + return next(ds.load_data([key], force_raw=True)) def get_artifact_by_location( self, ds_type, location, data_metadata, flow_name, run_id, diff --git a/metaflow/plugins/conda/conda_environment.py b/metaflow/plugins/conda/conda_environment.py index c34c902dc0e..37f0f47661c 100644 --- a/metaflow/plugins/conda/conda_environment.py +++ b/metaflow/plugins/conda/conda_environment.py @@ -3,6 +3,8 @@ import sys import tarfile +from io import BytesIO + from metaflow.metaflow_environment import MetaflowEnvironment from metaflow.exception import MetaflowException from metaflow.mflog import BASH_SAVE_LOGS @@ -109,12 +111,12 @@ def get_client_info(cls, flow_name, metadata): if info is None or env_id is None: return {'type': 'conda'} info = json.loads(info) - with cls._filecache.get_data(info['ds_type'], flow_name, info['location'], info['sha']) as f: - with tarfile.open(fileobj=f, mode='r:gz') as tar: - conda_file = tar.extractfile(CONDA_MAGIC_FILE) - if conda_file is None: - return {'type': 'conda'} - info = json.loads(conda_file.read().decode('utf-8')) + _, blobdata = cls._filecache.get_data(info['ds_type'], flow_name, info['location'], info['sha']) + with tarfile.open(fileobj=BytesIO(blobdata), mode='r:gz') as tar: + conda_file = tar.extractfile(CONDA_MAGIC_FILE) + if conda_file is None: + return {'type': 'conda'} + info = json.loads(conda_file.read().decode('utf-8')) new_info = { 'type': 'conda', 'explicit': info[env_id]['explicit'], From 320b8a9bed1b4a1112aabebcecb17d13aa6dde4f Mon Sep 17 00:00:00 2001 From: Savin Date: Sat, 2 Oct 2021 13:36:54 +0530 Subject: [PATCH 054/176] Fix bug with current.paramter_names (#731) * Fix bug with current.paramter_names * revert * Added test for parameter names. (#732) Co-authored-by: Valay Dave --- metaflow/task.py | 3 +-- test/core/tests/param_names.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 test/core/tests/param_names.py diff --git a/metaflow/task.py b/metaflow/task.py index 4f23793ee2a..b49bf77b0aa 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -67,8 +67,7 @@ def property_setter( setattr(self.flow.__class__, var, property(fget=property_setter)) - if passdown: - vars.append(var) + vars.append(var) if passdown: self.flow._datastore.passdown_partial(parameter_ds, vars) return vars diff --git a/test/core/tests/param_names.py b/test/core/tests/param_names.py new file mode 100644 index 00000000000..9dc1cd5b14c --- /dev/null +++ b/test/core/tests/param_names.py @@ -0,0 +1,13 @@ +from metaflow_test import MetaflowTest, steps + +class ParameterNameTest(MetaflowTest): + PRIORITY = 1 + PARAMETERS = { + "foo":{"default":1} + } + + @steps(0, ['all']) + def step_all(self): + from metaflow import current + assert_equals(len(current.parameter_names),1) + assert_equals(current.parameter_names[0],'foo') \ No newline at end of file From 6c1988182ee6c86cf125c0aa6240e1da43d95456 Mon Sep 17 00:00:00 2001 From: Savin Date: Mon, 4 Oct 2021 21:35:24 +0530 Subject: [PATCH 055/176] Bump actions/setup-python to v2 for tests (#736) * Bump actions/setup-python to v2 for tests --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f71b30b216a..1328701df5a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python 2.x if: matrix.lang == 'Python' - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: '2.x' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 56b3e31c790..15314a7dffc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: - name: Set up Python 2.x if: matrix.lang == 'Python' - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: '2.x' @@ -44,7 +44,7 @@ jobs: if: matrix.lang == 'R' uses: r-lib/actions/setup-r@v1 with: - r-version: '3.6.3' + r-version: '3.6.3' - name: Install R 3.6 system dependencies if: matrix.lang == 'R' && matrix.os == 'ubuntu-latest' From ab8a9c63510933828eec039ffad84e261619cd92 Mon Sep 17 00:00:00 2001 From: Savin Date: Tue, 5 Oct 2021 03:59:08 +0530 Subject: [PATCH 056/176] Mute R tests (#737) * Mute R tests --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1328701df5a..4e5564fc94d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - lang: [Python, R] + lang: [Python] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 15314a7dffc..fc6c13f5898 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - lang: [Python, R] + lang: [Python] steps: - uses: actions/checkout@v2 @@ -53,9 +53,9 @@ jobs: - name: Install R 3.6 Rlang dependencies if: matrix.lang == 'R' run: | - python3 -m pip install . Rscript -e 'install.packages("devtools", repos="https://cloud.r-project.org", Ncpus=8)' Rscript -e 'devtools::install_deps("R", dependencies=TRUE, repos="https://cloud.r-project.org", upgrade="default")' + python3 -m pip install . R CMD INSTALL R Rscript -e 'install.packages(c("data.table", "caret", "glmnet", "Matrix", "rjson"), repos="https://cloud.r-project.org", Ncpus=8)' From 0d3ef3387db83f804f33e7bfcbaa300bcfd9c745 Mon Sep 17 00:00:00 2001 From: Romain Date: Mon, 4 Oct 2021 15:30:50 -0700 Subject: [PATCH 057/176] Minor release - 2.4.0 (#738) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 347b7c3f1cb..0a13b9aca61 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '2.3.6' +version = '2.4.0' setup(name='metaflow', version=version, From 221e539a3697b124cffd92ff0bd8f33e3f8aa9af Mon Sep 17 00:00:00 2001 From: Romain Date: Fri, 8 Oct 2021 03:06:34 -0700 Subject: [PATCH 058/176] Fix case when UBF is 'none' and should be a None value (#743) --- metaflow/cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metaflow/cli.py b/metaflow/cli.py index 07b6b7e06e6..7f48b8117e9 100644 --- a/metaflow/cli.py +++ b/metaflow/cli.py @@ -429,7 +429,9 @@ def step(ctx, clone_only=None, clone_run_id=None, decospecs=None, - ubf_context=None): + ubf_context='none'): + if ubf_context == 'none': + ubf_context = None if opt_namespace is not None: namespace(opt_namespace or None) From c5616227785e83964d662afc794e04b35db9c8b7 Mon Sep 17 00:00:00 2001 From: Savin Date: Tue, 12 Oct 2021 02:05:05 +0530 Subject: [PATCH 059/176] Add Python interpreter to execution PATH for AWS Batch (#735) * Fix bug with current.paramter_names * revert * Add Python interpreter to execution PATH for AWS Batch This is needed to ensure non-pythonic conda dependencies are visible to the user code on AWS Batch * update --- metaflow/plugins/conda/conda_step_decorator.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/metaflow/plugins/conda/conda_step_decorator.py b/metaflow/plugins/conda/conda_step_decorator.py index c73f84c1e3e..146b7c6cd5e 100644 --- a/metaflow/plugins/conda/conda_step_decorator.py +++ b/metaflow/plugins/conda/conda_step_decorator.py @@ -285,6 +285,14 @@ def task_pre_step(self, ubf_context, inputs): if self.is_enabled(ubf_context): + # Add the Python interpreter's parent to the path. This is to + # ensure that any non-pythonic dependencies introduced by the conda + # environment are visible to the user code. + env_path = os.path.dirname(sys.executable) + if os.environ.get('PATH') is not None: + env_path = os.pathsep.join([env_path, os.environ['PATH']]) + os.environ['PATH'] = env_path + meta.register_metadata(run_id, step_name, task_id, [MetaDatum(field='conda_env_id', value=self._env_id(), @@ -305,11 +313,7 @@ def runtime_step_cli(self, if self.addl_paths is not None: addl_paths = os.pathsep.join(self.addl_paths) python_path = os.pathsep.join([addl_paths, python_path]) - env_path = os.path.dirname(self.conda.python(self.env_id)) - if os.environ.get('PATH') is not None: - env_path = os.pathsep.join([env_path, os.environ['PATH']]) - - cli_args.env['PATH'] = env_path + cli_args.env['PYTHONPATH'] = python_path cli_args.env['_METAFLOW_CONDA_ENV'] = self.env_id cli_args.entrypoint[0] = self.conda.python(self.env_id) From 6728a64acaa8bb91d1b9cf3252c926e775cb43dc Mon Sep 17 00:00:00 2001 From: Romain Date: Mon, 11 Oct 2021 13:35:42 -0700 Subject: [PATCH 060/176] Remove the propagation of PYTHONPATH in the Conda environment (#745) Previously, Conda would use the pre-existing PYTHONPATH to add it to its own PYTHONPATH thereby allowing packages outside Conda to override packages within Conda which made running locally and remotely different. This also breaks the Conda reproducibility argument. --- metaflow/plugins/conda/conda_step_decorator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/metaflow/plugins/conda/conda_step_decorator.py b/metaflow/plugins/conda/conda_step_decorator.py index 146b7c6cd5e..cbbbeec6bbb 100644 --- a/metaflow/plugins/conda/conda_step_decorator.py +++ b/metaflow/plugins/conda/conda_step_decorator.py @@ -308,8 +308,6 @@ def runtime_step_cli(self, ubf_context): if self.is_enabled(ubf_context) and 'batch' not in cli_args.commands: python_path = self.metaflow_home - if os.environ.get('PYTHONPATH') is not None: - python_path = os.pathsep.join([os.environ['PYTHONPATH'], python_path]) if self.addl_paths is not None: addl_paths = os.pathsep.join(self.addl_paths) python_path = os.pathsep.join([addl_paths, python_path]) From cdd4783440758bc8a5d05b55d0279ad61317116a Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Wed, 13 Oct 2021 10:49:31 -0700 Subject: [PATCH 061/176] Making `DataStorage.list_content` have consistent Behaviour (#755) * Added adhoc fix for localstorage. * Tweeking fix based on prev implementation * changed extend to append --- metaflow/datastore/local_storage.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/metaflow/datastore/local_storage.py b/metaflow/datastore/local_storage.py index 6b8a21ed196..f4ddfc8cf14 100644 --- a/metaflow/datastore/local_storage.py +++ b/metaflow/datastore/local_storage.py @@ -78,11 +78,18 @@ def list_content(self, paths): if path == self.METADATA_DIR: continue full_path = self.full_uri(path) - results.extend([self.list_content_result( - path=self.path_join(path, f), - is_file=self.is_file( - [self.path_join(path, f)])[0]) for f in os.listdir(full_path) - if f != self.METADATA_DIR]) + try: + for f in os.listdir(full_path): + if f == self.METADATA_DIR: + continue + results.append( + self.list_content_result( + path=self.path_join(path, f), + is_file=self.is_file( + [self.path_join(path, f)])[0]) + ) + except FileNotFoundError as e: + pass return results def save_bytes(self, path_and_bytes_iter, overwrite=False, len_hint=0): From 2ab7ff5009924393276b31f83df9fc522ae9dd17 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 13 Oct 2021 22:36:10 -0700 Subject: [PATCH 062/176] Better cleanup of env escape client/server (#760) --- metaflow/plugins/env_escape/client.py | 6 ++++++ metaflow/plugins/env_escape/client_modules.py | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/metaflow/plugins/env_escape/client.py b/metaflow/plugins/env_escape/client.py index f288936d97b..c6f2307b60e 100644 --- a/metaflow/plugins/env_escape/client.py +++ b/metaflow/plugins/env_escape/client.py @@ -152,6 +152,9 @@ def __init__(self, python_path, max_pickle_version, config_dir): self._aliases = response[FIELD_CONTENT]["aliases"] def __del__(self): + self.cleanup() + + def cleanup(self): # Clean up the server; we drain all messages if any if self._poller is not None: # If we have self._poller, we have self._server_process @@ -166,6 +169,7 @@ def __del__(self): sys.stderr.write(self._server_process.stderr.readline()) sys.stdout.flush() sys.stderr.flush() + self._poller = None if self._server_process is not None: # Attempt to send it a terminate signal and then wait and kill try: @@ -177,8 +181,10 @@ def __del__(self): except: # noqa E722 pass # If there is any issue sending this message, just ignore it self._server_process.kill() + self._server_process = None if self._socket_path is not None and os.path.exists(self._socket_path): os.unlink(self._socket_path) + self._socket_path = None @property def name(self): diff --git a/metaflow/plugins/env_escape/client_modules.py b/metaflow/plugins/env_escape/client_modules.py index 9b591c237fd..ae5858b874b 100644 --- a/metaflow/plugins/env_escape/client_modules.py +++ b/metaflow/plugins/env_escape/client_modules.py @@ -1,3 +1,4 @@ +import atexit import importlib import itertools import pickle @@ -9,6 +10,9 @@ from .override_decorators import LocalException +def _clean_client(client): + client.cleanup() + class _WrappedModule(object): def __init__(self, loader, prefix, exports, exception_classes, client): self._loader = loader @@ -133,7 +137,10 @@ def load_module(self, fullname): # what version the current environment support and take the minimum # of those two max_pickle_version = min(self._max_pickle_version, pickle.HIGHEST_PROTOCOL) + self._client = Client(self._python_path, max_pickle_version, self._config_dir) + atexit.register(_clean_client, self._client) + exports = self._client.get_exports() sys.path.insert(0, self._config_dir) overrides = importlib.import_module("overrides") From 8a54a422a24795639cab02f776f6c03f7d60c855 Mon Sep 17 00:00:00 2001 From: Savin Date: Wed, 13 Oct 2021 22:50:01 -0700 Subject: [PATCH 063/176] Close datastore after task_finished has been invoked (#757) task_finished may have some business with datastore, so we should close the datastore after task_finished has been invoked and not before --- metaflow/task.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metaflow/task.py b/metaflow/task.py index b49bf77b0aa..b98a2c8c0ee 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -496,10 +496,6 @@ def run_step(self, output.save_metadata({'task_end': {}}) output.persist(self.flow) - # this writes a success marker indicating that the - # "transaction" is done - output.done() - # final decorator hook: The task results are now # queryable through the client API / datastore for deco in decorators: @@ -510,6 +506,10 @@ def run_step(self, retry_count, max_user_code_retries) + # this writes a success marker indicating that the + # "transaction" is done + output.done() + # terminate side cars logger.terminate() self.metadata.stop_heartbeat() From 100e1a996a0a04463feae09eaebf52394dd1b927 Mon Sep 17 00:00:00 2001 From: Savin Date: Thu, 14 Oct 2021 13:28:39 -0700 Subject: [PATCH 064/176] Revert "Close datastore after task_finished has been invoked (#757)" (#761) This reverts commit 8a54a422a24795639cab02f776f6c03f7d60c855. --- metaflow/task.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metaflow/task.py b/metaflow/task.py index b98a2c8c0ee..b49bf77b0aa 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -496,6 +496,10 @@ def run_step(self, output.save_metadata({'task_end': {}}) output.persist(self.flow) + # this writes a success marker indicating that the + # "transaction" is done + output.done() + # final decorator hook: The task results are now # queryable through the client API / datastore for deco in decorators: @@ -506,10 +510,6 @@ def run_step(self, retry_count, max_user_code_retries) - # this writes a success marker indicating that the - # "transaction" is done - output.done() - # terminate side cars logger.terminate() self.metadata.stop_heartbeat() From e2aa6a3e922000fbc836db961d4e6df92de32fad Mon Sep 17 00:00:00 2001 From: Savin Date: Thu, 14 Oct 2021 13:43:42 -0700 Subject: [PATCH 065/176] Refactor @batch decorator (#618) * Refactor @resources decorator @resources decorator is shared by all compute related decorators - @batch, @lambda, @k8s, @titus. This patch moves it out of batch_decorator.py so that other decorators can cleanly reference it. * Update __init__.py * Refactor @batch decorator * more change * more changes * more changes * update done marker * update batch * update comments --- metaflow/plugins/aws/aws_utils.py | 51 ++++ metaflow/plugins/aws/batch/batch.py | 2 - metaflow/plugins/aws/batch/batch_cli.py | 4 +- metaflow/plugins/aws/batch/batch_client.py | 101 +------- metaflow/plugins/aws/batch/batch_decorator.py | 225 +++++++++--------- 5 files changed, 163 insertions(+), 220 deletions(-) create mode 100644 metaflow/plugins/aws/aws_utils.py diff --git a/metaflow/plugins/aws/aws_utils.py b/metaflow/plugins/aws/aws_utils.py new file mode 100644 index 00000000000..2ff547c9860 --- /dev/null +++ b/metaflow/plugins/aws/aws_utils.py @@ -0,0 +1,51 @@ +import re + +def get_docker_registry(image_uri): + """ + Explanation: + (.+?(?:[:.].+?)\/)? - [GROUP 0] REGISTRY + .+? - A registry must start with at least one character + (?:[:.].+?)\/ - A registry must have ":" or "." and end with "/" + ? - Make a registry optional + (.*?) - [GROUP 1] REPOSITORY + .*? - Get repository name until separator + (?:[@:])? - SEPARATOR + ?: - Don't capture separator + [@:] - The separator must be either "@" or ":" + ? - The separator is optional + ((?<=[@:]).*)? - [GROUP 2] TAG / DIGEST + (?<=[@:]) - A tag / digest must be preceeded by "@" or ":" + .* - Capture rest of tag / digest + ? - A tag / digest is optional + Examples: + image + - None + - image + - None + example/image + - None + - example/image + - None + example/image:tag + - None + - example/image + - tag + example.domain.com/example/image:tag + - example.domain.com/ + - example/image + - tag + 123.123.123.123:123/example/image:tag + - 123.123.123.123:123/ + - example/image + - tag + example.domain.com/example/image@sha256:45b23dee0 + - example.domain.com/ + - example/image + - sha256:45b23dee0 + """ + + pattern = re.compile(r"^(.+?(?:[:.].+?)\/)?(.*?)(?:[@:])?((?<=[@:]).*)?$") + registry, repository, tag = pattern.match(image_uri).groups() + if registry is not None: + registry = registry.rstrip("/") + return registry \ No newline at end of file diff --git a/metaflow/plugins/aws/batch/batch.py b/metaflow/plugins/aws/batch/batch.py index 3ae13d1e101..4784857a61f 100644 --- a/metaflow/plugins/aws/batch/batch.py +++ b/metaflow/plugins/aws/batch/batch.py @@ -7,7 +7,6 @@ import time import warnings -from requests.exceptions import HTTPError from metaflow.exception import MetaflowException, MetaflowInternalError from metaflow.metaflow_config import BATCH_METADATA_SERVICE_URL, DATATOOLS_S3ROOT, \ DATASTORE_LOCAL_DIR, DATASTORE_SYSROOT_S3, DEFAULT_METADATA, \ @@ -240,7 +239,6 @@ def launch_job( cpu=None, gpu=None, memory=None, - platform=None, run_time_limit=None, shared_memory=None, max_swap=None, diff --git a/metaflow/plugins/aws/batch/batch_cli.py b/metaflow/plugins/aws/batch/batch_cli.py index a45319b1a26..89d8ec9beea 100644 --- a/metaflow/plugins/aws/batch/batch_cli.py +++ b/metaflow/plugins/aws/batch/batch_cli.py @@ -240,7 +240,7 @@ def _sync_metadata(): batch = Batch(ctx.obj.metadata, ctx.obj.environment) try: - with ctx.obj.monitor.measure("metaflow.batch.launch"): + with ctx.obj.monitor.measure("metaflow.aws.batch.launch_job"): batch.launch_job( step_name, step_cli, @@ -274,4 +274,4 @@ def _sync_metadata(): traceback.print_exc() sys.exit(METAFLOW_EXIT_DISALLOW_RETRY) finally: - _sync_metadata() + _sync_metadata() \ No newline at end of file diff --git a/metaflow/plugins/aws/batch/batch_client.py b/metaflow/plugins/aws/batch/batch_client.py index b40997258c2..c3b3e97e479 100644 --- a/metaflow/plugins/aws/batch/batch_client.py +++ b/metaflow/plugins/aws/batch/batch_client.py @@ -489,53 +489,6 @@ def wait_for_running(self): if not self.is_running and not self.is_done: BatchWaiter(self._client).wait_for_running(self.id) - @property - def log_stream_name(self): - return self.info['container'].get('logStreamName') - - def logs(self): - def get_log_stream(job): - log_stream_name = job.log_stream_name - if log_stream_name: - return BatchLogs('/aws/batch/job', log_stream_name, sleep_on_no_data=1) - else: - return None - - log_stream = None - while True: - if self.is_running or self.is_done or self.is_crashed: - log_stream = get_log_stream(self) - break - elif not self.is_done: - self.wait_for_running() - - if log_stream is None: - return - exception = None - for i in range(self.NUM_RETRIES + 1): - try: - check_after_done = 0 - for line in log_stream: - if not line: - if self.is_done: - if check_after_done > 1: - return - check_after_done += 1 - else: - pass - else: - i = 0 - yield line - return - except Exception as ex: - exception = ex - if self.is_crashed: - break - #sys.stderr.write(repr(ex) + '\n') - if i < self.NUM_RETRIES: - time.sleep(2 ** i + random.randint(0, 5)) - raise BatchJobException(repr(exception)) - def kill(self): if not self.is_done: self._client.terminate_job( @@ -592,56 +545,4 @@ def wait_for_running(self, job_id): ) self._waiter.create_waiter_with_client('JobRunning', model, self._client).wait( jobs=[job_id] - ) - -class BatchLogs(object): - def __init__(self, group, stream, pos=0, sleep_on_no_data=0): - from ..aws_client import get_aws_client - self._client = get_aws_client('logs') - self._group = group - self._stream = stream - self._pos = pos - self._sleep_on_no_data = sleep_on_no_data - self._buf = deque() - self._token = None - - def _get_events(self): - try: - if self._token: - response = self._client.get_log_events( - logGroupName=self._group, - logStreamName=self._stream, - startTime=self._pos, - nextToken=self._token, - startFromHead=True, - ) - else: - response = self._client.get_log_events( - logGroupName=self._group, - logStreamName=self._stream, - startTime=self._pos, - startFromHead=True, - ) - self._token = response['nextForwardToken'] - return response['events'] - except self._client.exceptions.ResourceNotFoundException as e: - # The logs might be delayed by a bit, so we can simply try - # again next time. - return [] - - def __iter__(self): - while True: - self._fill_buf() - if len(self._buf) == 0: - yield '' - if self._sleep_on_no_data > 0: - select.poll().poll(self._sleep_on_no_data * 1000) - else: - while self._buf: - yield self._buf.popleft() - - def _fill_buf(self): - events = self._get_events() - for event in events: - self._buf.append(event['message']) - self._pos = event['timestamp'] + ) \ No newline at end of file diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index 40d76e0c699..2ffb77c1144 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -1,26 +1,23 @@ import os import sys import platform -import re -import tarfile import requests +from metaflow import util +from metaflow import R + from metaflow.decorators import StepDecorator -from metaflow.metaflow_config import DATASTORE_LOCAL_DIR from metaflow.plugins import ResourcesDecorator from metaflow.plugins.timeout_decorator import get_run_time_limit_for_task from metaflow.metadata import MetaDatum from metaflow.metadata.util import sync_local_metadata_to_datastore - -from metaflow import util -from metaflow import R - -from .batch import BatchException from metaflow.metaflow_config import ECS_S3_ACCESS_IAM_ROLE, BATCH_JOB_QUEUE, \ BATCH_CONTAINER_IMAGE, BATCH_CONTAINER_REGISTRY, \ - ECS_FARGATE_EXECUTION_ROLE + ECS_FARGATE_EXECUTION_ROLE, DATASTORE_LOCAL_DIR from metaflow.sidecar import SidecarSubProcess +from .batch import BatchException +from ..aws_utils import get_docker_registry class BatchDecorator(StepDecorator): """ @@ -100,20 +97,33 @@ def my_step(self): def __init__(self, attributes=None, statically_defined=False): super(BatchDecorator, self).__init__(attributes, statically_defined) + # If no docker image is explicitly specified, impute a default image. if not self.attributes['image']: + # If metaflow-config specifies a docker image, just use that. if BATCH_CONTAINER_IMAGE: self.attributes['image'] = BATCH_CONTAINER_IMAGE + # If metaflow-config doesn't specify a docker image, assign a + # default docker image. else: + # Metaflow-R has it's own default docker image (rocker family) if R.use_r(): self.attributes['image'] = R.container_image() + # Default to vanilla Python image corresponding to major.minor + # version of the Python interpreter launching the flow. else: - self.attributes['image'] = 'python:%s.%s' % (platform.python_version_tuple()[0], - platform.python_version_tuple()[1]) - if not BatchDecorator._get_registry(self.attributes['image']): + self.attributes['image'] = \ + 'python:%s.%s' % (platform.python_version_tuple()[0], + platform.python_version_tuple()[1]) + # Assign docker registry URL for the image. + if not get_docker_registry(self.attributes['image']): if BATCH_CONTAINER_REGISTRY: - self.attributes['image'] = '%s/%s' % (BATCH_CONTAINER_REGISTRY.rstrip('/'), - self.attributes['image']) + self.attributes['image'] = \ + '%s/%s' % (BATCH_CONTAINER_REGISTRY.rstrip('/'), + self.attributes['image']) + # Refer https://github.com/Netflix/metaflow/blob/master/docs/lifecycle.png + # to understand where these functions are invoked in the lifecycle of a + # Metaflow flow. def step_init(self, flow, graph, @@ -125,6 +135,7 @@ def step_init(self, if flow_datastore.TYPE != 's3': raise BatchException('The *@batch* decorator requires --datastore=s3.') + # Set internal state. self.logger = logger self.environment = environment self.step = step @@ -132,16 +143,25 @@ def step_init(self, for deco in decos: if isinstance(deco, ResourcesDecorator): for k, v in deco.attributes.items(): - # we use the larger of @resources and @batch attributes + # We use the larger of @resources and @batch attributes + # TODO: Fix https://github.com/Netflix/metaflow/issues/467 my_val = self.attributes.get(k) if not (my_val is None and v is None): - self.attributes[k] = str(max(int(my_val or 0), int(v or 0))) + self.attributes[k] = \ + str(max(int(my_val or 0), int(v or 0))) + + # Set run time limit for the AWS Batch job. self.run_time_limit = get_run_time_limit_for_task(decos) if self.run_time_limit < 60: raise BatchException('The timeout for step *{step}* should be at ' - 'least 60 seconds for execution on AWS Batch'.format(step=step)) + 'least 60 seconds for execution on AWS Batch.'.format(step=step)) - def runtime_init(self, flow, graph, package, run_id): + def runtime_init(self, + flow, + graph, + package, + run_id): + # Set some more internal state. self.flow = flow self.graph = graph self.package = package @@ -164,7 +184,8 @@ def runtime_step_cli(self, ubf_context): if retry_count <= max_user_code_retries: # after all attempts to run the user code have failed, we don't need - # Batch anymore. We can execute possible fallback code locally. + # to execute on AWS Batch anymore. We can execute possible fallback + # code locally. cli_args.commands = ['batch', 'step'] cli_args.command_args.append(self.package_sha) cli_args.command_args.append(self.package_url) @@ -185,40 +206,47 @@ def task_pre_step(self, max_retries, ubf_context, inputs): - if metadata.TYPE == 'local': - self.task_datastore = task_datastore - else: - self.task_datastore = None - meta = {} - meta['aws-batch-job-id'] = os.environ['AWS_BATCH_JOB_ID'] - meta['aws-batch-job-attempt'] = os.environ['AWS_BATCH_JOB_ATTEMPT'] - meta['aws-batch-ce-name'] = os.environ['AWS_BATCH_CE_NAME'] - meta['aws-batch-jq-name'] = os.environ['AWS_BATCH_JQ_NAME'] - meta['aws-batch-execution-env'] = os.environ['AWS_EXECUTION_ENV'] + self.metadata = metadata + self.task_datastore = task_datastore - # Capture AWS Logs metadata. This is best effort only since - # only V4 of the metadata uri for the ECS container hosts this - # information and it is quite likely that not all consumers of - # Metaflow would be running the container agent compatible with - # version V4. - # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint.html - try: - logs_meta = requests.get( - url=os.environ['ECS_CONTAINER_METADATA_URI_V4']) \ - .json() \ - .get('LogOptions', {}) - meta['aws-batch-awslogs-group'] = logs_meta.get('awslogs-group') - meta['aws-batch-awslogs-region'] = logs_meta.get('awslogs-region') - meta['aws-batch-awslogs-stream'] = logs_meta.get('awslogs-stream') - except: - pass + # task_pre_step may run locally if fallback is activated for @catch + # decorator. In that scenario, we skip collecting AWS Batch execution + # metadata. A rudimentary way to detect non-local execution is to + # check for the existence of AWS_BATCH_JOB_ID environment variable. - entries = [MetaDatum( - field=k, value=v, type=k, tags=["attempt_id:{0}".format(retry_count)]) - for k, v in meta.items()] - # Register book-keeping metadata for debugging. - metadata.register_metadata(run_id, step_name, task_id, entries) - self._save_logs_sidecar = SidecarSubProcess('save_logs_periodically') + if 'AWS_BATCH_JOB_ID' in os.environ: + meta = {} + meta['aws-batch-job-id'] = os.environ['AWS_BATCH_JOB_ID'] + meta['aws-batch-job-attempt'] = os.environ['AWS_BATCH_JOB_ATTEMPT'] + meta['aws-batch-ce-name'] = os.environ['AWS_BATCH_CE_NAME'] + meta['aws-batch-jq-name'] = os.environ['AWS_BATCH_JQ_NAME'] + meta['aws-batch-execution-env'] = os.environ['AWS_EXECUTION_ENV'] + + + # Capture AWS Logs metadata. This is best effort only since + # only V4 of the metadata uri for the ECS container hosts this + # information and it is quite likely that not all consumers of + # Metaflow would be running the container agent compatible with + # version V4. + # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint.html + try: + logs_meta = requests.get( + url=os.environ['ECS_CONTAINER_METADATA_URI_V4']) \ + .json() \ + .get('LogOptions', {}) + meta['aws-batch-awslogs-group'] = logs_meta.get('awslogs-group') + meta['aws-batch-awslogs-region'] = logs_meta.get('awslogs-region') + meta['aws-batch-awslogs-stream'] = logs_meta.get('awslogs-stream') + except: + pass + + entries = [MetaDatum( + field=k, value=v, type=k, tags=["attempt_id:{0}".format(retry_count)]) + for k, v in meta.items()] + # Register book-keeping metadata for debugging. + metadata.register_metadata(run_id, step_name, task_id, entries) + + self._save_logs_sidecar = SidecarSubProcess('save_logs_periodically') def task_post_step(self, step_name, @@ -226,9 +254,18 @@ def task_post_step(self, graph, retry_count, max_user_code_retries): - if self.task_datastore: - sync_local_metadata_to_datastore(DATASTORE_LOCAL_DIR, - self.task_datastore) + # task_post_step may run locally if fallback is activated for @catch + # decorator. + if 'AWS_BATCH_JOB_ID' in os.environ: + # If `local` metadata is configured, we would need to copy task + # execution metadata from the AWS Batch container to user's + # local file system after the user code has finished execution. + # This happens via datastore as a communication bridge. + if self.metadata.TYPE == 'local': + # Note that the datastore is *always* Amazon S3 (see + # runtime_task_created function). + sync_local_metadata_to_datastore(DATASTORE_LOCAL_DIR, + self.task_datastore) def task_exception(self, exception, @@ -237,9 +274,18 @@ def task_exception(self, graph, retry_count, max_user_code_retries): - if self.task_datastore: - sync_local_metadata_to_datastore(DATASTORE_LOCAL_DIR, - self.task_datastore) + # task_exception may run locally if fallback is activated for @catch + # decorator. + if 'AWS_BATCH_JOB_ID' in os.environ: + # If `local` metadata is configured, we would need to copy task + # execution metadata from the AWS Batch container to user's + # local file system after the user code has finished execution. + # This happens via datastore as a communication bridge. + if self.metadata.TYPE == 'local': + # Note that the datastore is *always* Amazon S3 (see + # runtime_task_created function). + sync_local_metadata_to_datastore(DATASTORE_LOCAL_DIR, + self.task_datastore) def task_finished(self, step_name, @@ -248,67 +294,14 @@ def task_finished(self, is_task_ok, retry_count, max_retries): - try: - self._save_logs_sidecar.kill() - except: - pass + try: + self._save_logs_sidecar.kill() + except: + # Best effort kill + pass @classmethod def _save_package_once(cls, flow_datastore, package): if cls.package_url is None: cls.package_url, cls.package_sha = flow_datastore.save_data( - [package.blob], len_hint=1)[0] - - @classmethod - def _get_registry(cls, image): - """ - Explanation: - - (.+?(?:[:.].+?)\/)? - [GROUP 0] REGISTRY - .+? - A registry must start with at least one character - (?:[:.].+?)\/ - A registry must have ":" or "." and end with "/" - ? - Make a registry optional - (.*?) - [GROUP 1] REPOSITORY - .*? - Get repository name until separator - (?:[@:])? - SEPARATOR - ?: - Don't capture separator - [@:] - The separator must be either "@" or ":" - ? - The separator is optional - ((?<=[@:]).*)? - [GROUP 2] TAG / DIGEST - (?<=[@:]) - A tag / digest must be preceded by "@" or ":" - .* - Capture rest of tag / digest - ? - A tag / digest is optional - - Examples: - - image - - None - - image - - None - example/image - - None - - example/image - - None - example/image:tag - - None - - example/image - - tag - example.domain.com/example/image:tag - - example.domain.com/ - - example/image - - tag - 123.123.123.123:123/example/image:tag - - 123.123.123.123:123/ - - example/image - - tag - example.domain.com/example/image@sha256:45b23dee0 - - example.domain.com/ - - example/image - - sha256:45b23dee0 - """ - - pattern = re.compile(r"^(.+?(?:[:.].+?)\/)?(.*?)(?:[@:])?((?<=[@:]).*)?$") - registry, repository, tag = pattern.match(image).groups() - if registry is not None: - registry = registry.rstrip("/") - return registry \ No newline at end of file + [package.blob], len_hint=1)[0] \ No newline at end of file From f7d54d697c979000a0bc5bb57f09fa254804d656 Mon Sep 17 00:00:00 2001 From: Savin Date: Fri, 15 Oct 2021 15:51:10 -0700 Subject: [PATCH 066/176] Kubernetes support for Metaflow (#644) * Refactor @resources decorator @resources decorator is shared by all compute related decorators - @batch, @lambda, @k8s, @titus. This patch moves it out of batch_decorator.py so that other decorators can cleanly reference it. * Update __init__.py * Refactor @batch decorator * more change * more changes * more changes * @kubernetes * Kubernetes * More changes * More changes * more changes * some more changes * more changes * add disk space * Add todos * some fixes * add k8s testing context * more changes * some more changes * minor fixups * better error handling for evicted pods (#711) * fixes for pod/job metadata race conditions (#704) * K8S: label value sanitizer (#719) * rename name_space to namespace for k8s plugin (#750) * fix k8s attribute handling bug (#753) * tweak k8s test resources (to run on kind) (#754) * add k8s api retries (#756) * update done marker * Use linux binaries in @conda when run in k8s (#758) Conda environment should pack linux python binary when run on MacOS to avoid an error metaflow_PlayListFlow_osx-64_179c56284704ca8e53622f848a3df27cdd1f4327/bin/python: cannot execute binary file: Exec format error * fix comment * fix merge conflict * update char Co-authored-by: Oleg Avdeev Co-authored-by: Roman Kindruk <36699371+sappier@users.noreply.github.com> --- metaflow/plugins/__init__.py | 4 + metaflow/plugins/aws/batch/batch.py | 243 +++--- metaflow/plugins/aws/batch/batch_cli.py | 119 +-- metaflow/plugins/aws/batch/batch_client.py | 56 -- metaflow/plugins/aws/eks/__init__.py | 0 metaflow/plugins/aws/eks/kubernetes.py | 405 ++++++++++ metaflow/plugins/aws/eks/kubernetes_cli.py | 234 ++++++ metaflow/plugins/aws/eks/kubernetes_client.py | 716 ++++++++++++++++++ .../plugins/aws/eks/kubernetes_decorator.py | 291 +++++++ .../plugins/conda/conda_step_decorator.py | 8 +- metaflow/task.py | 8 +- test/core/contexts.json | 34 + test/unit/test_k8s_job_name_sanitizer.py | 26 + test/unit/test_k8s_label_sanitizer.py | 28 + 14 files changed, 1954 insertions(+), 218 deletions(-) create mode 100644 metaflow/plugins/aws/eks/__init__.py create mode 100644 metaflow/plugins/aws/eks/kubernetes.py create mode 100644 metaflow/plugins/aws/eks/kubernetes_cli.py create mode 100644 metaflow/plugins/aws/eks/kubernetes_client.py create mode 100644 metaflow/plugins/aws/eks/kubernetes_decorator.py create mode 100644 test/unit/test_k8s_job_name_sanitizer.py create mode 100644 test/unit/test_k8s_label_sanitizer.py diff --git a/metaflow/plugins/__init__.py b/metaflow/plugins/__init__.py index 5c62fc43d01..00352a69b88 100644 --- a/metaflow/plugins/__init__.py +++ b/metaflow/plugins/__init__.py @@ -88,11 +88,13 @@ def get_plugin_cli(): # Add new CLI commands in this list from . import package_cli from .aws.batch import batch_cli + from .aws.eks import kubernetes_cli from .aws.step_functions import step_functions_cli return _ext_plugins.get_plugin_cli() + [ package_cli.cli, batch_cli.cli, + kubernetes_cli.cli, step_functions_cli.cli] @@ -113,6 +115,7 @@ def _merge_lists(base, overrides, attr): from .retry_decorator import RetryDecorator from .resources_decorator import ResourcesDecorator from .aws.batch.batch_decorator import BatchDecorator +from .aws.eks.kubernetes_decorator import KubernetesDecorator from .aws.step_functions.step_functions_decorator \ import StepFunctionsInternalDecorator from .test_unbounded_foreach_decorator\ @@ -125,6 +128,7 @@ def _merge_lists(base, overrides, attr): ResourcesDecorator, RetryDecorator, BatchDecorator, + KubernetesDecorator, StepFunctionsInternalDecorator, CondaStepDecorator, InternalTestUnboundedForeachDecorator], diff --git a/metaflow/plugins/aws/batch/batch.py b/metaflow/plugins/aws/batch/batch.py index 4784857a61f..07eac64e259 100644 --- a/metaflow/plugins/aws/batch/batch.py +++ b/metaflow/plugins/aws/batch/batch.py @@ -1,40 +1,46 @@ -import os -import time +import atexit import json +import os import select -import atexit import shlex import time -import warnings -from metaflow.exception import MetaflowException, MetaflowInternalError -from metaflow.metaflow_config import BATCH_METADATA_SERVICE_URL, DATATOOLS_S3ROOT, \ - DATASTORE_LOCAL_DIR, DATASTORE_SYSROOT_S3, DEFAULT_METADATA, \ - BATCH_METADATA_SERVICE_HEADERS, BATCH_EMIT_TAGS from metaflow import util - -from .batch_client import BatchClient - from metaflow.datatools.s3tail import S3Tail +from metaflow.exception import MetaflowException, MetaflowInternalError +from metaflow.metaflow_config import ( + BATCH_METADATA_SERVICE_URL, + DATATOOLS_S3ROOT, + DATASTORE_LOCAL_DIR, + DATASTORE_SYSROOT_S3, + DEFAULT_METADATA, + BATCH_METADATA_SERVICE_HEADERS, + BATCH_EMIT_TAGS +) from metaflow.mflog.mflog import refine, set_should_persist -from metaflow.mflog import export_mflog_env_vars,\ - bash_capture_logs,\ - update_delay,\ - BASH_SAVE_LOGS +from metaflow.mflog import ( + export_mflog_env_vars, + bash_capture_logs, + update_delay, + BASH_SAVE_LOGS, +) + +from .batch_client import BatchClient # Redirect structured logs to /logs/ -LOGS_DIR = '/logs' -STDOUT_FILE = 'mflog_stdout' -STDERR_FILE = 'mflog_stderr' +LOGS_DIR = "/logs" +STDOUT_FILE = "mflog_stdout" +STDERR_FILE = "mflog_stderr" STDOUT_PATH = os.path.join(LOGS_DIR, STDOUT_FILE) STDERR_PATH = os.path.join(LOGS_DIR, STDERR_FILE) + class BatchException(MetaflowException): - headline = 'AWS Batch error' + headline = "AWS Batch error" class BatchKilledException(MetaflowException): - headline = 'AWS Batch task killed' + headline = "AWS Batch task killed" class Batch(object): @@ -42,22 +48,24 @@ def __init__(self, metadata, environment): self.metadata = metadata self.environment = environment self._client = BatchClient() - atexit.register(lambda: self.job.kill() if hasattr(self, 'job') else None) + atexit.register( + lambda: self.job.kill() if hasattr(self, "job") else None + ) - def _command(self, - environment, - code_package_url, - step_name, - step_cmds, - task_spec): - mflog_expr = export_mflog_env_vars(datastore_type='s3', - stdout_path=STDOUT_PATH, - stderr_path=STDERR_PATH, - **task_spec) + def _command( + self, environment, code_package_url, step_name, step_cmds, task_spec + ): + mflog_expr = export_mflog_env_vars( + datastore_type="s3", + stdout_path=STDOUT_PATH, + stderr_path=STDERR_PATH, + **task_spec + ) init_cmds = environment.get_package_commands(code_package_url) - init_expr = ' && '.join(init_cmds) - step_expr = bash_capture_logs(' && '.join( - environment.bootstrap_commands(step_name) + step_cmds)) + init_expr = " && ".join(init_cmds) + step_expr = bash_capture_logs( + " && ".join(environment.bootstrap_commands(step_name) + step_cmds) + ) # construct an entry point that # 1) initializes the mflog environment (mflog_expr) @@ -67,47 +75,52 @@ def _command(self, # the `true` command is to make sure that the generated command # plays well with docker containers which have entrypoint set as # eval $@ - cmd_str = 'true && mkdir -p /logs && %s && %s && %s; ' % \ - (mflog_expr, init_expr, step_expr) + cmd_str = "true && mkdir -p /logs && %s && %s && %s; " % ( + mflog_expr, + init_expr, + step_expr, + ) # after the task has finished, we save its exit code (fail/success) # and persist the final logs. The whole entrypoint should exit # with the exit code (c) of the task. # # Note that if step_expr OOMs, this tail expression is never executed. - # We lose the last logs in this scenario (although they are visible + # We lose the last logs in this scenario (although they are visible # still through AWS CloudWatch console). - cmd_str += 'c=$?; %s; exit $c' % BASH_SAVE_LOGS - return shlex.split('bash -c \"%s\"' % cmd_str) + cmd_str += "c=$?; %s; exit $c" % BASH_SAVE_LOGS + return shlex.split('bash -c "%s"' % cmd_str) def _search_jobs(self, flow_name, run_id, user): if user is None: - regex = '-{flow_name}-'.format(flow_name=flow_name) + regex = "-{flow_name}-".format(flow_name=flow_name) else: - regex = '{user}-{flow_name}-'.format( - user=user, flow_name=flow_name - ) + regex = "{user}-{flow_name}-".format(user=user, flow_name=flow_name) jobs = [] for job in self._client.unfinished_jobs(): - if regex in job['jobName']: - jobs.append(job['jobId']) + if regex in job["jobName"]: + jobs.append(job["jobId"]) if run_id is not None: - run_id = run_id[run_id.startswith('sfn-') and len('sfn-'):] + run_id = run_id[run_id.startswith("sfn-") and len("sfn-") :] for job in self._client.describe_jobs(jobs): - parameters = job['parameters'] - match = (user is None or parameters['metaflow.user'] == user) and \ - (parameters['metaflow.flow_name'] == flow_name) and \ - (run_id is None or parameters['metaflow.run_id'] == run_id) + parameters = job["parameters"] + match = ( + (user is None or parameters["metaflow.user"] == user) + and (parameters["metaflow.flow_name"] == flow_name) + and (run_id is None or parameters["metaflow.run_id"] == run_id) + ) if match: yield job - def _job_name(self, user, flow_name, run_id, step_name, task_id, retry_count): - return '{user}-{flow_name}-{run_id}-{step_name}-{task_id}-{retry_count}'.format( + def _job_name( + self, user, flow_name, run_id, step_name, task_id, retry_count + ): + return "{user}-{flow_name}-{run_id}-{step_name}-{task_id}-{retry_count}".format( user=user, flow_name=flow_name, - run_id=str(run_id) if run_id is not None else '', + run_id=str(run_id) if run_id is not None else "", step_name=step_name, - task_id=str(task_id) if task_id is not None else '', - retry_count=str(retry_count) if retry_count is not None else '' + task_id=str(task_id) if task_id is not None else "", + retry_count=str(retry_count) if retry_count is not None else "", ) def list_jobs(self, flow_name, run_id, user, echo): @@ -116,12 +129,12 @@ def list_jobs(self, flow_name, run_id, user, echo): for job in jobs: found = True echo( - '{name} [{id}] ({status})'.format( - name=job['jobName'], id=job['jobId'], status=job['status'] + "{name} [{id}] ({status})".format( + name=job["jobName"], id=job["jobId"], status=job["status"] ) ) if not found: - echo('No running AWS Batch jobs found.') + echo("No running AWS Batch jobs found.") def kill_jobs(self, flow_name, run_id, user, echo): jobs = self._search_jobs(flow_name, run_id, user) @@ -129,19 +142,21 @@ def kill_jobs(self, flow_name, run_id, user, echo): for job in jobs: found = True try: - self._client.attach_job(job['jobId']).kill() + self._client.attach_job(job["jobId"]).kill() echo( - 'Killing AWS Batch job: {name} [{id}] ({status})'.format( - name=job['jobName'], id=job['jobId'], status=job['status'] + "Killing AWS Batch job: {name} [{id}] ({status})".format( + name=job["jobName"], + id=job["jobId"], + status=job["status"], ) ) except Exception as e: echo( - 'Failed to terminate AWS Batch job %s [%s]' - % (job['jobId'], repr(e)) + "Failed to terminate AWS Batch job %s [%s]" + % (job["jobId"], repr(e)) ) if not found: - echo('No running AWS Batch jobs found.') + echo("No running AWS Batch jobs found.") def create_job( self, @@ -167,17 +182,17 @@ def create_job( host_volumes=None, ): job_name = self._job_name( - attrs.get('metaflow.user'), - attrs.get('metaflow.flow_name'), - attrs.get('metaflow.run_id'), - attrs.get('metaflow.step_name'), - attrs.get('metaflow.task_id'), - attrs.get('metaflow.retry_count') + attrs.get("metaflow.user"), + attrs.get("metaflow.flow_name"), + attrs.get("metaflow.run_id"), + attrs.get("metaflow.step_name"), + attrs.get("metaflow.task_id"), + attrs.get("metaflow.retry_count"), ) - job = self._client.job() - job \ - .job_name(job_name) \ - .job_queue(queue) \ + job = ( + self._client.job() + .job_name(job_name) + .job_queue(queue) .command( self._command(self.environment, code_package_url, step_name, [step_cli], task_spec)) \ @@ -204,7 +219,7 @@ def create_job( .environment_variable('METAFLOW_DATASTORE_SYSROOT_S3', DATASTORE_SYSROOT_S3) \ .environment_variable('METAFLOW_DATATOOLS_S3ROOT', DATATOOLS_S3ROOT) \ .environment_variable('METAFLOW_DEFAULT_DATASTORE', 's3') \ - .environment_variable('METAFLOW_DEFAULT_METADATA', DEFAULT_METADATA) + .environment_variable('METAFLOW_DEFAULT_METADATA', DEFAULT_METADATA)) # Skip setting METAFLOW_DATASTORE_SYSROOT_LOCAL because metadata sync between the local user # instance and the remote AWS Batch instance assumes metadata is stored in DATASTORE_LOCAL_DIR # on the remote AWS Batch instance; this happens when METAFLOW_DATASTORE_SYSROOT_LOCAL @@ -235,7 +250,7 @@ def launch_job( image, queue, iam_role=None, - execution_role=None, # for FARGATE compatibility + execution_role=None, # for FARGATE compatibility cpu=None, gpu=None, memory=None, @@ -246,13 +261,13 @@ def launch_job( host_volumes=None, env={}, attrs={}, - ): + ): if queue is None: queue = next(self._client.active_job_queues(), None) if queue is None: raise BatchException( - 'Unable to launch AWS Batch job. No job queue ' - ' specified and no valid & enabled queue found.' + "Unable to launch AWS Batch job. No job queue " + " specified and no valid & enabled queue found." ) job = self.create_job( step_name, @@ -279,28 +294,29 @@ def launch_job( self.job = job.execute() def wait(self, stdout_location, stderr_location, echo=None): - def wait_for_launch(job): status = job.status - echo('Task is starting (status %s)...' % status, - 'stderr', - batch_id=job.id) + echo( + "Task is starting (status %s)..." % status, + "stderr", + batch_id=job.id, + ) t = time.time() while True: - if status != job.status or (time.time()-t) > 30: + if status != job.status or (time.time() - t) > 30: status = job.status echo( - 'Task is starting (status %s)...' % status, - 'stderr', - batch_id=job.id + "Task is starting (status %s)..." % status, + "stderr", + batch_id=job.id, ) t = time.time() if job.is_running or job.is_done or job.is_crashed: break select.poll().poll(200) - prefix = b'[%s] ' % util.to_bytes(self.job.id) - + prefix = b"[%s] " % util.to_bytes(self.job.id) + def _print_available(tail, stream, should_persist=False): # print the latest batch of lines from S3Tail try: @@ -309,11 +325,14 @@ def _print_available(tail, stream, should_persist=False): line = set_should_persist(line) else: line = refine(line, prefix=prefix) - echo(line.strip().decode('utf-8', errors='replace'), stream) + echo(line.strip().decode("utf-8", errors="replace"), stream) except Exception as ex: - echo('[ temporary error in fetching logs: %s ]' % ex, - 'stderr', - batch_id=self.job.id) + echo( + "[ temporary error in fetching logs: %s ]" % ex, + "stderr", + batch_id=self.job.id, + ) + stdout_tail = S3Tail(stdout_location) stderr_tail = S3Tail(stderr_location) @@ -328,8 +347,8 @@ def _print_available(tail, stream, should_persist=False): while is_running: if time.time() > next_log_update: - _print_available(stdout_tail, 'stdout') - _print_available(stderr_tail, 'stderr') + _print_available(stdout_tail, "stdout") + _print_available(stderr_tail, "stderr") now = time.time() log_update_delay = update_delay(now - start_time) next_log_update = now + log_update_delay @@ -340,7 +359,7 @@ def _print_available(tail, stream, should_persist=False): # a long delay, regardless of the log tailing schedule d = min(log_update_delay, 5.0) select.poll().poll(d * 1000) - + # 3) Fetch remaining logs # # It is possible that we exit the loop above before all logs have been @@ -349,29 +368,33 @@ def _print_available(tail, stream, should_persist=False): # TODO if we notice AWS Batch failing to upload logs to S3, we can add a # HEAD request here to ensure that the file exists prior to calling # S3Tail and note the user about truncated logs if it doesn't - _print_available(stdout_tail, 'stdout') - _print_available(stderr_tail, 'stderr') + _print_available(stdout_tail, "stdout") + _print_available(stderr_tail, "stderr") # In case of hard crashes (OOM), the final save_logs won't happen. - # We fetch the remaining logs from AWS CloudWatch and persist them to + # We fetch the remaining logs from AWS CloudWatch and persist them to # Amazon S3. - # - # TODO: AWS CloudWatch fetch logs if self.job.is_crashed: - msg = next(msg for msg in - [self.job.reason, self.job.status_reason, 'Task crashed.'] - if msg is not None) + msg = next( + msg + for msg in [ + self.job.reason, + self.job.status_reason, + "Task crashed.", + ] + if msg is not None + ) raise BatchException( - '%s ' - 'This could be a transient error. ' - 'Use @retry to retry.' % msg + "%s " + "This could be a transient error. " + "Use @retry to retry." % msg ) else: if self.job.is_running: # Kill the job if it is still running by throwing an exception. raise BatchException("Task failed!") echo( - 'Task finished with exit code %s.' % self.job.status_code, - 'stderr', - batch_id=self.job.id + "Task finished with exit code %s." % self.job.status_code, + "stderr", + batch_id=self.job.id, ) diff --git a/metaflow/plugins/aws/batch/batch_cli.py b/metaflow/plugins/aws/batch/batch_cli.py index 89d8ec9beea..b5640874a0f 100644 --- a/metaflow/plugins/aws/batch/batch_cli.py +++ b/metaflow/plugins/aws/batch/batch_cli.py @@ -43,18 +43,28 @@ def _execute_cmd(func, flow_name, run_id, user, my_runs, echo): if not run_id and latest_run: run_id = util.get_latest_run_id(echo, flow_name) if run_id is None: - raise CommandException("A previous run id was not found. Specify --run-id.") + raise CommandException( + "A previous run id was not found. Specify --run-id." + ) func(flow_name, run_id, user, echo) @batch.command(help="List unfinished AWS Batch tasks of this flow") -@click.option("--my-runs", default=False, is_flag=True, - help="List all my unfinished tasks.") -@click.option("--user", default=None, - help="List unfinished tasks for the given user.") -@click.option("--run-id", default=None, - help="List unfinished tasks corresponding to the run id.") +@click.option( + "--my-runs", + default=False, + is_flag=True, + help="List all my unfinished tasks.", +) +@click.option( + "--user", default=None, help="List unfinished tasks for the given user." +) +@click.option( + "--run-id", + default=None, + help="List unfinished tasks corresponding to the run id.", +) @click.pass_context def list(ctx, run_id, user, my_runs): batch = Batch(ctx.obj.metadata, ctx.obj.environment) @@ -64,12 +74,22 @@ def list(ctx, run_id, user, my_runs): @batch.command(help="Terminate unfinished AWS Batch tasks of this flow.") -@click.option("--my-runs", default=False, is_flag=True, - help="Kill all my unfinished tasks.") -@click.option("--user", default=None, - help="Terminate unfinished tasks for the given user.") -@click.option("--run-id", default=None, - help="Terminate unfinished tasks corresponding to the run id.") +@click.option( + "--my-runs", + default=False, + is_flag=True, + help="Kill all my unfinished tasks.", +) +@click.option( + "--user", + default=None, + help="Terminate unfinished tasks for the given user.", +) +@click.option( + "--run-id", + default=None, + help="Terminate unfinished tasks corresponding to the run id.", +) @click.pass_context def kill(ctx, run_id, user, my_runs): batch = Batch(ctx.obj.metadata, ctx.obj.environment) @@ -79,24 +99,23 @@ def kill(ctx, run_id, user, my_runs): @batch.command( - help="Execute a single task using AWS Batch. This command " - "calls the top-level step command inside a AWS Batch " - "job with the given options. Typically you do not " - "call this command directly; it is used internally " - "by Metaflow." + help="Execute a single task using AWS Batch. This command calls the " + "top-level step command inside a AWS Batch job with the given options. " + "Typically you do not call this command directly; it is used internally by " + "Metaflow." ) @click.argument("step-name") @click.argument("code-package-sha") @click.argument("code-package-url") @click.option("--executable", help="Executable requirement for AWS Batch.") @click.option( - "--image", help="Docker image requirement for AWS Batch. In name:version format." -) -@click.option( - "--iam-role", help="IAM role requirement for AWS Batch." + "--image", + help="Docker image requirement for AWS Batch. In name:version format.", ) +@click.option("--iam-role", help="IAM role requirement for AWS Batch.") @click.option( - "--execution-role", help="Execution role requirement for AWS Batch on Fargate." + "--execution-role", + help="Execution role requirement for AWS Batch on Fargate.", ) @click.option("--cpu", help="CPU requirement for AWS Batch.") @click.option("--gpu", help="GPU requirement for AWS Batch.") @@ -111,17 +130,23 @@ def kill(ctx, run_id, user, my_runs): @click.option( "--tag", multiple=True, default=None, help="Passed to the top-level 'step'." ) -@click.option("--namespace", default=None, help="Passed to the top-level 'step'.") -@click.option("--retry-count", default=0, help="Passed to the top-level 'step'.") +@click.option( + "--namespace", default=None, help="Passed to the top-level 'step'." +) +@click.option( + "--retry-count", default=0, help="Passed to the top-level 'step'." +) @click.option( "--max-user-code-retries", default=0, help="Passed to the top-level 'step'." ) @click.option( "--run-time-limit", default=5 * 24 * 60 * 60, - help="Run time limit in seconds for the AWS Batch job. " "Default is 5 days." + help="Run time limit in seconds for the AWS Batch job. Default is 5 days.", +) +@click.option( + "--shared-memory", help="Shared Memory requirement for AWS Batch." ) -@click.option("--shared-memory", help="Shared Memory requirement for AWS Batch.") @click.option("--max-swap", help="Max Swap requirement for AWS Batch.") @click.option("--swappiness", help="Swappiness requirement for AWS Batch.") #TODO: Maybe remove it altogether since it's not used here @@ -148,10 +173,10 @@ def step( host_volumes=None, **kwargs ): - def echo(msg, stream='stderr', batch_id=None): + def echo(msg, stream="stderr", batch_id=None): msg = util.to_unicode(msg) if batch_id: - msg = '[%s] %s' % (batch_id, msg) + msg = "[%s] %s" % (batch_id, msg) ctx.obj.echo_always(msg, err=(stream == sys.stderr)) if R.use_r(): @@ -159,8 +184,7 @@ def echo(msg, stream='stderr', batch_id=None): else: if executable is None: executable = ctx.obj.environment.executable(step_name) - entrypoint = '%s -u %s' % (executable, - os.path.basename(sys.argv[0])) + entrypoint = "%s -u %s" % (executable, os.path.basename(sys.argv[0])) top_args = " ".join(util.dict_to_cli_options(ctx.parent.parent.params)) @@ -169,14 +193,18 @@ def echo(msg, stream='stderr', batch_id=None): if input_paths: max_size = 30 * 1024 split_vars = { - "METAFLOW_INPUT_PATHS_%d" % (i // max_size): input_paths[i : i + max_size] + "METAFLOW_INPUT_PATHS_%d" + % (i // max_size): input_paths[i : i + max_size] for i in range(0, len(input_paths), max_size) } kwargs["input_paths"] = "".join("${%s}" % s for s in split_vars.keys()) step_args = " ".join(util.dict_to_cli_options(kwargs)) step_cli = u"{entrypoint} {top_args} step {step} {step_args}".format( - entrypoint=entrypoint, top_args=top_args, step=step_name, step_args=step_args + entrypoint=entrypoint, + top_args=top_args, + step=step_name, + step_args=step_args, ) node = ctx.obj.graph[step_name] @@ -191,17 +219,17 @@ def echo(msg, stream='stderr', batch_id=None): # Set batch attributes task_spec = { - 'flow_name': ctx.obj.flow.name, - 'step_name': step_name, - 'run_id': kwargs['run_id'], - 'task_id': kwargs['task_id'], - 'retry_count': str(retry_count) + "flow_name": ctx.obj.flow.name, + "step_name": step_name, + "run_id": kwargs["run_id"], + "task_id": kwargs["task_id"], + "retry_count": str(retry_count), } - attrs = {'metaflow.%s' % k: v for k, v in task_spec.items()} - attrs['metaflow.user'] = util.get_username() - attrs['metaflow.version'] = ctx.obj.environment.get_environment_info()[ - "metaflow_version" - ] + attrs = {"metaflow.%s" % k: v for k, v in task_spec.items()} + attrs["metaflow.user"] = util.get_username() + attrs["metaflow.version"] = ctx.obj.environment.get_environment_info()[ + "metaflow_version" + ] env_deco = [deco for deco in node.decorators if deco.name == "environment"] if env_deco: @@ -215,7 +243,8 @@ def echo(msg, stream='stderr', batch_id=None): if retry_count: ctx.obj.echo_always( - "Sleeping %d minutes before the next AWS Batch retry" % minutes_between_retries + "Sleeping %d minutes before the next AWS Batch retry" + % minutes_between_retries ) time.sleep(minutes_between_retries * 60) @@ -264,7 +293,7 @@ def _sync_metadata(): host_volumes=host_volumes, ) except Exception as e: - print(e) + traceback.print_exc() _sync_metadata() sys.exit(METAFLOW_EXIT_DISALLOW_RETRY) try: diff --git a/metaflow/plugins/aws/batch/batch_client.py b/metaflow/plugins/aws/batch/batch_client.py index c3b3e97e479..64e988d5c2e 100644 --- a/metaflow/plugins/aws/batch/batch_client.py +++ b/metaflow/plugins/aws/batch/batch_client.py @@ -485,64 +485,8 @@ def status_code(self): self.update() return self.info['container'].get('exitCode') - def wait_for_running(self): - if not self.is_running and not self.is_done: - BatchWaiter(self._client).wait_for_running(self.id) - def kill(self): if not self.is_done: self._client.terminate_job( jobId=self._id, reason='Metaflow initiated job termination.') return self.update() - - -class BatchWaiter(object): - def __init__(self, client): - try: - from botocore import waiter - except: - raise BatchJobException( - 'Could not import module \'botocore\' which ' - 'is required for Batch jobs. Install botocore ' - 'first.' - ) - self._client = client - self._waiter = waiter - - def wait_for_running(self, job_id): - model = self._waiter.WaiterModel( - { - 'version': 2, - 'waiters': { - 'JobRunning': { - 'delay': 1, - 'operation': 'DescribeJobs', - 'description': 'Wait until job starts running', - 'maxAttempts': 1000000, - 'acceptors': [ - { - 'argument': 'jobs[].status', - 'expected': 'SUCCEEDED', - 'matcher': 'pathAll', - 'state': 'success', - }, - { - 'argument': 'jobs[].status', - 'expected': 'FAILED', - 'matcher': 'pathAny', - 'state': 'success', - }, - { - 'argument': 'jobs[].status', - 'expected': 'RUNNING', - 'matcher': 'pathAny', - 'state': 'success', - }, - ], - } - }, - } - ) - self._waiter.create_waiter_with_client('JobRunning', model, self._client).wait( - jobs=[job_id] - ) \ No newline at end of file diff --git a/metaflow/plugins/aws/eks/__init__.py b/metaflow/plugins/aws/eks/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/metaflow/plugins/aws/eks/kubernetes.py b/metaflow/plugins/aws/eks/kubernetes.py new file mode 100644 index 00000000000..6592e6aa177 --- /dev/null +++ b/metaflow/plugins/aws/eks/kubernetes.py @@ -0,0 +1,405 @@ +import os +import time +import json +import select +import shlex +import time +import re +import hashlib + +from metaflow import util +from metaflow.datatools.s3tail import S3Tail +from metaflow.exception import MetaflowException, MetaflowInternalError +from metaflow.metaflow_config import ( + BATCH_METADATA_SERVICE_URL, + DATATOOLS_S3ROOT, + DATASTORE_LOCAL_DIR, + DATASTORE_SYSROOT_S3, + DEFAULT_METADATA, + BATCH_METADATA_SERVICE_HEADERS, +) +from metaflow.mflog import ( + export_mflog_env_vars, + bash_capture_logs, + update_delay, + BASH_SAVE_LOGS +) +from metaflow.mflog.mflog import refine, set_should_persist + +from .kubernetes_client import KubernetesClient + +# Redirect structured logs to /logs/ +LOGS_DIR = "/logs" +STDOUT_FILE = "mflog_stdout" +STDERR_FILE = "mflog_stderr" +STDOUT_PATH = os.path.join(LOGS_DIR, STDOUT_FILE) +STDERR_PATH = os.path.join(LOGS_DIR, STDERR_FILE) + + +class KubernetesException(MetaflowException): + headline = "Kubernetes error" + + +class KubernetesKilledException(MetaflowException): + headline = "Kubernetes Batch job killed" + + +def generate_rfc1123_name(flow_name, + run_id, + step_name, + task_id, + attempt +): + """ + Generate RFC 1123 compatible name. Specifically, the format is: + [*[]] + + The generated name consists from a human-readable prefix, derived from + flow/step/task/attempt, and a hash suffux. + """ + long_name = "-".join( + [ + flow_name, + run_id, + step_name, + task_id, + attempt, + ] + ) + hash = hashlib.sha256(long_name.encode('utf-8')).hexdigest() + + if long_name.startswith('_'): + # RFC 1123 names can't start with hyphen so slap an extra prefix on it + sanitized_long_name = 'u' + long_name.replace('_', '-').lower() + else: + sanitized_long_name = long_name.replace('_', '-').lower() + + # the name has to be under 63 chars total + return sanitized_long_name[:57] + '-' + hash[:5] + + +LABEL_VALUE_REGEX = re.compile(r'^[a-zA-Z0-9]([a-zA-Z0-9\-\_\.]{0,61}[a-zA-Z0-9])?$') + + +def sanitize_label_value(val): + # Label sanitization: if the value can be used as is, return it as is. + # If it can't, sanitize and add a suffix based on hash of the original + # value, replace invalid chars and truncate. + # + # The idea here is that even if there are non-allowed chars in the same + # position, this function will likely return distinct values, so you can + # still filter on those. For example, "alice$" and "alice&" will be + # sanitized into different values "alice_b3f201" and "alice_2a6f13". + if val == '' or LABEL_VALUE_REGEX.match(val): + return val + hash = hashlib.sha256(val.encode('utf-8')).hexdigest() + + # Replace invalid chars with dots, and if the first char is + # non-alphahanumeric, replace it with 'u' to make it valid + sanitized_val = re.sub('^[^A-Z0-9a-z]', 'u', re.sub(r"[^A-Za-z0-9.\-_]", "_", val)) + return sanitized_val[:57] + '-' + hash[:5] + + +class Kubernetes(object): + def __init__( + self, + datastore, + metadata, + environment, + flow_name, + run_id, + step_name, + task_id, + attempt, + ): + self._datastore = datastore + self._metadata = metadata + self._environment = environment + + self._flow_name = flow_name + self._run_id = run_id + self._step_name = step_name + self._task_id = task_id + self._attempt = str(attempt) + + def _command( + self, + code_package_url, + step_cmds, + ): + mflog_expr = export_mflog_env_vars( + flow_name=self._flow_name, + run_id=self._run_id, + step_name=self._step_name, + task_id=self._task_id, + retry_count=self._attempt, + datastore_type=self._datastore.TYPE, + stdout_path=STDOUT_PATH, + stderr_path=STDERR_PATH, + ) + init_cmds = self._environment.get_package_commands(code_package_url) + init_expr = " && ".join(init_cmds) + step_expr = bash_capture_logs( + " && ".join( + self._environment.bootstrap_commands(self._step_name) + + step_cmds + ) + ) + + # Construct an entry point that + # 1) initializes the mflog environment (mflog_expr) + # 2) bootstraps a metaflow environment (init_expr) + # 3) executes a task (step_expr) + + # The `true` command is to make sure that the generated command + # plays well with docker containers which have entrypoint set as + # eval $@ + cmd_str = "true && mkdir -p /logs && %s && %s && %s; " % ( + mflog_expr, + init_expr, + step_expr, + ) + # After the task has finished, we save its exit code (fail/success) + # and persist the final logs. The whole entrypoint should exit + # with the exit code (c) of the task. + # + # Note that if step_expr OOMs, this tail expression is never executed. + # We lose the last logs in this scenario. + # + # TODO: Find a way to capture hard exit logs in Kubernetes. + cmd_str += "c=$?; %s; exit $c" % BASH_SAVE_LOGS + return shlex.split('bash -c "%s"' % cmd_str) + + def launch_job(self, **kwargs): + self._job = self.create_job(**kwargs).execute() + + def create_job( + self, + user, + code_package_sha, + code_package_url, + code_package_ds, + step_cli, + docker_image, + service_account=None, + secrets=None, + node_selector=None, + namespace=None, + cpu=None, + gpu=None, + disk=None, + memory=None, + run_time_limit=None, + env={}, + ): + # TODO: Test for DNS-1123 compliance. Python names can have underscores + # which are not valid Kubernetes names. We can potentially make + # the pathspec DNS-1123 compliant by stripping away underscores + # etc. and relying on Kubernetes to attach a suffix to make the + # name unique within a namespace. + # + # Set the pathspec (along with attempt) as the Kubernetes job name. + # Kubernetes job names are supposed to be unique within a Kubernetes + # namespace and compliant with DNS-1123. The pathspec (with attempt) + # can provide that guarantee, however, for flows launched via AWS Step + # Functions (and potentially Argo), we may not get the task_id or the + # attempt_id while submitting the job to the Kubernetes cluster. If + # that is indeed the case, we can rely on Kubernetes to generate a name + # for us. + job_name = generate_rfc1123_name( + self._flow_name, + self._run_id, + self._step_name, + self._task_id, + self._attempt, + ) + + job = ( + KubernetesClient() + .job( + name=job_name, + namespace=namespace, + service_account=service_account, + secrets=secrets, + node_selector=node_selector, + command=self._command( + code_package_url=code_package_url, + step_cmds=[step_cli], + ), + image=docker_image, + cpu=cpu, + memory=memory, + disk=disk, + timeout_in_seconds=run_time_limit, + # Retries are handled by Metaflow runtime + retries=0, + ) + .environment_variable( + # This is needed since `boto3` is not smart enough to figure out + # AWS region by itself. + # TODO: Fix this. + "AWS_DEFAULT_REGION", + "us-west-2", + ) + .environment_variable("METAFLOW_CODE_SHA", code_package_sha) + .environment_variable("METAFLOW_CODE_URL", code_package_url) + .environment_variable("METAFLOW_CODE_DS", code_package_ds) + .environment_variable("METAFLOW_USER", user) + .environment_variable( + "METAFLOW_SERVICE_URL", BATCH_METADATA_SERVICE_URL + ) + .environment_variable( + "METAFLOW_SERVICE_HEADERS", + json.dumps(BATCH_METADATA_SERVICE_HEADERS), + ) + .environment_variable( + "METAFLOW_DATASTORE_SYSROOT_S3", DATASTORE_SYSROOT_S3 + ) + .environment_variable("METAFLOW_DATATOOLS_S3ROOT", DATATOOLS_S3ROOT) + .environment_variable("METAFLOW_DEFAULT_DATASTORE", "s3") + .environment_variable("METAFLOW_DEFAULT_METADATA", DEFAULT_METADATA) + .environment_variable("METAFLOW_KUBERNETES_WORKLOAD", 1) + .label("app", "metaflow") + .label("metaflow/flow_name", sanitize_label_value(self._flow_name)) + .label("metaflow/run_id", sanitize_label_value(self._run_id)) + .label("metaflow/step_name", sanitize_label_value(self._step_name)) + .label("metaflow/task_id", sanitize_label_value(self._task_id)) + .label("metaflow/attempt", sanitize_label_value(self._attempt)) + ) + + # Skip setting METAFLOW_DATASTORE_SYSROOT_LOCAL because metadata sync + # between the local user instance and the remote Kubernetes pod + # assumes metadata is stored in DATASTORE_LOCAL_DIR on the Kubernetes + # pod; this happens when METAFLOW_DATASTORE_SYSROOT_LOCAL is NOT set ( + # see get_datastore_root_from_config in datastore/local.py). + for name, value in env.items(): + job.environment_variable(name, value) + + # Add labels to the Kubernetes job + # + # Apply recommended labels https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/ + # + # TODO: 1. Verify the behavior of high cardinality labels like instance, + # version etc. in the app.kubernetes.io namespace before + # introducing them here. + job.label("app.kubernetes.io/name", "metaflow-task").label( + "app.kubernetes.io/part-of", "metaflow" + ).label("app.kubernetes.io/created-by", sanitize_label_value(user)) + # Add Metaflow system tags as labels as well! + for sys_tag in self._metadata.sticky_sys_tags: + job.label( + "metaflow/%s" % sys_tag[: sys_tag.index(":")], + sanitize_label_value(sys_tag[sys_tag.index(":") + 1 :]) + ) + # TODO: Add annotations based on https://kubernetes.io/blog/2021/04/20/annotating-k8s-for-humans/ + + return job.create() + + def wait(self, stdout_location, stderr_location, echo=None): + + def wait_for_launch(job): + status = job.status + echo( + "Task is starting (Status %s)..." % status, + "stderr", + job_id=job.id, + ) + t = time.time() + while True: + new_status = job.status + if status != new_status or (time.time() - t) > 30: + status = new_status + echo( + "Task is starting (Status %s)..." % status, + "stderr", + job_id=job.id, + ) + t = time.time() + if job.is_running or job.is_done: + break + time.sleep(1) + + def _print_available(tail, stream, should_persist=False): + # print the latest batch of lines from S3Tail + prefix = b"[%s] " % util.to_bytes(self._job.id) + try: + for line in tail: + if should_persist: + line = set_should_persist(line) + else: + line = refine(line, prefix=prefix) + echo(line.strip().decode("utf-8", errors="replace"), stream) + except Exception as ex: + echo( + "[ temporary error in fetching logs: %s ]" % ex, + "stderr", + job_id=self._job.id, + ) + + stdout_tail = S3Tail(stdout_location) + stderr_tail = S3Tail(stderr_location) + + # 1) Loop until the job has started + wait_for_launch(self._job) + + # 2) Loop until the job has finished + start_time = time.time() + is_running = True + next_log_update = start_time + log_update_delay = 1 + + while is_running: + if time.time() > next_log_update: + _print_available(stdout_tail, "stdout") + _print_available(stderr_tail, "stderr") + now = time.time() + log_update_delay = update_delay(now - start_time) + next_log_update = now + log_update_delay + is_running = self._job.is_running + + # This sleep should never delay log updates. On the other hand, + # we should exit this loop when the task has finished without + # a long delay, regardless of the log tailing schedule + time.sleep(min(log_update_delay, 5.0)) + + # 3) Fetch remaining logs + # + # It is possible that we exit the loop above before all logs have been + # shown. + # + # TODO (savin): If we notice Kubernetes failing to upload logs to S3, + # we can add a HEAD request here to ensure that the file + # exists prior to calling S3Tail and note the user about + # truncated logs if it doesn't. + # TODO (savin): For hard crashes, we can fetch logs from the pod. + _print_available(stdout_tail, "stdout") + _print_available(stderr_tail, "stderr") + + if self._job.has_failed: + exit_code, reason = self._job.reason + msg = next( + msg + for msg in [ + reason, + "Task crashed", + ] + if msg is not None + ) + if exit_code: + if int(exit_code) == 139: + raise KubernetesException( + "Task failed with a segmentation fault." + ) + else: + msg = "%s (exit code %s)" % (msg, exit_code) + raise KubernetesException( + "%s. This could be a transient error. " + "Use @retry to retry." % msg + ) + + exit_code, _ = self._job.reason + echo( + "Task finished with exit code %s." % exit_code, + "stderr", + job_id=self._job.id, + ) diff --git a/metaflow/plugins/aws/eks/kubernetes_cli.py b/metaflow/plugins/aws/eks/kubernetes_cli.py new file mode 100644 index 00000000000..32315986b47 --- /dev/null +++ b/metaflow/plugins/aws/eks/kubernetes_cli.py @@ -0,0 +1,234 @@ +import click +import os +import sys +import time +import traceback + +from metaflow import util +from metaflow.exception import CommandException, METAFLOW_EXIT_DISALLOW_RETRY +from metaflow.metadata.util import sync_local_metadata_from_datastore +from metaflow.metaflow_config import DATASTORE_LOCAL_DIR +from metaflow.mflog import TASK_LOG_SOURCE + +from .kubernetes import Kubernetes, KubernetesKilledException + +# TODO(s): +# 1. Compatibility for Metaflow-R (not a blocker for release). +# 2. Add more CLI commands to manage Kubernetes objects. + + +@click.group() +def cli(): + pass + + +@cli.group(help="Commands related to Kubernetes on Amazon EKS.") +def kubernetes(): + pass + + +@kubernetes.command( + help="Execute a single task on Kubernetes using Amazon EKS. This command " + "calls the top-level step command inside a Kubernetes job with the given " + "options. Typically you do not call this command directly; it is used " + "internally by Metaflow." +) +@click.argument("step-name") +@click.argument("code-package-sha") +@click.argument("code-package-url") +@click.option( + "--executable", + help="Executable requirement for Kubernetes job on Amazon EKS.", +) +@click.option( + "--image", help="Docker image requirement for Kubernetes job on Amazon EKS." +) +@click.option( + "--service-account", + help="IRSA requirement for Kubernetes job on Amazon EKS.", +) +@click.option( + "--secrets", + multiple=True, + default=None, + help="Secrets for Kubernetes job on Amazon EKS.", +) +@click.option( + "--node-selector", + multiple=True, + default=None, + help="NodeSelector for Kubernetes job on Amazon EKS.", +) +@click.option( + # Note that ideally we would have liked to use `namespace` rather than + # `k8s-namespace` but unfortunately, `namespace` is already reserved for + # Metaflow namespaces. + "--k8s-namespace", + default=None, + help="Namespace for Kubernetes job on Amazon EKS.", +) +@click.option("--cpu", help="CPU requirement for Kubernetes job on Amazon EKS.") +@click.option("--gpu", help="GPU requirement for Kubernetes job on Amazon EKS.") +@click.option( + "--disk", help="Disk requirement for Kubernetes job on Amazon EKS." +) +@click.option( + "--memory", help="Memory requirement for Kubernetes job on Amazon EKS." +) +@click.option("--run-id", help="Passed to the top-level 'step'.") +@click.option("--task-id", help="Passed to the top-level 'step'.") +@click.option("--input-paths", help="Passed to the top-level 'step'.") +@click.option("--split-index", help="Passed to the top-level 'step'.") +@click.option("--clone-path", help="Passed to the top-level 'step'.") +@click.option("--clone-run-id", help="Passed to the top-level 'step'.") +@click.option( + "--tag", multiple=True, default=None, help="Passed to the top-level 'step'." +) +@click.option( + "--namespace", default=None, help="Passed to the top-level 'step'." +) +@click.option( + "--retry-count", default=0, help="Passed to the top-level 'step'." +) +@click.option( + "--max-user-code-retries", default=0, help="Passed to the top-level 'step'." +) +@click.option( + "--run-time-limit", + default=5 * 24 * 60 * 60, # Default is set to 5 days + help="Run time limit in seconds for Kubernetes job.", +) +@click.pass_context +def step( + ctx, + step_name, + code_package_sha, + code_package_url, + executable=None, + image=None, + service_account=None, + secrets=None, + node_selector=None, + k8s_namespace=None, + cpu=None, + gpu=None, + disk=None, + memory=None, + run_time_limit=None, + **kwargs +): + def echo(msg, stream="stderr", job_id=None): + msg = util.to_unicode(msg) + if job_id: + msg = "[%s] %s" % (job_id, msg) + ctx.obj.echo_always(msg, err=(stream == sys.stderr)) + + node = ctx.obj.graph[step_name] + + # Construct entrypoint CLI + if executable is None: + executable = ctx.obj.environment.executable(step_name) + + # Set environment + env = {} + env_deco = [deco for deco in node.decorators if deco.name == "environment"] + if env_deco: + env = env_deco[0].attributes["vars"] + + # Set input paths. + input_paths = kwargs.get("input_paths") + split_vars = None + if input_paths: + max_size = 30 * 1024 + split_vars = { + "METAFLOW_INPUT_PATHS_%d" + % (i // max_size): input_paths[i : i + max_size] + for i in range(0, len(input_paths), max_size) + } + kwargs["input_paths"] = "".join("${%s}" % s for s in split_vars.keys()) + env.update(split_vars) + + # Set retry policy. + retry_count = int(kwargs.get("retry_count", 0)) + retry_deco = [deco for deco in node.decorators if deco.name == "retry"] + minutes_between_retries = None + if retry_deco: + minutes_between_retries = int( + retry_deco[0].attributes.get("minutes_between_retries", 2) + ) + if retry_count: + ctx.obj.echo_always( + "Sleeping %d minutes before the next retry" + % minutes_between_retries + ) + time.sleep(minutes_between_retries * 60) + + step_cli = u"{entrypoint} {top_args} step {step} {step_args}".format( + entrypoint="%s -u %s" % (executable, os.path.basename(sys.argv[0])), + top_args=" ".join(util.dict_to_cli_options(ctx.parent.parent.params)), + step=step_name, + step_args=" ".join(util.dict_to_cli_options(kwargs)), + ) + + # this information is needed for log tailing + ds = ctx.obj.flow_datastore.get_task_datastore( + mode='w', + run_id=kwargs['run_id'], + step_name=step_name, + task_id=kwargs['task_id'], + attempt=int(retry_count) + ) + stdout_location = ds.get_log_location(TASK_LOG_SOURCE, 'stdout') + stderr_location = ds.get_log_location(TASK_LOG_SOURCE, 'stderr') + + def _sync_metadata(): + if ctx.obj.metadata.TYPE == 'local': + sync_local_metadata_from_datastore( + DATASTORE_LOCAL_DIR, + ctx.obj.flow_datastore.get_task_datastore(kwargs['run_id'], + step_name, + kwargs['task_id'])) + + try: + kubernetes = Kubernetes( + datastore=ctx.obj.flow_datastore, + metadata=ctx.obj.metadata, + environment=ctx.obj.environment, + flow_name=ctx.obj.flow.name, + run_id=kwargs["run_id"], + step_name=step_name, + task_id=kwargs["task_id"], + attempt=retry_count, + ) + # Configure and launch Kubernetes job. + with ctx.obj.monitor.measure("metaflow.aws.eks.launch_job"): + kubernetes.launch_job( + user=util.get_username(), + code_package_sha=code_package_sha, + code_package_url=code_package_url, + code_package_ds=ctx.obj.flow_datastore.TYPE, + step_cli=step_cli, + docker_image=image, + service_account=service_account, + secrets=secrets, + node_selector=node_selector, + namespace=k8s_namespace, + cpu=cpu, + gpu=gpu, + disk=disk, + memory=memory, + run_time_limit=run_time_limit, + env=env, + ) + except Exception as e: + traceback.print_exc() + _sync_metadata() + sys.exit(METAFLOW_EXIT_DISALLOW_RETRY) + try: + kubernetes.wait(stdout_location, stderr_location, echo=echo) + except KubernetesKilledException: + # don't retry killed tasks + traceback.print_exc() + sys.exit(METAFLOW_EXIT_DISALLOW_RETRY) + finally: + _sync_metadata() \ No newline at end of file diff --git a/metaflow/plugins/aws/eks/kubernetes_client.py b/metaflow/plugins/aws/eks/kubernetes_client.py new file mode 100644 index 00000000000..b0c143328c0 --- /dev/null +++ b/metaflow/plugins/aws/eks/kubernetes_client.py @@ -0,0 +1,716 @@ +import os +import time +import math +import random + + +try: + unicode +except NameError: + unicode = str + basestring = str + +from metaflow.exception import MetaflowException + +CLIENT_REFRESH_INTERVAL_SECONDS = 300 + + +class KubernetesJobException(MetaflowException): + headline = "Kubernetes job error" + + +# Implements truncated exponential backoff from https://cloud.google.com/storage/docs/retry-strategy#exponential-backoff +def k8s_retry(deadline_seconds=60, max_backoff=32): + def decorator(function): + from functools import wraps + + @wraps(function) + def wrapper(*args, **kwargs): + from kubernetes import client + + deadline = time.time() + deadline_seconds + retry_number = 0 + + while True: + try: + result = function(*args, **kwargs) + return result + except client.rest.ApiException as e: + if e.status == 500: + current_t = time.time() + backoff_delay = min(math.pow(2, retry_number) + random.random(), max_backoff) + if current_t + backoff_delay < deadline: + time.sleep(backoff_delay) + retry_number += 1 + continue # retry again + else: + raise + else: + raise + + return wrapper + return decorator + + +class KubernetesClient(object): + def __init__(self): + # TODO: Look into removing the usage of Kubernetes Python SDK + # at some point in the future. Given that Kubernetes Python SDK + # aggressively drops support for older kubernetes clusters, continued + # dependency on it may bite our users. + + try: + # Kubernetes is a soft dependency. + from kubernetes import client, config + except (NameError, ImportError): + raise MetaflowException( + "Could not import module 'kubernetes'. Install kubernetes " + "Python package (https://pypi.org/project/kubernetes/) first." + ) + self._refresh_client() + + def _refresh_client(self): + from kubernetes import client, config + + if os.getenv("KUBERNETES_SERVICE_HOST"): + # We are inside a pod, authenticate via ServiceAccount assigned to us + config.load_incluster_config() + else: + # Use kubeconfig, likely $HOME/.kube/config + # TODO (savin): + # 1. Support generating kubeconfig on the fly using boto3 + # 2. Support auth via OIDC - https://docs.aws.amazon.com/eks/latest/userguide/authenticate-oidc-identity-provider.html + # Supporting the above auth mechanisms (atleast 1.) should be + # good enough for the initial rollout. + config.load_kube_config() + self._client = client + self._client_refresh_timestamp = time.time() + + def job(self, **kwargs): + return KubernetesJob(self, **kwargs) + + def get(self): + if ( + time.time() - self._client_refresh_timestamp + > CLIENT_REFRESH_INTERVAL_SECONDS + ): + self._refresh_client() + + return self._client + + +class KubernetesJob(object): + def __init__(self, client_wrapper, **kwargs): + self._client_wrapper = client_wrapper + self._kwargs = kwargs + + # Kubernetes namespace defaults to `default` + self._kwargs["namespace"] = self._kwargs["namespace"] or "default" + + def create(self): + # Check that job attributes are sensible. + + # CPU value should be greater than 0 + if not ( + isinstance(self._kwargs["cpu"], (int, unicode, basestring, float)) + and float(self._kwargs["cpu"]) > 0 + ): + raise KubernetesJobException( + "Invalid CPU value ({}); it should be greater than 0".format( + self._kwargs["cpu"] + ) + ) + + # Memory value should be greater than 0 + if not ( + isinstance(self._kwargs["memory"], (int, unicode, basestring)) + and int(self._kwargs["memory"]) > 0 + ): + raise KubernetesJobException( + "Invalid memory value ({}); it should be greater than 0".format( + self._kwargs["memory"] + ) + ) + + # Disk value should be greater than 0 + if not ( + isinstance(self._kwargs["disk"], (int, unicode, basestring)) + and int(self._kwargs["disk"]) > 0 + ): + raise KubernetesJobException( + "Invalid disk value ({}); it should be greater than 0".format( + self._kwargs["disk"] + ) + ) + + # TODO(s) (savin) + # 1. Add support for GPUs. + + # A discerning eye would notice and question the choice of using the + # V1Job construct over the V1Pod construct given that we don't rely much + # on any of the V1Job semantics. The major reasons at the moment are - + # 1. It makes the Kubernetes UIs (Octant, Lens) a bit more easy on + # the eyes, although even that can be questioned. + # 2. AWS Step Functions, at the moment (Aug' 21) only supports + # executing Jobs and not Pods as part of it's publicly declared + # API. When we ship the AWS Step Functions integration with EKS, + # it will hopefully lessen our workload. + # + # Note: This implementation ensures that there is only one unique Pod + # (unique UID) per Metaflow task attempt. + client = self._client_wrapper.get() + self._job = client.V1Job( + api_version="batch/v1", + kind="Job", + metadata=client.V1ObjectMeta( + # Annotations are for humans + annotations=self._kwargs.get("annotations", {}), + # While labels are for Kubernetes + labels=self._kwargs.get("labels", {}), + name=self._kwargs["name"], # Unique within the namespace + namespace=self._kwargs["namespace"], # Defaults to `default` + ), + spec=client.V1JobSpec( + # Retries are handled by Metaflow when it is responsible for + # executing the flow. The responsibility is moved to Kubernetes + # when AWS Step Functions / Argo are responsible for the + # execution. + backoff_limit=self._kwargs.get("retries", 0), + completions=1, # A single non-indexed pod job + # TODO (savin): Implement a job clean-up option in the + # kubernetes CLI. + ttl_seconds_after_finished=7 + * 60 + * 60 # Remove job after a week. TODO (savin): Make this + * 24, # configurable + template=client.V1PodTemplateSpec( + metadata=client.V1ObjectMeta( + annotations=self._kwargs.get("annotations", {}), + labels=self._kwargs.get("labels", {}), + name=self._kwargs["name"], + namespace=self._kwargs["namespace"], + ), + spec=client.V1PodSpec( + # Timeout is set on the pod and not the job (important!) + active_deadline_seconds=self._kwargs["timeout_in_seconds"], + # TODO (savin): Enable affinities for GPU scheduling. + # This requires some thought around the + # UX since specifying affinities can get + # complicated quickly. We may well decide + # to move it out of scope for the initial + # roll out. + # affinity=?, + containers=[ + client.V1Container( + command=self._kwargs["command"], + env=[ + client.V1EnvVar(name=k, value=str(v)) + for k, v in self._kwargs.get( + "environment_variables", {} + ).items() + ] + # And some downward API magic. Add (key, value) + # pairs below to make pod metadata available + # within Kubernetes container. + # + # TODO: Figure out a way to make job + # metadata visible within the container + + [ + client.V1EnvVar( + name=k, + value_from=client.V1EnvVarSource( + field_ref=client.V1ObjectFieldSelector( + field_path=str(v) + ) + ), + ) + for k, v in { + "METAFLOW_KUBERNETES_POD_NAMESPACE": "metadata.namespace", + "METAFLOW_KUBERNETES_POD_NAME": "metadata.name", + "METAFLOW_KUBERNETES_POD_ID": "metadata.uid", + }.items() + ], + env_from=[ + client.V1EnvFromSource( + secret_ref=client.V1SecretEnvSource(name=str(k)) + ) + for k in self._kwargs.get("secrets", []) + ], + image=self._kwargs["image"], + name=self._kwargs["name"], + resources=client.V1ResourceRequirements( + requests={ + "cpu": str(self._kwargs["cpu"]), + "memory": "%sM" % str(self._kwargs["memory"]), + "ephemeral-storage": "%sM" + % str(self._kwargs["disk"]), + } + ), + ) + ], + node_selector={ + # TODO: What should be the format of node selector - + # key:value or key=value? + str(k.split("=", 1)[0]): str(k.split("=", 1)[1]) + for k in self._kwargs.get("node_selector", []) + }, + # TODO (savin): At some point in the very near future, + # support docker access secrets. + # image_pull_secrets=?, + # + # TODO (savin): We should, someday, get into the pod + # priority business + # preemption_policy=?, + # + # A Container in a Pod may fail for a number of + # reasons, such as because the process in it exited + # with a non-zero exit code, or the Container was + # killed due to OOM etc. If this happens, fail the pod + # and let Metaflow handle the retries. + restart_policy="Never", + service_account_name=self._kwargs["service_account"], + # Terminate the container immediately on SIGTERM + termination_grace_period_seconds=0, + # TODO (savin): Enable tolerations for GPU scheduling. + # This requires some thought around the + # UX since specifying tolerations can get + # complicated quickly. + # tolerations=?, + # + # TODO (savin): At some point in the very near future, + # support custom volumes (PVCs/EVCs). + # volumes=?, + # + # TODO (savin): Set termination_message_policy + ), + ), + ), + ) + return self + + def execute(self): + client = self._client_wrapper.get() + try: + # TODO (savin): Make job submission back-pressure aware. Currently + # there doesn't seem to be a kubernetes-native way to + # achieve the guarantees that we are seeking. + # Hopefully, we will be able to get creative soon. + response = ( + client.BatchV1Api() + .create_namespaced_job( + body=self._job, namespace=self._kwargs["namespace"] + ) + .to_dict() + ) + return RunningJob( + client_wrapper=self._client_wrapper, + name=response["metadata"]["name"], + uid=response["metadata"]["uid"], + namespace=response["metadata"]["namespace"], + ) + except client.rest.ApiException as e: + raise KubernetesJobException( + "Unable to launch Kubernetes job.\n %s" % str(e) + ) + + def namespace(self, namespace): + self._kwargs["namespace"] = namespace + return self + + def name(self, name): + self._kwargs["name"] = name + return self + + def command(self, command): + self._kwargs["command"] = command + return self + + def image(self, image): + self._kwargs["image"] = image + return self + + def cpu(self, cpu): + self._kwargs["cpu"] = cpu + return self + + def memory(self, mem): + self._kwargs["memory"] = mem + return self + + def environment_variable(self, name, value): + self._kwargs["environment_variables"] = dict( + self._kwargs.get("environment_variables", {}), **{name: value} + ) + return self + + def label(self, name, value): + self._kwargs["labels"] = dict(self._kwargs.get("labels", {}), **{name: value}) + return self + + def annotation(self, name, value): + self._kwargs["annotations"] = dict( + self._kwargs.get("annotations", {}), **{name: value} + ) + return self + + +class RunningJob(object): + + # State Machine implementation for the lifecycle behavior documented in + # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/ + # + # This object encapsulates *both* V1Job and V1Pod. It simplifies the status + # to "running" and "done" (failed/succeeded) state. Note that V1Job and V1Pod + # status fields are not guaranteed to be always in sync due to the way job + # controller works. + # + # For example, for a successful job, RunningJob states and their corresponding + # K8S object states look like this: + # + # | V1JobStatus.active | V1JobStatus.succeeded | V1PodStatus.phase | RunningJob.is_running | RunningJob.is_done | + # |--------------------|-----------------------|-------------------|-----------------------|--------------------| + # | 0 | 0 | N/A | False | False | + # | 0 | 0 | Pending | False | False | + # | 1 | 0 | Running | True | False | + # | 1 | 0 | Succeeded | True | True | + # | 0 | 1 | Succeeded | False | True | + + # To ascertain the status of V1Job, we peer into the lifecycle status of + # the pod it is responsible for executing. Unfortunately, the `phase` + # attributes (pending, running, succeeded, failed etc.) only provide + # partial answers and the official API conventions guide suggests that + # it may soon be deprecated (however, not anytime soon - see + # https://github.com/kubernetes/kubernetes/issues/7856). `conditions` otoh + # provide a deeper understanding about the state of the pod; however + # conditions are not state machines and can be oscillating - from the + # offical API conventions guide: + # In general, condition values may change back and forth, but some + # condition transitions may be monotonic, depending on the resource and + # condition type. However, conditions are observations and not, + # themselves, state machines, nor do we define comprehensive state + # machines for objects, nor behaviors associated with state + # transitions. The system is level-based rather than edge-triggered, + # and should assume an Open World. + # In this implementation, we synthesize our notion of "phase" state + # machine from `conditions`, since Kubernetes won't do it for us (for + # many good reasons). + # + # + # + # `conditions` can be of the following types - + # 1. (kubelet) Initialized (always True since we don't rely on init + # containers) + # 2. (kubelet) ContainersReady + # 3. (kubelet) Ready (same as ContainersReady since we don't use + # ReadinessGates - + # https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/status/generate.go) + # 4. (kube-scheduler) PodScheduled + # (https://github.com/kubernetes/kubernetes/blob/master/pkg/scheduler/scheduler.go) + # 5. (kube-scheduler) Unschedulable + # + # WIP... + + JOB_ACTIVE = "job:active" + JOB_FAILED = "" + + def __init__(self, client_wrapper, name, uid, namespace): + self._client_wrapper = client_wrapper + self._name = name + self._id = uid + self._namespace = namespace + + self._job = self._fetch_job() + self._pod = self._fetch_pod() + + import atexit + + atexit.register(self.kill) + + def __repr__(self): + return "{}('{}/{}')".format( + self.__class__.__name__, self._namespace, self._name + ) + + @k8s_retry() + def _fetch_job(self): + client = self._client_wrapper.get() + try: + return ( + client.BatchV1Api() + .read_namespaced_job(name=self._name, namespace=self._namespace) + .to_dict() + ) + except client.rest.ApiException as e: + # TODO: Handle failures as well as the fact that a different + # process can delete the job. + raise e + + @k8s_retry() + def _fetch_pod(self): + """Fetch pod metadata. May return None if pod does not exist.""" + client = self._client_wrapper.get() + + pods = ( + client.CoreV1Api() + .list_namespaced_pod( + namespace=self._namespace, + label_selector="job-name={}".format(self._name), + ) + .to_dict()["items"] + ) + + if pods: + return pods[0] + else: + return None + + def kill(self): + # Terminating a Kubernetes job is a bit tricky. Issuing a + # `BatchV1Api.delete_namespaced_job` will also remove all traces of the + # job object from the Kubernetes API server which may not be desirable. + # This forces us to be a bit creative in terms of how we handle kill: + # + # 1. If the container is alive and kicking inside the pod, we simply + # attach ourselves to the container and issue a kill signal. The + # way we have initialized the Job ensures that the job will cleanly + # terminate. + # 2. In scenarios where either the pod (unschedulable etc.) or the + # container (ImagePullError etc.) hasn't come up yet, we become a + # bit creative by patching the job parallelism to 0. This ensures + # that the underlying node's resources are made available to + # kube-scheduler again. The downside is that the Job wouldn't mark + # itself as done and the pod metadata disappears from the API + # server. There is an open issue in the Kubernetes GH to provide + # better support for job terminations - + # https://github.com/kubernetes/enhancements/issues/2232 but + # meanwhile as a quick follow-up, we should investigate ways to + # terminate the pod without deleting the object. + # 3. If the pod object hasn't shown up yet, we set the parallelism to 0 + # to preempt it. + client = self._client_wrapper.get() + if not self._check_is_done(): + if self._check_is_running(): + + # Unless there is a bug in the code, self._pod cannot be None + # if we're in "running" state. + assert self._pod is not None + + # Case 1. + from kubernetes.stream import stream + + api_instance = client.CoreV1Api + try: + # TODO (savin): stream opens a web-socket connection. It may + # not be desirable to open multiple web-socket + # connections frivolously (think killing a + # workflow during a for-each step). + stream( + api_instance().connect_get_namespaced_pod_exec, + name=self._pod["metadata"]["name"], + namespace=self._namespace, + command=[ + "/bin/sh", + "-c", + "/sbin/killall5", + ], + stderr=True, + stdin=False, + stdout=True, + tty=False, + ) + except: + # Best effort. It's likely that this API call could be + # blocked for the user. + # TODO (savin): Forward the error to the user. + # pass + raise + else: + # Case 2. + try: + # TODO (savin): Also patch job annotation to reflect this + # action. + client.BatchV1Api().patch_namespaced_job( + name=self._name, + namespace=self._namespace, + field_manager="metaflow", + body={"spec": {"parallelism": 0}}, + ) + except: + # Best effort. + # TODO (savin): Forward the error to the user. + # pass + raise + return self + + @property + def id(self): + # TODO (savin): Should we use pod id instead? + return self._id + + def _check_is_done(self): + def _job_done(): + # Either the job succeeds or fails naturally or we may have + # forced the pod termination causing the job to still be in an + # active state but for all intents and purposes dead to us. + + # TODO (savin): check for self._job + return ( + bool(self._job["status"].get("succeeded")) + or bool(self._job["status"].get("failed")) + or (self._job["spec"]["parallelism"] == 0) + ) + + if not _job_done(): + # If not done, check for newer status + self._job = self._fetch_job() + if _job_done(): + return True + else: + # It is possible for the job metadata to not be updated yet, but the + # Pod has already succeeded or failed. + self._pod = self._fetch_pod() + if self._pod and (self._pod["status"]["phase"] in ("Succeeded", "Failed")): + return True + else: + return False + + def _get_status(self): + if not self._check_is_done(): + # If not done, check for newer pod status + self._pod = self._fetch_pod() + # Success! + if bool(self._job["status"].get("succeeded")): + return "Job:Succeeded" + # Failure! + if bool(self._job["status"].get("failed")) or ( + self._job["spec"]["parallelism"] == 0 + ): + return "Job:Failed" + if bool(self._job["status"].get("active")): + msg = "Job:Active" + if self._pod: + msg += " Pod:%s" % self._pod["status"]["phase"].title() + # TODO (savin): parse Pod conditions + container_status = ( + self._pod["status"].get("container_statuses") or [None] + )[0] + if container_status: + # We have a single container inside the pod + status = {"status": "waiting"} + for k, v in container_status["state"].items(): + if v is not None: + status["status"] = k + status.update(v) + msg += " Container:%s" % status["status"].title() + reason = "" + if status.get("reason"): + reason = status["reason"] + if status.get("message"): + reason += ":%s" % status["message"] + if reason: + msg += " [%s]" % reason + # TODO (savin): This message should be shortened before release. + return msg + return "Job:Unknown" + + def _check_has_succeeded(self): + # Job is in a terminal state and the status is marked as succeeded + if self._check_is_done(): + if bool(self._job["status"].get("succeeded")) or ( + self._pod and self._pod["status"]["phase"] == "Succeeded" + ): + return True + else: + return False + else: + return False + + def _check_has_failed(self): + # Job is in a terminal state and either the status is marked as failed + # or the Job is not allowed to launch any more pods + + if self._check_is_done(): + if ( + bool(self._job["status"].get("failed")) + or (self._job["spec"]["parallelism"] == 0) + or (self._pod and self._pod["status"]["phase"] == "Failed") + ): + return True + else: + return False + else: + return False + + def _check_is_running(self): + # Returns true if the container is running. + if not self._check_is_done(): + # If not done, check if pod has been assigned and is in Running + # phase + if self._pod is None: + return False + pod_phase = self._pod.get("status", {}).get("phase") + return pod_phase == "Running" + return False + + def _get_done_reason(self): + if self._check_is_done(): + if self._check_has_succeeded(): + return 0, None + # Best effort since Pod object can disappear on us at anytime + else: + + def _done(): + return self._pod.get("status", {}).get("phase") in ( + "Succeeded", + "Failed", + ) + + if not _done(): + # If pod status is dirty, check for newer status + self._pod = self._fetch_pod() + if self._pod: + pod_status = self._pod["status"] + if pod_status.get("container_statuses") is None: + # We're done, but no container_statuses is set + # This can happen when the pod is evicted + return None, ": ".join( + filter( + None, + [pod_status.get("reason"), pod_status.get("message")], + ) + ) + + for k, v in ( + pod_status.get("container_statuses", [{}])[0] + .get("state", {}) + .items() + ): + if v is not None: + return v.get("exit_code"), ": ".join( + filter( + None, + [v.get("reason"), v.get("message")], + ) + ) + + return None, None + + @property + def is_done(self): + return self._check_is_done() + + @property + def has_failed(self): + return self._check_has_failed() + + @property + def is_running(self): + return self._check_is_running() + + @property + def reason(self): + return self._get_done_reason() + + @property + def status(self): + return self._get_status() diff --git a/metaflow/plugins/aws/eks/kubernetes_decorator.py b/metaflow/plugins/aws/eks/kubernetes_decorator.py new file mode 100644 index 00000000000..46e937d674c --- /dev/null +++ b/metaflow/plugins/aws/eks/kubernetes_decorator.py @@ -0,0 +1,291 @@ +import os +import sys +import platform +import requests + +from metaflow import util +from metaflow.decorators import StepDecorator +from metaflow.metadata import MetaDatum +from metaflow.metadata.util import sync_local_metadata_to_datastore +from metaflow.metaflow_config import ( + ECS_S3_ACCESS_IAM_ROLE, + BATCH_JOB_QUEUE, + BATCH_CONTAINER_IMAGE, + BATCH_CONTAINER_REGISTRY, + ECS_FARGATE_EXECUTION_ROLE, + DATASTORE_LOCAL_DIR, +) +from metaflow.plugins import ResourcesDecorator +from metaflow.plugins.timeout_decorator import get_run_time_limit_for_task +from metaflow.sidecar import SidecarSubProcess + +from .kubernetes import KubernetesException +from ..aws_utils import get_docker_registry + + +class KubernetesDecorator(StepDecorator): + """ + TODO (savin): Update this docstring. + Step decorator to specify that this step should execute on Kubernetes. + + This decorator indicates that your step should execute on Kubernetes. Note + that you can apply this decorator automatically to all steps using the + ```--with kubernetes``` argument when calling run/resume. Step level + decorators within the code are overrides and will force a step to execute + on Kubernetes regardless of the ```--with``` specification. + + To use, annotate your step as follows: + ``` + @kubernetes + @step + def my_step(self): + ... + ``` + Parameters + ---------- + cpu : int + Number of CPUs required for this step. Defaults to 1. If @resources is + also present, the maximum value from all decorators is used + gpu : int + Number of GPUs required for this step. Defaults to 0. If @resources is + also present, the maximum value from all decorators is used + memory : int + Memory size (in MB) required for this step. Defaults to 4096. If + @resources is also present, the maximum value from all decorators is + used + image : string + Docker image to use when launching on Kubernetes. If not specified, a + default docker image mapping to the current version of Python is used + shared_memory : int + The value for the size (in MiB) of the /dev/shm volume for this step. + This parameter maps to the --shm-size option to docker run. + """ + + name = "kubernetes" + defaults = { + "cpu": "1", + "memory": "4096", + "disk": "10240", + "image": None, + "service_account": None, + "secrets": None, # e.g., mysecret + "node_selector": None, # e.g., kubernetes.io/os=linux + "gpu": "0", + # "shared_memory": None, + "namespace": None, + } + package_url = None + package_sha = None + run_time_limit = None + + def __init__(self, attributes=None, statically_defined=False): + super(KubernetesDecorator, self).__init__( + attributes, statically_defined + ) + + # TODO: Unify the logic with AWS Batch + # If no docker image is explicitly specified, impute a default image. + if not self.attributes["image"]: + # If metaflow-config specifies a docker image, just use that. + if BATCH_CONTAINER_IMAGE: + self.attributes["image"] = BATCH_CONTAINER_IMAGE + # If metaflow-config doesn't specify a docker image, assign a + # default docker image. + else: + # Default to vanilla Python image corresponding to major.minor + # version of the Python interpreter launching the flow. + self.attributes["image"] = "python:%s.%s" % ( + platform.python_version_tuple()[0], + platform.python_version_tuple()[1], + ) + # Assign docker registry URL for the image. + if not get_docker_registry(self.attributes["image"]): + if BATCH_CONTAINER_REGISTRY: + self.attributes["image"] = "%s/%s" % ( + BATCH_CONTAINER_REGISTRY.rstrip("/"), + self.attributes["image"], + ) + + # Refer https://github.com/Netflix/metaflow/blob/master/docs/lifecycle.png + # to understand where these functions are invoked in the lifecycle of a + # Metaflow flow. + def step_init( + self, flow, graph, step, decos, environment, flow_datastore, logger + ): + # Executing Kubernetes jobs requires a non-local datastore at the + # moment. + # TODO: To support MiniKube we need to enable local datastore execution. + if flow_datastore.TYPE != "s3": + raise KubernetesException( + "The *@kubernetes* decorator requires --datastore=s3 " + "at the moment." + ) + + # Set internal state. + self.logger = logger + self.environment = environment + self.step = step + self.flow_datastore = flow_datastore + for deco in decos: + if isinstance(deco, ResourcesDecorator): + for k, v in deco.attributes.items(): + # We use the larger of @resources and @k8s attributes + # TODO: Fix https://github.com/Netflix/metaflow/issues/467 + my_val = self.attributes.get(k) + if not (my_val is None and v is None): + self.attributes[k] = str( + max(int(my_val or 0), int(v or 0)) + ) + + # Set run time limit for the Kubernetes job. + self.run_time_limit = get_run_time_limit_for_task(decos) + if self.run_time_limit < 60: + raise KubernetesException( + "The timeout for step *{step}* should be " + "at least 60 seconds for execution on " + "Kubernetes.".format(step=step) + ) + + def runtime_init(self, flow, graph, package, run_id): + # Set some more internal state. + self.flow = flow + self.graph = graph + self.package = package + self.run_id = run_id + + def runtime_task_created(self, + task_datastore, + task_id, + split_index, + input_paths, + is_cloned, + ubf_context): + # To execute the Kubernetes job, the job container needs to have + # access to the code package. We store the package in the datastore + # which the pod is able to download as part of it's entrypoint. + if not is_cloned: + self._save_package_once(self.flow_datastore, self.package) + + def runtime_step_cli( + self, cli_args, retry_count, max_user_code_retries, ubf_context + ): + if retry_count <= max_user_code_retries: + # After all attempts to run the user code have failed, we don't need + # to execute on Kubernetes anymore. We can execute possible fallback + # code locally. + cli_args.commands = ["kubernetes", "step"] + cli_args.command_args.append(self.package_sha) + cli_args.command_args.append(self.package_url) + + # --namespace is used to specify Metaflow namespace (different + # concept from k8s namespace). + for k,v in self.attributes.items(): + if k == 'namespace': + cli_args.command_options['k8s_namespace'] = v + else: + cli_args.command_options[k] = v + cli_args.command_options["run-time-limit"] = self.run_time_limit + cli_args.entrypoint[0] = sys.executable + + def task_pre_step(self, + step_name, + task_datastore, + metadata, + run_id, + task_id, + flow, + graph, + retry_count, + max_retries, + ubf_context, + inputs): + self.metadata = metadata + self.task_datastore = task_datastore + + # task_pre_step may run locally if fallback is activated for @catch + # decorator. In that scenario, we skip collecting Kubernetes execution + # metadata. A rudimentary way to detect non-local execution is to + # check for the existence of METAFLOW_KUBERNETES_WORKLOAD environment + # variable. + + if "METAFLOW_KUBERNETES_WORKLOAD" in os.environ: + meta = {} + # TODO: Get kubernetes job id and job name + meta["kubernetes-pod-id"] = os.environ["METAFLOW_KUBERNETES_POD_ID"] + meta["kubernetes-pod-name"] = os.environ[ + "METAFLOW_KUBERNETES_POD_NAME" + ] + meta["kubernetes-pod-namespace"] = os.environ[ + "METAFLOW_KUBERNETES_POD_NAMESPACE" + ] + # meta['kubernetes-job-attempt'] = ? + + entries = [ + MetaDatum(field=k, value=v, type=k, tags=[]) + for k, v in meta.items() + ] + # Register book-keeping metadata for debugging. + metadata.register_metadata(run_id, step_name, task_id, entries) + + # Start MFLog sidecar to collect task logs. + self._save_logs_sidecar = SidecarSubProcess( + "save_logs_periodically" + ) + + def task_post_step(self, + step_name, + flow, + graph, + retry_count, + max_user_code_retries): + # task_post_step may run locally if fallback is activated for @catch + # decorator. + if 'METAFLOW_KUBERNETES_WORKLOAD' in os.environ: + # If `local` metadata is configured, we would need to copy task + # execution metadata from the AWS Batch container to user's + # local file system after the user code has finished execution. + # This happens via datastore as a communication bridge. + if self.metadata.TYPE == 'local': + # Note that the datastore is *always* Amazon S3 (see + # runtime_task_created function). + sync_local_metadata_to_datastore(DATASTORE_LOCAL_DIR, + self.task_datastore) + + def task_exception(self, + exception, + step_name, + flow, + graph, + retry_count, + max_user_code_retries): + # task_exception may run locally if fallback is activated for @catch + # decorator. + if 'METAFLOW_KUBERNETES_WORKLOAD' in os.environ: + # If `local` metadata is configured, we would need to copy task + # execution metadata from the AWS Batch container to user's + # local file system after the user code has finished execution. + # This happens via datastore as a communication bridge. + if self.metadata.TYPE == 'local': + # Note that the datastore is *always* Amazon S3 (see + # runtime_task_created function). + sync_local_metadata_to_datastore(DATASTORE_LOCAL_DIR, + self.task_datastore) + + def task_finished(self, + step_name, + flow, + graph, + is_task_ok, + retry_count, + max_retries): + try: + self._save_logs_sidecar.kill() + except: + # Best effort kill + pass + + @classmethod + def _save_package_once(cls, flow_datastore, package): + if cls.package_url is None: + cls.package_url, cls.package_sha = flow_datastore.save_data( + [package.blob], len_hint=1)[0] \ No newline at end of file diff --git a/metaflow/plugins/conda/conda_step_decorator.py b/metaflow/plugins/conda/conda_step_decorator.py index cbbbeec6bbb..10a9b9894f2 100644 --- a/metaflow/plugins/conda/conda_step_decorator.py +++ b/metaflow/plugins/conda/conda_step_decorator.py @@ -193,13 +193,13 @@ def _disable_safety_checks(self, decos): # a macOS. This is needed because of gotchas around inconsistently # case-(in)sensitive filesystems for macOS and linux. for deco in decos: - if deco.name == 'batch' and platform.system() == 'Darwin': + if deco.name in ('batch', 'kubernetes') and platform.system() == 'Darwin': return True return False def _architecture(self, decos): for deco in decos: - if deco.name == 'batch': + if deco.name in ('batch', 'kubernetes'): # force conda resolution for linux-64 architectures return 'linux-64' bit = '32' @@ -306,7 +306,9 @@ def runtime_step_cli(self, retry_count, max_user_code_retries, ubf_context): - if self.is_enabled(ubf_context) and 'batch' not in cli_args.commands: + no_batch = 'batch' not in cli_args.commands + no_kubernetes = 'kubernetes' not in cli_args.commands + if self.is_enabled(ubf_context) and no_batch and no_kubernetes: python_path = self.metaflow_home if self.addl_paths is not None: addl_paths = os.pathsep.join(self.addl_paths) diff --git a/metaflow/task.py b/metaflow/task.py index b49bf77b0aa..b98a2c8c0ee 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -496,10 +496,6 @@ def run_step(self, output.save_metadata({'task_end': {}}) output.persist(self.flow) - # this writes a success marker indicating that the - # "transaction" is done - output.done() - # final decorator hook: The task results are now # queryable through the client API / datastore for deco in decorators: @@ -510,6 +506,10 @@ def run_step(self, retry_count, max_user_code_retries) + # this writes a success marker indicating that the + # "transaction" is done + output.done() + # terminate side cars logger.terminate() self.metadata.stop_heartbeat() diff --git a/test/core/contexts.json b/test/core/contexts.json index 6d0e79b238c..c4bf3d98cee 100644 --- a/test/core/contexts.json +++ b/test/core/contexts.json @@ -125,6 +125,40 @@ "DetectSegFaultTest", "TimeoutDecoratorTest" ] + }, + { + "name": "python3-k8s", + "disabled": true, + "python": "python3", + "top_options": [ + "--event-logger=nullSidecarLogger", + "--no-pylint", + "--quiet", + "--with=kubernetes:memory=256,disk=1024", + "--datastore=s3" + ], + "env": { + "METAFLOW_USER": "tester", + "METAFLOW_RUN_BOOL_PARAM": "False", + "METAFLOW_RUN_NO_DEFAULT_PARAM": "test_str", + "METAFLOW_DEFAULT_METADATA": "service" + }, + "run_options": [ + "--max-workers", "50", + "--max-num-splits", "10000", + "--tag", "\u523a\u8eab means sashimi", + "--tag", "multiple tags should be ok" + ], + "checks": ["python3-cli", "python3-metadata"], + "disabled_tests": [ + "LargeArtifactTest", + "WideForeachTest", + "TagCatchTest", + "BasicUnboundedForeachTest", + "NestedUnboundedForeachTest", + "DetectSegFaultTest", + "TimeoutDecoratorTest" + ] } ], "checks": { diff --git a/test/unit/test_k8s_job_name_sanitizer.py b/test/unit/test_k8s_job_name_sanitizer.py new file mode 100644 index 00000000000..e019a47bea3 --- /dev/null +++ b/test/unit/test_k8s_job_name_sanitizer.py @@ -0,0 +1,26 @@ +import re +from metaflow.plugins.aws.eks.kubernetes import generate_rfc1123_name + +rfc1123 = re.compile(r'^[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?$') + +def test_job_name_santitizer(): + # Basic name + assert rfc1123.match(generate_rfc1123_name('HelloFlow', '1', 'end', '321', '1')) + + # Step name ends with _ + assert rfc1123.match(generate_rfc1123_name('HelloFlow', '1', '_end', '321', '1')) + + # Step name starts and ends with _ + assert rfc1123.match(generate_rfc1123_name('HelloFlow', '1', '_end_', '321', '1')) + + # Flow name ends with _ + assert rfc1123.match(generate_rfc1123_name('HelloFlow_', '1', 'end', '321', '1')) + + # Same flow name, different case must produce different job names + assert generate_rfc1123_name('Helloflow', '1', 'end', '321', '1') != generate_rfc1123_name('HelloFlow', '1', 'end', '321', '1') + + # Very long step name should be fine + assert rfc1123.match(generate_rfc1123_name('Helloflow', '1', 'end'*50, '321', '1')) + + # Very long run id should be fine too + assert rfc1123.match(generate_rfc1123_name('Helloflow', '1'*100, 'end', '321', '1')) \ No newline at end of file diff --git a/test/unit/test_k8s_label_sanitizer.py b/test/unit/test_k8s_label_sanitizer.py new file mode 100644 index 00000000000..6fcfbd5553f --- /dev/null +++ b/test/unit/test_k8s_label_sanitizer.py @@ -0,0 +1,28 @@ +import re +from metaflow.plugins.aws.eks.kubernetes import sanitize_label_value, LABEL_VALUE_REGEX + + +def test_label_value_santitizer(): + assert LABEL_VALUE_REGEX.match(sanitize_label_value('HelloFlow')) + + # The value is too long + assert LABEL_VALUE_REGEX.match(sanitize_label_value('a' * 1000)) + + # Different long values should still not be equal after sanitization + assert sanitize_label_value('a' * 1000) != sanitize_label_value('a' * 1001) + assert sanitize_label_value('-' * 1000) != sanitize_label_value('-' * 1001) + + # Different long values should still not be equal after sanitization + assert sanitize_label_value('alice!') != sanitize_label_value('alice?') + + # ends with dash + assert LABEL_VALUE_REGEX.match(sanitize_label_value('HelloFlow-')) + + # non-ascii + assert LABEL_VALUE_REGEX.match(sanitize_label_value('метафлоу')) + + # different only in case + assert sanitize_label_value('Alice') != sanitize_label_value('alice') + + # spaces + assert LABEL_VALUE_REGEX.match(sanitize_label_value('Meta flow')) \ No newline at end of file From 6fa25be004553380378628fec86dc8fa0c6ffde0 Mon Sep 17 00:00:00 2001 From: Savin Date: Fri, 15 Oct 2021 16:18:06 -0700 Subject: [PATCH 067/176] k8s configuration wizard (#766) Co-authored-by: Oleg Avdeev --- metaflow/main_cli.py | 515 ++++++++++++++++++++++++------------ metaflow/metaflow_config.py | 12 + 2 files changed, 362 insertions(+), 165 deletions(-) diff --git a/metaflow/main_cli.py b/metaflow/main_cli.py index 8229ff8cbb8..09650507b63 100644 --- a/metaflow/main_cli.py +++ b/metaflow/main_cli.py @@ -415,44 +415,78 @@ def sandbox(profile): # Persist to a file. persist_env(env_dict, profile) -@configure.command(help='Configure metaflow to access self-managed AWS resources.') -@click.option('--profile', '-p', default='', - help='Configure a named profile. Activate the profile by setting ' - '`METAFLOW_PROFILE` environment variable.') -@click.pass_context -def aws(ctx, profile): - def cyan(string): - return click.style(string, fg='cyan') - def yellow(string): - return click.style(string, fg='yellow') +def cyan(string): + return click.style(string, fg='cyan') - # Greet the user! - echo('Welcome to Metaflow! Follow the prompts to configure your ' - 'installation.\n', - bold=True) +def yellow(string): + return click.style(string, fg='yellow') - # Check for existing configuration. - if not overwrite_config(profile): - ctx.abort() +def red(string): + return click.style(string, fg='red') - # Verify that the user has configured AWS credentials on their computer. - if not click.confirm('\nMetaflow relies on ' + - yellow('AWS access credentials') + - ' present on your computer to access resources on AWS.' - '\nBefore proceeding further, please confirm that you ' - 'have already configured these access credentials on ' - 'this computer.', - default=True): - echo('There are many ways to setup your AWS access credentials. You ' - 'can get started by following this guide: ', - nl=False, - fg='yellow') - echo('https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html', - fg='cyan') - ctx.abort() +def configure_s3_datastore(existing_env): + env = {} + # Set Amazon S3 as default datastore. + env['METAFLOW_DEFAULT_DATASTORE'] = 's3' + # Set Amazon S3 folder for datastore. + env['METAFLOW_DATASTORE_SYSROOT_S3'] =\ + click.prompt(cyan('[METAFLOW_DATASTORE_SYSROOT_S3]') + + ' Amazon S3 folder for Metaflow artifact storage ' + + '(s3:///).', + default=\ + existing_env.get('METAFLOW_DATASTORE_SYSROOT_S3'), + show_default=True) + # Set Amazon S3 folder for datatools. + env['METAFLOW_DATATOOLS_SYSROOT_S3'] =\ + click.prompt(cyan('[METAFLOW_DATATOOLS_SYSROOT_S3]') + + yellow(' (optional)') + + ' Amazon S3 folder for Metaflow datatools ' + + '(s3:///).', + default=\ + existing_env.get('METAFLOW_DATATOOLS_SYSROOT_S3', + os.path.join( + env['METAFLOW_DATASTORE_SYSROOT_S3'], + 'data')), + show_default=True) + return env + +def configure_metadata_service(existing_env): + empty_profile = False + if not existing_env: + empty_profile = True + env = {} - existing_env = get_env(profile) + # Set Metadata Service as default. + env['METAFLOW_DEFAULT_METADATA'] = 'service' + # Set URL for the Metadata Service. + env['METAFLOW_SERVICE_URL'] =\ + click.prompt(cyan('[METAFLOW_SERVICE_URL]') + + ' URL for Metaflow Service.', + default=existing_env.get('METAFLOW_SERVICE_URL'), + show_default=True) + # Set internal URL for the Metadata Service. + env['METAFLOW_SERVICE_INTERNAL_URL'] =\ + click.prompt(cyan('[METAFLOW_SERVICE_INTERNAL_URL]') + + yellow(' (optional)') + + ' URL for Metaflow Service ' + + '(Accessible only within VPC).', + default=\ + existing_env.get('METAFLOW_SERVICE_INTERNAL_URL', + env['METAFLOW_SERVICE_URL']), + show_default=True) + # Set Auth Key for the Metadata Service. + env['METAFLOW_SERVICE_AUTH_KEY'] =\ + click.prompt(cyan('[METAFLOW_SERVICE_AUTH_KEY]') + + yellow(' (optional)') + + ' Auth Key for Metaflow Service.', + default=\ + existing_env.get('METAFLOW_SERVICE_AUTH_KEY', ''), + show_default=True) + return env + + +def configure_datastore_and_metadata(existing_env): empty_profile = False if not existing_env: empty_profile = True @@ -472,28 +506,7 @@ def yellow(string): 'METAFLOW_DEFAULT_DATASTORE', '') == 's3', abort=False) if use_s3_as_datastore: - # Set Amazon S3 as default datastore. - env['METAFLOW_DEFAULT_DATASTORE'] = 's3' - # Set Amazon S3 folder for datastore. - env['METAFLOW_DATASTORE_SYSROOT_S3'] =\ - click.prompt(cyan('[METAFLOW_DATASTORE_SYSROOT_S3]') + - ' Amazon S3 folder for Metaflow artifact storage ' + - '(s3:///).', - default=\ - existing_env.get('METAFLOW_DATASTORE_SYSROOT_S3'), - show_default=True) - # Set Amazon S3 folder for datatools. - env['METAFLOW_DATATOOLS_SYSROOT_S3'] =\ - click.prompt(cyan('[METAFLOW_DATATOOLS_SYSROOT_S3]') + - yellow(' (optional)') + - ' Amazon S3 folder for Metaflow datatools ' + - '(s3:///).', - default=\ - existing_env.get('METAFLOW_DATATOOLS_SYSROOT_S3', - os.path.join( - env['METAFLOW_DATASTORE_SYSROOT_S3'], - 'data')), - show_default=True) + env.update(configure_s3_datastore(existing_env)) # Configure Metadata service for tracking. if click.confirm('\nMetaflow can use a ' + @@ -507,116 +520,288 @@ def yellow(string): 'service' or\ 'METAFLOW_SFN_IAM_ROLE' in env, abort=False): - # Set Metadata Service as default. - env['METAFLOW_DEFAULT_METADATA'] = 'service' - # Set URL for the Metadata Service. - env['METAFLOW_SERVICE_URL'] =\ - click.prompt(cyan('[METAFLOW_SERVICE_URL]') + - ' URL for Metaflow Service.', - default=existing_env.get('METAFLOW_SERVICE_URL'), - show_default=True) - # Set internal URL for the Metadata Service. - env['METAFLOW_SERVICE_INTERNAL_URL'] =\ - click.prompt(cyan('[METAFLOW_SERVICE_INTERNAL_URL]') + - yellow(' (optional)') + - ' URL for Metaflow Service ' + - '(Accessible only within VPC).', - default=\ - existing_env.get('METAFLOW_SERVICE_INTERNAL_URL', - env['METAFLOW_SERVICE_URL']), - show_default=True) - # Set Auth Key for the Metadata Service. - env['METAFLOW_SERVICE_AUTH_KEY'] =\ - click.prompt(cyan('[METAFLOW_SERVICE_AUTH_KEY]') + - yellow(' (optional)') + - ' Auth Key for Metaflow Service.', - default=\ - existing_env.get('METAFLOW_SERVICE_AUTH_KEY', ''), - show_default=True) - - # Configure AWS Batch for compute. - if use_s3_as_datastore: + env.update(configure_metadata_service(existing_env)) + return env + + +def configure_aws_batch(existing_env): + empty_profile = False + if not existing_env: + empty_profile = True + env = {} + + + # Set AWS Batch Job Queue. + env['METAFLOW_BATCH_JOB_QUEUE'] =\ + click.prompt(cyan('[METAFLOW_BATCH_JOB_QUEUE]') + + ' AWS Batch Job Queue.', + default=\ + existing_env.get('METAFLOW_BATCH_JOB_QUEUE'), + show_default=True) + # Set IAM role for AWS Batch jobs to assume. + env['METAFLOW_ECS_S3_ACCESS_IAM_ROLE'] =\ + click.prompt(cyan('[METAFLOW_ECS_S3_ACCESS_IAM_ROLE]') + + ' IAM role for AWS Batch jobs to access AWS ' + + 'resources (Amazon S3 etc.).', + default=\ + existing_env.get('METAFLOW_ECS_S3_ACCESS_IAM_ROLE'), + show_default=True) + # Set default Docker repository for AWS Batch jobs. + env['METAFLOW_BATCH_CONTAINER_REGISTRY'] =\ + click.prompt(cyan('[METAFLOW_BATCH_CONTAINER_REGISTRY]') + + yellow(' (optional)') + + ' Default Docker image repository for AWS ' + + 'Batch jobs. If nothing is specified, ' + + 'dockerhub (hub.docker.com/) is ' + + 'used as default.', + default=\ + existing_env.get('METAFLOW_BATCH_CONTAINER_REGISTRY', ''), + show_default=True) + # Set default Docker image for AWS Batch jobs. + env['METAFLOW_BATCH_CONTAINER_IMAGE'] =\ + click.prompt(cyan('[METAFLOW_BATCH_CONTAINER_IMAGE]') + + yellow(' (optional)') + + ' Default Docker image for AWS Batch jobs. ' + + 'If nothing is specified, an appropriate ' + + 'python image is used as default.', + default=\ + existing_env.get('METAFLOW_BATCH_CONTAINER_IMAGE', ''), + show_default=True) + + # Configure AWS Step Functions for scheduling. + if click.confirm('\nMetaflow can ' + + yellow('schedule your flows on AWS Step ' + 'Functions') + + ' and trigger them at a specific cadence using ' + 'Amazon EventBridge.\nTo support flows involving ' + 'foreach steps, you would need access to AWS ' + 'DynamoDB.\nWould you like to configure AWS Step ' + 'Functions for scheduling?', + default=empty_profile or + 'METAFLOW_SFN_IAM_ROLE' in existing_env, + abort=False): + # Configure IAM role for AWS Step Functions. + env['METAFLOW_SFN_IAM_ROLE'] =\ + click.prompt(cyan('[METAFLOW_SFN_IAM_ROLE]') + + ' IAM role for AWS Step Functions to ' + + 'access AWS resources (AWS Batch, ' + + 'AWS DynamoDB).', + default=\ + existing_env.get('METAFLOW_SFN_IAM_ROLE'), + show_default=True) + # Configure IAM role for AWS Events Bridge. + env['METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE'] =\ + click.prompt(cyan('[METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE]') + + ' IAM role for Amazon EventBridge to ' + + 'access AWS Step Functions.', + default=\ + existing_env.get('METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE'), + show_default=True) + # Configure AWS DynamoDB Table for AWS Step Functions. + env['METAFLOW_SFN_DYNAMO_DB_TABLE'] =\ + click.prompt(cyan('[METAFLOW_SFN_DYNAMO_DB_TABLE]') + + ' AWS DynamoDB table name for tracking '+ + 'AWS Step Functions execution metadata.', + default=\ + existing_env.get('METAFLOW_SFN_DYNAMO_DB_TABLE'), + show_default=True) + return env + + +def check_kubernetes_client(ctx): + try: + import kubernetes + except ImportError: + echo("Please install python kubernetes client first " + \ + "(run " + yellow('pip install kubernetes') + \ + " or equivalent in your favorite python package manager)" + ) + ctx.abort() + + +def check_kubernetes_config(ctx): + from kubernetes import config + try: + all_contexts, current_context = config.list_kube_config_contexts() + click.confirm("You have a valid kubernetes configuration. The current context is set to " + \ + yellow(current_context["name"]) + " " + \ + "Proceed?", + default=True, + abort=True + ) + except config.config_exception.ConfigException as e: + click.confirm("\nYou don't seem to have a valid kubernetes configuration file. " + \ + "The error from kubernetes client library: " + \ + red(str(e)) + "." + \ + "To create a kubernetes configuration for EKS, you typically need to run " + yellow("aws eks update-kubeconfig --name ") + \ + ". For further details, refer to AWS Documentation at https://docs.aws.amazon.com/eks/latest/userguide/create-kubeconfig.html\n" + 'Do you want to proceed with configuring Metaflow for EKS anyway?', + default=False, + abort=True) + +def configure_eks(existing_env): + empty_profile = False + if not existing_env: + empty_profile = True + env = {} + + # Set K8S Namespace + env['METAFLOW_KUBERNETES_NAMESPACE'] =\ + click.prompt(cyan('[METAFLOW_KUBERNETES_NAMESPACE]') + + yellow(' (optional)') + + ' Kubernetes Namespace ', + default="default", + show_default=True) + + # Set K8S SA + env['METAFLOW_KUBERNETES_SERVICE_ACCOUNT'] =\ + click.prompt(cyan('[METAFLOW_KUBERNETES_SERVICE_ACCOUNT]') + + yellow(' (optional)') + + ' Kubernetes Service Account ', + default="default", + show_default=True) + + # Set default Docker repository for K8S jobs. + env['METAFLOW_KUBERNETES_CONTAINER_REGISTRY'] =\ + click.prompt(cyan('[METAFLOW_KUBERNETES_CONTAINER_REGISTRY]') + + yellow(' (optional)') + + ' Default Docker image repository for K8S ' + + 'jobs. If nothing is specified, ' + + 'dockerhub (hub.docker.com/) is ' + + 'used as default.', + default=\ + existing_env.get('METAFLOW_KUBERNETES_CONTAINER_REGISTRY', ''), + show_default=True) + # Set default Docker image for K8S jobs. + env['METAFLOW_KUBERNETES_CONTAINER_IMAGE'] =\ + click.prompt(cyan('[METAFLOW_KUBERNETES_CONTAINER_IMAGE]') + + yellow(' (optional)') + + ' Default Docker image for K8S jobs. ' + + 'If nothing is specified, an appropriate ' + + 'python image is used as default.', + default=\ + existing_env.get('METAFLOW_KUBERNETES_CONTAINER_IMAGE', ''), + show_default=True) + + return env + + +def verify_aws_credentials(ctx): + # Verify that the user has configured AWS credentials on their computer. + if not click.confirm('\nMetaflow relies on ' + + yellow('AWS access credentials') + + ' present on your computer to access resources on AWS.' + '\nBefore proceeding further, please confirm that you ' + 'have already configured these access credentials on ' + 'this computer.', + default=True): + echo('There are many ways to setup your AWS access credentials. You ' + 'can get started by following this guide: ', + nl=False, + fg='yellow') + echo('https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html', + fg='cyan') + ctx.abort() + + +@configure.command(help='Configure metaflow to access self-managed AWS resources.') +@click.option('--profile', '-p', default='', + help='Configure a named profile. Activate the profile by setting ' + '`METAFLOW_PROFILE` environment variable.') +@click.pass_context +def aws(ctx, profile): + + # Greet the user! + echo('Welcome to Metaflow! Follow the prompts to configure your ' + 'installation.\n', + bold=True) + + # Check for existing configuration. + if not overwrite_config(profile): + ctx.abort() + + verify_aws_credentials(ctx) + + existing_env = get_env(profile) + empty_profile = False + if not existing_env: + empty_profile = True + + env = {} + env.update(configure_datastore_and_metadata(existing_env)) + + # Configure AWS Batch for compute if using S3 + if env.get('METAFLOW_DEFAULT_DATASTORE') == 's3': if click.confirm('\nMetaflow can scale your flows by ' + - yellow('executing your steps on AWS Batch') + - '.\nAWS Batch is a strict requirement if you intend ' - 'to schedule your flows on AWS Step Functions.\nWould ' - 'you like to configure AWS Batch as your compute ' - 'backend?', - default=empty_profile or + yellow('executing your steps on AWS Batch') + + '.\nAWS Batch is a strict requirement if you intend ' + 'to schedule your flows on AWS Step Functions.\nWould ' + 'you like to configure AWS Batch as your compute ' + 'backend?', + default=empty_profile or 'METAFLOW_BATCH_JOB_QUEUE' in existing_env, - abort=False): - # Set AWS Batch Job Queue. - env['METAFLOW_BATCH_JOB_QUEUE'] =\ - click.prompt(cyan('[METAFLOW_BATCH_JOB_QUEUE]') + - ' AWS Batch Job Queue.', - default=\ - existing_env.get('METAFLOW_BATCH_JOB_QUEUE'), - show_default=True) - # Set IAM role for AWS Batch jobs to assume. - env['METAFLOW_ECS_S3_ACCESS_IAM_ROLE'] =\ - click.prompt(cyan('[METAFLOW_ECS_S3_ACCESS_IAM_ROLE]') + - ' IAM role for AWS Batch jobs to access AWS ' + - 'resources (Amazon S3 etc.).', - default=\ - existing_env.get('METAFLOW_ECS_S3_ACCESS_IAM_ROLE'), - show_default=True) - # Set default Docker repository for AWS Batch jobs. - env['METAFLOW_BATCH_CONTAINER_REGISTRY'] =\ - click.prompt(cyan('[METAFLOW_BATCH_CONTAINER_REGISTRY]') + - yellow(' (optional)') + - ' Default Docker image repository for AWS ' + - 'Batch jobs. If nothing is specified, ' + - 'dockerhub (hub.docker.com/) is ' + - 'used as default.', - default=\ - existing_env.get('METAFLOW_BATCH_CONTAINER_REGISTRY', ''), - show_default=True) - # Set default Docker image for AWS Batch jobs. - env['METAFLOW_BATCH_CONTAINER_IMAGE'] =\ - click.prompt(cyan('[METAFLOW_BATCH_CONTAINER_IMAGE]') + - yellow(' (optional)') + - ' Default Docker image for AWS Batch jobs. ' + - 'If nothing is specified, an appropriate ' + - 'python image is used as default.', - default=\ - existing_env.get('METAFLOW_BATCH_CONTAINER_IMAGE', ''), - show_default=True) - - # Configure AWS Step Functions for scheduling. - if click.confirm('\nMetaflow can ' + - yellow('schedule your flows on AWS Step ' - 'Functions') + - ' and trigger them at a specific cadence using ' - 'Amazon EventBridge.\nTo support flows involving ' - 'foreach steps, you would need access to AWS ' - 'DynamoDB.\nWould you like to configure AWS Step ' - 'Functions for scheduling?', - default=empty_profile or - 'METAFLOW_SFN_IAM_ROLE' in existing_env, - abort=False): - # Configure IAM role for AWS Step Functions. - env['METAFLOW_SFN_IAM_ROLE'] =\ - click.prompt(cyan('[METAFLOW_SFN_IAM_ROLE]') + - ' IAM role for AWS Step Functions to ' + - 'access AWS resources (AWS Batch, ' + - 'AWS DynamoDB).', - default=\ - existing_env.get('METAFLOW_SFN_IAM_ROLE'), - show_default=True) - # Configure IAM role for AWS Events Bridge. - env['METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE'] =\ - click.prompt(cyan('[METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE]') + - ' IAM role for Amazon EventBridge to ' + - 'access AWS Step Functions.', - default=\ - existing_env.get('METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE'), - show_default=True) - # Configure AWS DynamoDB Table for AWS Step Functions. - env['METAFLOW_SFN_DYNAMO_DB_TABLE'] =\ - click.prompt(cyan('[METAFLOW_SFN_DYNAMO_DB_TABLE]') + - ' AWS DynamoDB table name for tracking '+ - 'AWS Step Functions execution metadata.', - default=\ - existing_env.get('METAFLOW_SFN_DYNAMO_DB_TABLE'), - show_default=True) + abort=False): + env.update(configure_aws_batch(existing_env)) + + persist_env({k: v for k, v in env.items() if v}, profile) + + +@configure.command(help='Configure metaflow to use AWS EKS.') +@click.option('--profile', '-p', default='', + help='Configure a named profile. Activate the profile by setting ' + '`METAFLOW_PROFILE` environment variable.') +@click.pass_context +def eks(ctx, profile): + + check_kubernetes_client(ctx) + + # Greet the user! + echo('Welcome to Metaflow! Follow the prompts to configure your ' + 'installation.\n', + bold=True) + + check_kubernetes_config(ctx) + + # Check for existing configuration. + if not overwrite_config(profile): + ctx.abort() + + verify_aws_credentials(ctx) + + existing_env = get_env(profile) + + env = existing_env.copy() + + if existing_env.get('METAFLOW_DEFAULT_DATASTORE') == 's3': + # Skip S3 configuration if it is already configured + pass + elif not existing_env.get('METAFLOW_DEFAULT_DATASTORE'): + env.update(configure_s3_datastore(existing_env)) + else: + # If configured to use something else, offer to switch to S3 + click.confirm('\nMetaflow on EKS needs to use S3 as a datastore, ' + + "but your existing configuration is not using S3. " + + 'Would you like to reconfigure it to use S3?', + default=True, + abort=True) + env.update(configure_s3_datastore(existing_env)) + + # Configure remote metadata. + if existing_env.get('METAFLOW_DEFAULT_METADATA') == 'service': + # Skip metadata service configuration if it is already configured + pass + else: + if click.confirm('\nMetaflow can use a ' + + yellow('remote Metadata Service to track') + + ' and persist flow execution metadata. \nWould you like to ' + 'configure the Metadata Service?', + default=True, + abort=False): + env.update(configure_metadata_service(existing_env)) + + # Configure AWS EKS for compute. + env.update(configure_eks(existing_env)) + persist_env({k: v for k, v in env.items() if v}, profile) + +main() \ No newline at end of file diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py index 663a678e194..a1a28db9327 100644 --- a/metaflow/metaflow_config.py +++ b/metaflow/metaflow_config.py @@ -142,6 +142,18 @@ def from_conf(name, default=None): # `step-functions create --log-execution-history` command. SFN_EXECUTION_LOG_GROUP_ARN = from_conf("METAFLOW_SFN_EXECUTION_LOG_GROUP_ARN") +### +# Kubernetes configuration +### +# Kubernetes namespace to use for all objects created by Metaflow +KUBERNETES_NAMESPACE = from_conf("METAFLOW_KUBERNETES_NAMESPACE") +# Service account to use by K8S jobs created by Metaflow +KUBERNETES_SERVICE_ACCOUNT = from_conf("METAFLOW_KUBERNETES_SERVICE_ACCOUNT") +# +KUBERNETES_CONTAINER_REGISTRY = from_conf("METAFLOW_KUBERNETES_CONTAINER_REGISTRY") +# +KUBERNETES_CONTAINER_IMAGE = from_conf("METAFLOW_KUBERNETES_CONTAINER_IMAGE") + ### # Conda configuration ### From 96f67392032f968e29c414241830dd124c274ce1 Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Mon, 18 Oct 2021 09:36:09 -0700 Subject: [PATCH 068/176] handling exceptions for non-picklizable artifacts (#764) * handling exceptions for non-picklizable artifacts * Nit fixes. * exception name nit fix. --- metaflow/datastore/exceptions.py | 10 +++++++++- metaflow/datastore/task_datastore.py | 15 ++++++++++++--- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/metaflow/datastore/exceptions.py b/metaflow/datastore/exceptions.py index 139a2c25f43..1ed108f93f7 100644 --- a/metaflow/datastore/exceptions.py +++ b/metaflow/datastore/exceptions.py @@ -1,4 +1,12 @@ from ..exception import MetaflowException class DataException(MetaflowException): - headline = "Data store error" \ No newline at end of file + headline = "Data store error" + + +class UnpicklableArtifactException(MetaflowException): + headline = "Cannot pickle artifact" + + def __init__(self,artifact_name): + msg = 'Cannot pickle dump artifact named "%s"' % artifact_name + super().__init__(msg=msg, lineno=None) \ No newline at end of file diff --git a/metaflow/datastore/task_datastore.py b/metaflow/datastore/task_datastore.py index f2d352f099b..b617aaf94ec 100644 --- a/metaflow/datastore/task_datastore.py +++ b/metaflow/datastore/task_datastore.py @@ -13,7 +13,7 @@ from ..parameters import Parameter from ..util import Path, is_stringish, to_fileobj -from .exceptions import DataException +from .exceptions import DataException, UnpicklableArtifactException def only_if_not_done(f): @wraps(f) @@ -234,7 +234,10 @@ def pickle_iter(): raise DataException( "Artifact *%s* requires a serialization encoding that " "requires Python 3.4 or newer." % name) - blob = pickle.dumps(obj, protocol=4) + try: + blob = pickle.dumps(obj, protocol=4) + except TypeError as e: + raise UnpicklableArtifactException(name) else: try: blob = pickle.dumps(obj, protocol=2) @@ -246,7 +249,13 @@ def pickle_iter(): "Artifact *%s* is very large (over 2GB). " "You need to use Python 3.4 or newer if you want to " "serialize large objects." % name) - blob = pickle.dumps(obj, protocol=4) + try: + blob = pickle.dumps(obj, protocol=4) + except TypeError as e: + raise UnpicklableArtifactException(name) + except TypeError as e: + raise UnpicklableArtifactException(name) + self._info[name] = { 'size': len(blob), 'type': str(type(obj)), From b9592e08f3b1e68f44ae9e3aed6bed398b28e64b Mon Sep 17 00:00:00 2001 From: Savin Date: Mon, 18 Oct 2021 10:05:21 -0700 Subject: [PATCH 069/176] Update R tests to use R 4.1.1 (#773) * Revert "Mute R tests (#737)" This reverts commit ab8a9c63510933828eec039ffad84e261619cd92. * Update test.yml * Update publish.yml --- .github/workflows/publish.yml | 10 +++++----- .github/workflows/test.yml | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4e5564fc94d..c903616c649 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - lang: [Python] + lang: [Python, R] steps: - uses: actions/checkout@v2 @@ -36,17 +36,17 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install tox numpy - - name: Set up R 3.6 + - name: Set up R 4.1 if: matrix.lang == 'R' uses: r-lib/actions/setup-r@v1 with: - r-version: '3.6.3' + r-version: '4.1.1' - - name: Install R 3.6 system dependencies + - name: Install R 4.1 system dependencies if: matrix.lang == 'R' && matrix.os == 'ubuntu-latest' run: sudo apt-get update; sudo apt-get install -y libcurl4-openssl-dev qpdf libgit2-dev - - name: Install R 3.6 Rlang dependencies + - name: Install R 4.1 Rlang dependencies if: matrix.lang == 'R' run: | python3 -m pip install . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fc6c13f5898..28ad3cf0404 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - lang: [Python] + lang: [Python, R] steps: - uses: actions/checkout@v2 @@ -40,22 +40,22 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install tox numpy - - name: Set up R 3.6 + - name: Set up R 4.1 if: matrix.lang == 'R' uses: r-lib/actions/setup-r@v1 with: - r-version: '3.6.3' + r-version: '4.1.1' - - name: Install R 3.6 system dependencies + - name: Install R 4.1 system dependencies if: matrix.lang == 'R' && matrix.os == 'ubuntu-latest' run: sudo apt-get update; sudo apt-get install -y libcurl4-openssl-dev qpdf libgit2-dev - - name: Install R 3.6 Rlang dependencies + - name: Install R 4.1 Rlang dependencies if: matrix.lang == 'R' run: | + python3 -m pip install . Rscript -e 'install.packages("devtools", repos="https://cloud.r-project.org", Ncpus=8)' Rscript -e 'devtools::install_deps("R", dependencies=TRUE, repos="https://cloud.r-project.org", upgrade="default")' - python3 -m pip install . R CMD INSTALL R Rscript -e 'install.packages(c("data.table", "caret", "glmnet", "Matrix", "rjson"), repos="https://cloud.r-project.org", Ncpus=8)' From 80363680497a8b30328c4320d68e31b31a9b8456 Mon Sep 17 00:00:00 2001 From: Romain Date: Mon, 18 Oct 2021 11:58:31 -0700 Subject: [PATCH 070/176] Make tags sets instead of lists; remove defaults of [] in favor of None (#765) --- metaflow/metadata/metadata.py | 52 +++++++++++++++------------- metaflow/plugins/metadata/local.py | 23 +++++++----- metaflow/plugins/metadata/service.py | 26 ++++++++------ 3 files changed, 56 insertions(+), 45 deletions(-) diff --git a/metaflow/metadata/metadata.py b/metaflow/metadata/metadata.py index d5818e8e8b9..871015cd441 100644 --- a/metaflow/metadata/metadata.py +++ b/metaflow/metadata/metadata.py @@ -95,7 +95,7 @@ def version(self): ''' return '' - def new_run_id(self, tags=[], sys_tags=[]): + def new_run_id(self, tags=None, sys_tags=None): ''' Creates an ID and registers this new run. @@ -104,9 +104,9 @@ def new_run_id(self, tags=[], sys_tags=[]): Parameters ---------- tags : list, optional - Tags to apply to this particular run, by default [] + Tags to apply to this particular run, by default None sys_tags : list, optional - System tags to apply to this particular run, by default [] + System tags to apply to this particular run, by default None Returns ------- @@ -115,7 +115,7 @@ def new_run_id(self, tags=[], sys_tags=[]): ''' raise NotImplementedError() - def register_run_id(self, run_id, tags=[], sys_tags=[]): + def register_run_id(self, run_id, tags=None, sys_tags=None): ''' No-op operation in this implementation. @@ -124,13 +124,13 @@ def register_run_id(self, run_id, tags=[], sys_tags=[]): run_id : int Run ID for this run tags : list, optional - Tags to apply to this particular run, by default [] + Tags to apply to this particular run, by default None sys_tags : list, optional - System tags to apply to this particular run, by default [] + System tags to apply to this particular run, by default None ''' raise NotImplementedError() - def new_task_id(self, run_id, step_name, tags=[], sys_tags=[]): + def new_task_id(self, run_id, step_name, tags=None, sys_tags=None): ''' Creates an ID and registers this new task. @@ -143,9 +143,9 @@ def new_task_id(self, run_id, step_name, tags=[], sys_tags=[]): step_name : string Name of the step tags : list, optional - Tags to apply to this particular task, by default [] + Tags to apply to this particular task, by default None sys_tags : list, optional - System tags to apply to this particular task, by default [] + System tags to apply to this particular task, by default None Returns ------- @@ -155,7 +155,7 @@ def new_task_id(self, run_id, step_name, tags=[], sys_tags=[]): raise NotImplementedError() def register_task_id( - self, run_id, step_name, task_id, attempt=0, tags=[], sys_tags=[]): + self, run_id, step_name, task_id, attempt=0, tags=None, sys_tags=None): ''' No-op operation in this implementation. @@ -278,7 +278,7 @@ def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters, ''' raise NotImplementedError() - def add_sticky_tags(self, tags=[], sys_tags=[]): + def add_sticky_tags(self, tags=None, sys_tags=None): ''' Adds tags to be added to every run and task @@ -290,12 +290,14 @@ def add_sticky_tags(self, tags=[], sys_tags=[]): Parameters ---------- tags : list, optional - Tags to add to every run/task, by default [] + Tags to add to every run/task, by default None sys_tags : list, optional - System tags to add to every run/task, by default [] + System tags to add to every run/task, by default None ''' - self.sticky_tags.extend(tags) - self.sticky_sys_tags.extend(sys_tags) + if tags: + self.sticky_tags.update(tags) + if sys_tags: + self.sticky_sys_tags.update(sys_tags) @classmethod def get_object(cls, obj_type, sub_type, filters, *args): @@ -366,13 +368,13 @@ def get_object(cls, obj_type, sub_type, filters, *args): return cls._get_object_internal(obj_type, type_order, sub_type, sub_order, filters, *args) - def _all_obj_elements(self, tags=[], sys_tags=[]): + def _all_obj_elements(self, tags=None, sys_tags=None): user = get_username() return { 'flow_id': self._flow_name, 'user_name': user, - 'tags': tags, - 'system_tags': sys_tags, + 'tags': list(tags) if tags else [], + 'system_tags': list(sys_tags) if sys_tags else [], 'ts_epoch': int(round(time.time() * 1000))} def _flow_to_json(self): @@ -383,7 +385,7 @@ def _flow_to_json(self): 'flow_id': self._flow_name, 'ts_epoch': int(round(time.time() * 1000))} - def _run_to_json(self, run_id=None, tags=[], sys_tags=[]): + def _run_to_json(self, run_id=None, tags=None, sys_tags=None): if run_id is not None: d = {'run_number': run_id} else: @@ -391,14 +393,14 @@ def _run_to_json(self, run_id=None, tags=[], sys_tags=[]): d.update(self._all_obj_elements(tags, sys_tags)) return d - def _step_to_json(self, run_id, step_name, tags=[], sys_tags=[]): + def _step_to_json(self, run_id, step_name, tags=None, sys_tags=None): d = { 'run_number': run_id, 'step_name': step_name} d.update(self._all_obj_elements(tags, sys_tags)) return d - def _task_to_json(self, run_id, step_name, task_id=None, tags=[], sys_tags=[]): + def _task_to_json(self, run_id, step_name, task_id=None, tags=None, sys_tags=None): d = { 'run_number': run_id, 'step_name': step_name} @@ -408,7 +410,7 @@ def _task_to_json(self, run_id, step_name, task_id=None, tags=[], sys_tags=[]): return d def _object_to_json( - self, obj_type, run_id=None, step_name=None, task_id=None, tags=[], sys_tags=[]): + self, obj_type, run_id=None, step_name=None, task_id=None, tags=None, sys_tags=None): if obj_type == 'task': return self._task_to_json(run_id, step_name, task_id, tags, sys_tags) if obj_type == 'step': @@ -445,7 +447,7 @@ def _metadata_to_json(self, run_id, step_name, task_id, metadata): 'field_name': datum.field, 'type': datum.type, 'value': datum.value, - 'tags': datum.tags, + 'tags': list(set(datum.tags)) if datum.tags else [], 'user_name': user, 'ts_epoch': int(round(time.time() * 1000))} for datum in metadata] @@ -505,8 +507,8 @@ def _apply_filter(elts, filters): def __init__(self, environment, flow, event_logger, monitor): self._task_id_seq = -1 - self.sticky_tags = [] - self.sticky_sys_tags = [] + self.sticky_tags = set() + self.sticky_sys_tags = set() self._flow_name = flow.name self._event_logger = event_logger self._monitor = monitor diff --git a/metaflow/plugins/metadata/local.py b/metaflow/plugins/metadata/local.py index ab3b0c36a53..def6bc0f808 100644 --- a/metaflow/plugins/metadata/local.py +++ b/metaflow/plugins/metadata/local.py @@ -37,7 +37,7 @@ def print_clean(line, **kwargs): def version(self): return 'local' - def new_run_id(self, tags=[], sys_tags=[]): + def new_run_id(self, tags=None, sys_tags=None): # We currently just use the timestamp to create an ID. We can be reasonably certain # that it is unique and this makes it possible to do without coordination or # reliance on POSIX locks in the filesystem. @@ -45,7 +45,7 @@ def new_run_id(self, tags=[], sys_tags=[]): self._new_run(run_id, tags, sys_tags) return run_id - def register_run_id(self, run_id, tags=[], sys_tags=[]): + def register_run_id(self, run_id, tags=None, sys_tags=None): try: # This metadata provider only generates integer IDs so if this is # an integer, we don't register it again (since it was "registered" @@ -56,7 +56,7 @@ def register_run_id(self, run_id, tags=[], sys_tags=[]): except ValueError: return self._new_run(run_id, tags, sys_tags) - def new_task_id(self, run_id, step_name, tags=[], sys_tags=[]): + def new_task_id(self, run_id, step_name, tags=None, sys_tags=None): self._task_id_seq += 1 task_id = str(self._task_id_seq) self._new_task(run_id, step_name, task_id, tags, sys_tags) @@ -67,8 +67,8 @@ def register_task_id(self, step_name, task_id, attempt=0, - tags=[], - sys_tags=[]): + tags=None, + sys_tags=None): try: # Same logic as register_run_id int(task_id) @@ -175,7 +175,11 @@ def _makedirs(path): raise def _ensure_meta( - self, obj_type, run_id, step_name, task_id, tags=[], sys_tags=[]): + self, obj_type, run_id, step_name, task_id, tags=None, sys_tags=None): + if tags is None: + tags = set() + if sys_tags is None: + sys_tags = set() subpath = self._create_and_get_metadir(self._flow_name, run_id, step_name, task_id) selfname = os.path.join(subpath, '_self.json') self._makedirs(subpath) @@ -189,13 +193,14 @@ def _ensure_meta( run_id, step_name, task_id, - tags + self.sticky_tags, sys_tags + self.sticky_sys_tags)}) + self.sticky_tags.union(tags), + self.sticky_sys_tags.union(sys_tags))}) - def _new_run(self, run_id, tags=[], sys_tags=[]): + def _new_run(self, run_id, tags=None, sys_tags=None): self._ensure_meta('flow', None, None, None) self._ensure_meta('run', run_id, None, None, tags, sys_tags) - def _new_task(self, run_id, step_name, task_id, attempt=0, tags=[], sys_tags=[]): + def _new_task(self, run_id, step_name, task_id, attempt=0, tags=None, sys_tags=None): self._ensure_meta('step', run_id, step_name, None) self._ensure_meta('task', run_id, step_name, task_id, tags, sys_tags) self._register_code_package_metadata(run_id, step_name, task_id, attempt) diff --git a/metaflow/plugins/metadata/service.py b/metaflow/plugins/metadata/service.py index 507b037cb55..96d99d046a6 100644 --- a/metaflow/plugins/metadata/service.py +++ b/metaflow/plugins/metadata/service.py @@ -54,10 +54,10 @@ def default_info(cls): def version(self): return self._version(self._monitor) - def new_run_id(self, tags=[], sys_tags=[]): + def new_run_id(self, tags=None, sys_tags=None): return self._new_run(tags=tags, sys_tags=sys_tags) - def register_run_id(self, run_id, tags=[], sys_tags=[]): + def register_run_id(self, run_id, tags=None, sys_tags=None): try: # don't try to register an integer ID which was obtained # from the metadata service in the first place @@ -66,7 +66,7 @@ def register_run_id(self, run_id, tags=[], sys_tags=[]): except ValueError: return self._new_run(run_id, tags=tags, sys_tags=sys_tags) - def new_task_id(self, run_id, step_name, tags=[], sys_tags=[]): + def new_task_id(self, run_id, step_name, tags=None, sys_tags=None): return self._new_task(run_id, step_name, tags=tags, sys_tags=sys_tags) def register_task_id(self, @@ -74,8 +74,8 @@ def register_task_id(self, step_name, task_id, attempt=0, - tags=[], - sys_tags=[]): + tags=None, + sys_tags=None): try: # don't try to register an integer ID which was obtained # from the metadata service in the first place @@ -186,7 +186,7 @@ def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters= return None raise - def _new_run(self, run_id=None, tags=[], sys_tags=[]): + def _new_run(self, run_id=None, tags=None, sys_tags=None): # first ensure that the flow exists self._get_or_create('flow') run = self._get_or_create('run', run_id, tags=tags, sys_tags=sys_tags) @@ -197,8 +197,8 @@ def _new_task(self, step_name, task_id=None, attempt=0, - tags=[], - sys_tags=[]): + tags=None, + sys_tags=None): # first ensure that the step exists self._get_or_create('step', run_id, step_name) task = self._get_or_create('task', run_id, step_name, task_id, tags=tags, sys_tags=sys_tags) @@ -232,16 +232,20 @@ def _create_path(obj_type, flow_name, run_id=None, step_name=None): return create_path + '/task' def _get_or_create( - self, obj_type, run_id=None, step_name=None, task_id=None, tags=[], sys_tags=[]): + self, obj_type, run_id=None, step_name=None, task_id=None, tags=None, sys_tags=None): + if tags is None: + tags = set() + if sys_tags is None: + sys_tags = set() def create_object(): data = self._object_to_json( obj_type, run_id, step_name, task_id, - tags + self.sticky_tags, - sys_tags + self.sticky_sys_tags) + self.sticky_tags.union(tags), + self.sticky_sys_tags.union(sys_tags)) return self._request(self._monitor, create_path, data, obj_path) always_create = False From 0ebdd2aa3f7198ca7268f24461ad9c0104c0c5ec Mon Sep 17 00:00:00 2001 From: Savin Date: Mon, 18 Oct 2021 14:12:21 -0700 Subject: [PATCH 071/176] remove region env variable for @kubernetes (#774) --- metaflow/plugins/aws/eks/kubernetes.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/metaflow/plugins/aws/eks/kubernetes.py b/metaflow/plugins/aws/eks/kubernetes.py index 6592e6aa177..764b362d4c5 100644 --- a/metaflow/plugins/aws/eks/kubernetes.py +++ b/metaflow/plugins/aws/eks/kubernetes.py @@ -234,13 +234,6 @@ def create_job( # Retries are handled by Metaflow runtime retries=0, ) - .environment_variable( - # This is needed since `boto3` is not smart enough to figure out - # AWS region by itself. - # TODO: Fix this. - "AWS_DEFAULT_REGION", - "us-west-2", - ) .environment_variable("METAFLOW_CODE_SHA", code_package_sha) .environment_variable("METAFLOW_CODE_URL", code_package_url) .environment_variable("METAFLOW_CODE_DS", code_package_ds) From 4e2bea4cc1fc12a6862eb3d595655a63e2b86b5a Mon Sep 17 00:00:00 2001 From: Romain Date: Mon, 18 Oct 2021 14:40:41 -0700 Subject: [PATCH 072/176] Per attempt task (#725) * Missed attempt_id on attempt_done * Allow access to per-attempt Task and DataArtifact You can now specify a specific attempt for a Task or a DataArtifact in the client like so: - Task('flow/run/step/id/attempt') - DataArtifact('flow/run/step/id/name/attempt') This gives you a specific view of that particular attempt. Note that attempts are only valid for Task and Artifacts. * Added service component for task/artifact attempt This requires the attempt-fix branch of the metadata service. TODO: - still need to add version check to make sure we are hitting a modern enough service * Py2 compatibility * Moved the attempt specification from the pathspec to a separate argument Also added the version check (make sure the service returns 2.0.6 to test it out). Also addressed comments. * Typos * Add check to make sure attempts are only on Task and DataArtifact objects * feat: add properties for accessing artifact, stdout and stderr file sizes (#752) * wip: rough implementation of artifact size gets and suggestion for stdout/stderr log size property * add file_size to the datastore interface, implement for s3storage and use for artifact file size checks. * wip: implement log sizes for legacy and MFLOG type logs. * implement file_size for LocalStorage as well. update datastorage file_size docs * cleanup core docstrings for log_size properties * update docs and rename get_size to be specific about artifact size * refactor: move current attempt to a property * cleanup artifact size return * cleanup comment and rename file_size to be in line with other methods * change to require_mode('r') for size getters * fix indent * use cached filesize found in 'info' metadata for artifacts instead of continuously requesting filesizes. Fix possible issue with task_datastore not retaining passed in task attempt for further use. * change artifact size function to return an iterator to adhere to existing styles. * Remove visible tags/system_tags from metadata * Address issue when None is the value returned for all_tags * Add TaskDatastore caching to filecache; a few other fixes * Fix bug * Updated comment strings to more accurately reflect reality * Addressed comments Co-authored-by: Sakari Ikonen <64256562+saikonen@users.noreply.github.com> --- metaflow/client/core.py | 244 +++++++++++++++++++----- metaflow/client/filecache.py | 111 ++++++++++- metaflow/datastore/datastore_storage.py | 16 ++ metaflow/datastore/local_storage.py | 10 + metaflow/datastore/s3_storage.py | 6 + metaflow/datastore/task_datastore.py | 65 ++++++- metaflow/metadata/metadata.py | 74 ++++++- metaflow/metaflow_config.py | 9 +- metaflow/plugins/metadata/local.py | 17 +- metaflow/plugins/metadata/service.py | 45 +++-- 10 files changed, 515 insertions(+), 82 deletions(-) diff --git a/metaflow/client/core.py b/metaflow/client/core.py index 07c1ce903e1..d482e6a7236 100644 --- a/metaflow/client/core.py +++ b/metaflow/client/core.py @@ -12,7 +12,7 @@ MetaflowNamespaceMismatch,\ MetaflowInternalError -from metaflow.metaflow_config import DEFAULT_METADATA +from metaflow.metaflow_config import DEFAULT_METADATA, MAX_ATTEMPTS from metaflow.plugins import ENVIRONMENTS, METADATA_PROVIDERS from metaflow.unbounded_foreach import CONTROL_TASK_TAG from metaflow.util import cached_property, resolve_identity, to_unicode @@ -247,7 +247,7 @@ def __iter__(self): # filtering on namespace on flows means finding at least one # run in this namespace. This is_in_namespace() function # does this properly in this case - all_flows = self.metadata.get_object('root', 'flow', None) + all_flows = self.metadata.get_object('root', 'flow', None, None) all_flows = all_flows if all_flows else [] for flow in all_flows: try: @@ -317,12 +317,33 @@ class MetaflowObject(object): def __init__(self, pathspec=None, + attempt=None, _object=None, _parent=None, _namespace_check=True): self._metaflow = Metaflow() self._parent = _parent self._path_components = None + self._attempt = attempt + + if self._attempt is not None: + if self._NAME not in ['task', 'artifact']: + raise MetaflowNotFound( + "Attempts can only be specified for Task or DataArtifact") + try: + self._attempt = int(self._attempt) + except ValueError: + raise MetaflowNotFound("Attempt can only be an integer") + + if self._attempt < 0: + raise MetaflowNotFound("Attempt can only be non-negative") + elif self._attempt >= MAX_ATTEMPTS: + raise MetaflowNotFound( + "Attempt can only be smaller than %d" % MAX_ATTEMPTS) + # NOTE: It is possible that no attempt exists but we can't + # distinguish between "attempt will happen" and "no such + # attempt exists". + if pathspec: ids = pathspec.split('/') @@ -353,7 +374,8 @@ def __init__(self, raise MetaflowNamespaceMismatch(current_namespace) def _get_object(self, *path_components): - result = self._metaflow.metadata.get_object(self._NAME, 'self', None, *path_components) + result = self._metaflow.metadata.get_object( + self._NAME, 'self', None, self._attempt, *path_components) if not result: raise MetaflowNotFound("%s does not exist" % self) return result @@ -374,11 +396,13 @@ def __iter__(self): query_filter = {'any_tags': current_namespace} unfiltered_children = self._metaflow.metadata.get_object( - self._NAME, _CLASSES[self._CHILD_CLASS]._NAME, query_filter, *self.path_components) + self._NAME, _CLASSES[self._CHILD_CLASS]._NAME, query_filter, + self._attempt, *self.path_components) unfiltered_children = unfiltered_children if unfiltered_children else [] children = filter( lambda x: self._iter_filter(x), - (_CLASSES[self._CHILD_CLASS](_object=obj, _parent=self, _namespace_check=False) + (_CLASSES[self._CHILD_CLASS](attempt=self._attempt, + _object=obj, _parent=self, _namespace_check=False) for obj in unfiltered_children)) if children: @@ -422,6 +446,9 @@ def is_in_namespace(self): current_namespace in self._tags def __str__(self): + if self._attempt is not None: + return "%s('%s', attempt=%d)" % ( + self.__class__.__name__, self.pathspec, self._attempt) return "%s('%s')" % (self.__class__.__name__, self.pathspec) def __repr__(self): @@ -433,7 +460,7 @@ def _get_child(self, id): result.append(p) result.append(id) return self._metaflow.metadata.get_object( - _CLASSES[self._CHILD_CLASS]._NAME, 'self', None, *result) + _CLASSES[self._CHILD_CLASS]._NAME, 'self', None, self._attempt, *result) def __getitem__(self, id): """ @@ -456,7 +483,8 @@ def __getitem__(self, id): """ obj = self._get_child(id) if obj: - return _CLASSES[self._CHILD_CLASS](_object=obj, _parent=self) + return _CLASSES[self._CHILD_CLASS](attempt=self._attempt, + _object=obj, _parent=self) else: raise KeyError(id) @@ -522,10 +550,14 @@ def parent(self): if self._parent is None: pathspec = self.pathspec parent_pathspec = pathspec[:pathspec.rfind('/')] + # Only artifacts and tasks have attempts right now so we get the + # right parent if we are an artifact. + attempt_to_pass = self._attempt if self._NAME == 'artifact' else None # We can skip the namespace check because if self._NAME = 'run', # the parent object is guaranteed to be in namespace. # Otherwise the check is moot for Flow since parent is singular. - self._parent = _CLASSES[self._PARENT_CLASS](parent_pathspec, _namespace_check=False) + self._parent = _CLASSES[self._PARENT_CLASS]( + parent_pathspec, attempt=attempt_to_pass, _namespace_check=False) return self._parent @property @@ -534,7 +566,9 @@ def pathspec(self): Returns a string representation uniquely identifying this object. The string is the same as the one you would pass into the constructor - to build this object. + to build this object except if you are looking for a specific attempt of + a task or a data artifact (in which case you need to add `attempt=` + in the constructor). Returns ------- @@ -559,22 +593,10 @@ def path_components(self): List[string] Individual components of the pathspec """ - # Compute url_path from pathspec. - ids = self.pathspec.split('/') - - def traverse(cls, ids_r, lst): - lst.insert(0, ids_r[-1]) - if cls._PARENT_CLASS is None: - return lst - if len(ids_r) > 1: - cls = _CLASSES[cls._PARENT_CLASS] - return traverse(cls, ids_r[:-1], lst) - else: - return lst - if self._path_components is None: - self._path_components = traverse(_CLASSES[self._NAME], ids, []) - return self._path_components + ids = self.pathspec.split('/') + self._path_components = ids + return list(self._path_components) class MetaflowData(object): @@ -739,9 +761,30 @@ def data(self): return filecache.get_artifact_by_location( ds_type, location, meta, *components) - # TODO add - # @property - # def size(self) + @property + def size(self): + """ + Returns the size (in bytes) of the pickled object representing this + DataArtifact + + Returns + ------- + int + size of the pickled representation of data artifact (in bytes) + """ + global filecache + + ds_type = self._object['ds_type'] + location = self._object['location'] + components = self.path_components + + if filecache is None: + # TODO: Pass proper environment to properly extract artifacts + filecache = FileCache() + if location.startswith(':root:'): + return filecache.get_artifact_size(ds_type, location[6:], self._attempt, *components) + else: + return filecache.get_artifact_size_by_location(ds_type, location, self._attempt, *components) # TODO add # @property @@ -780,8 +823,16 @@ class Task(MetaflowObject): """ A Task represents an execution of a step. - As such, it contains all data artifacts associated with that execution as well as all metadata - associated with the execution. + As such, it contains all data artifacts associated with that execution as + well as all metadata associated with the execution. + + Note that you can also get information about a specific *attempt* of a + task. By default, the latest finished attempt is returned but you can + explicitly get information about a specific attempt by using the + following syntax when creating a task: + `Task('flow/run/step/task', attempt=)`. Note that you will not be able to + access a specific attempt of a task through the `.tasks` method of a step + for example (that will always return the latest attempt). Attributes ---------- @@ -828,7 +879,8 @@ def _iter_filter(self, x): @property def metadata(self): """ - Metadata events produced by this task. + Metadata events produced by this task across all attempts of the task + *except* if you selected a specific task attempt. Note that Metadata is different from tags. @@ -838,7 +890,7 @@ def metadata(self): Metadata produced by this task """ all_metadata = self._metaflow.metadata.get_object( - self._NAME, 'metadata', None, *self.path_components) + self._NAME, 'metadata', None, self._attempt, *self.path_components) all_metadata = all_metadata if all_metadata else [] return [Metadata(name=obj.get('field_name'), value=obj.get('value'), @@ -1017,32 +1069,97 @@ def stdout(self): """ Returns the full standard out of this task. - This information relates to the latest task that completed (in case of retries). In other - words, this does not return the realtime logs of execution. + If you specify a specific attempt for this task, it will return the + standard out for that attempt. If you do not specify an attempt, + this will return the current standard out for the latest *started* + attempt of the task. In both cases, multiple calls to this + method will return the most up-to-date log (so if an attempt is not + done, each call will fetch the latest log). Returns ------- string Standard output of this task """ - logtype = 'stdout' - return self._load_log(logtype) + return self._load_log('stdout') + + @property + def stdout_size(self): + """ + Returns the size of the stdout log of this task. + + Similar to `stdout`, the size returned is the latest size of the log + (so for a running attempt, this value will increase as the task produces + more output). + + Returns + ------- + int + Size of the stdout log content (in bytes) + """ + return self._get_logsize('stdout') @property def stderr(self): """ Returns the full standard error of this task. - This information relates to the latest task that completed (in case of retries). In other - words, this does not return the realtime logs of execution. + If you specify a specific attempt for this task, it will return the + standard error for that attempt. If you do not specify an attempt, + this will return the current standard error for the latest *started* + attempt. In both cases, multiple calls to this + method will return the most up-to-date log (so if an attempt is not + done, each call will fetch the latest log). Returns ------- string Standard error of this task """ - logtype = 'stderr' - return self._load_log(logtype) + return self._load_log('stderr') + + @property + def stderr_size(self): + """ + Returns the size of the stderr log of this task. + + Similar to `stderr`, the size returned is the latest size of the log + (so for a running attempt, this value will increase as the task produces + more output). + + Returns + ------- + int + Size of the stderr log content (in bytes) + """ + return self._get_logsize('stderr') + + @property + def current_attempt(self): + """ + Get the relevant attempt for this Task. + + Returns the specific attempt used when + initializing the instance, or the latest *started* attempt for the Task. + + Returns + ------- + int + attempt id for this task object + """ + if self._attempt is not None: + attempt = self._attempt + else: + # It is possible that a task fails before any metadata has been + # recorded. In this case, we assume that we are executing the + # first attempt. + # + # FIXME: Technically we are looking at the latest *recorded* attempt + # here. It is possible that logs exists for a newer attempt that + # just failed to record metadata. We could make this logic more robust + # and guarantee that we always return the latest available log. + attempt = int(self.metadata_dict.get('attempt', 0)) + return attempt @cached_property def code(self): @@ -1081,7 +1198,8 @@ def environment_info(self): env_type = my_code.info['environment_type'] if not env_type: return None - env = [m for m in ENVIRONMENTS + [MetaflowEnvironment] if m.TYPE == env_type][0] + env = [m for m in ENVIRONMENTS + + [MetaflowEnvironment] if m.TYPE == env_type][0] return env.get_client_info(self.path_components[0], self.metadata_dict) def _load_log(self, stream): @@ -1091,6 +1209,13 @@ def _load_log(self, stream): else: return ''.join(line + '\n' for _, line in self.loglines(stream)) + def _get_logsize(self, stream): + log_location = self.metadata_dict.get('log_location_%s' % stream) + if log_location: + return self._legacy_log_size(log_location, stream) + else: + return self._log_size(stream) + def loglines(self, stream, as_unicode=True): """ Return an iterator over (utc_timestamp, logline) tuples. @@ -1108,15 +1233,8 @@ def loglines(self, stream, as_unicode=True): return if filecache is None: filecache = FileCache() - # It is possible that a task fails before any metadata has been - # recorded. In this case, we assume that we are executing the - # first attempt. - # - # FIXME: Technically we are looking at the latest *recorded* attempt - # here. It is possible that logs exists for a newer attempt that - # just failed to record metadata. We could make this logic more robust - # and guarantee that we always return the latest available log. - attempt = int(self.metadata_dict.get('attempt', 0)) + + attempt = self.current_attempt logs = filecache.get_logs_stream( ds_type, ds_root, stream, attempt, *self.path_components) for line in merge_logs([blob for _, blob in logs]): @@ -1140,6 +1258,34 @@ def _load_log_legacy(self, log_location, logtype, as_unicode=True): else: return ret_val + def _legacy_log_size(self, log_location, logtype): + global filecache + + log_info = json.loads(log_location) + location = log_info['location'] + ds_type = log_info['ds_type'] + attempt = log_info['attempt'] + if filecache is None: + filecache = FileCache() + + return filecache.get_legacy_log_size( + ds_type, location, logtype, int(attempt), *self.path_components) + + def _log_size(self, stream): + global filecache + + ds_type = self.metadata_dict.get('ds-type') + ds_root = self.metadata_dict.get('ds-root') + if ds_type is None or ds_root is None: + return 0 + if filecache is None: + filecache = FileCache() + attempt = self.current_attempt + + return filecache.get_log_size( + ds_type, ds_root, stream, attempt, *self.path_components) + + class Step(MetaflowObject): """ diff --git a/metaflow/client/filecache.py b/metaflow/client/filecache.py index 8d538b82e9f..17b1a165974 100644 --- a/metaflow/client/filecache.py +++ b/metaflow/client/filecache.py @@ -1,5 +1,7 @@ from __future__ import print_function +from collections import OrderedDict import os +import sys import time from tempfile import NamedTemporaryFile from hashlib import sha1 @@ -7,14 +9,25 @@ from metaflow.datastore import DATASTORES, FlowDataStore from metaflow.datastore.content_addressed_store import BlobCache from metaflow.exception import MetaflowException -from metaflow.metaflow_config import CLIENT_CACHE_PATH, CLIENT_CACHE_MAX_SIZE +from metaflow.metaflow_config import CLIENT_CACHE_PATH, CLIENT_CACHE_MAX_SIZE, \ + CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT, CLIENT_CACHE_MAX_TASKDATASTORE_COUNT NEW_FILE_QUARANTINE = 10 +if sys.version_info[0] >= 3 and sys.version_info[1] >= 2: + def od_move_to_end(od, key): + od.move_to_end(key) +else: + # Not very efficient but works and most people are on 3.2+ + def od_move_to_end(od, key): + v = od.get(key) + del od[key] + od[key] = v class FileCacheException(MetaflowException): headline = 'File cache error' + class FileCache(object): def __init__(self, cache_dir=None, max_size=None): self._cache_dir = cache_dir @@ -31,10 +44,16 @@ def __init__(self, cache_dir=None, max_size=None): # We also keep a cache for FlowDataStore objects because some of them # may have long-lived persistent connections; this is purely a - # performance optimization. We do *not* keep track of task datastores - # to refresh them as needed. Caching FlowDataStore has no adverse - # affect in terms of having to refresh the cache. - self._store_caches = {} + # performance optimization. Uses OrderedDict to implement a kind of LRU + # cache and keep only a certain number of these caches around. + self._store_caches = OrderedDict() + + # We also keep a cache of data_metadata for TaskDatastore. This is used + # when querying for sizes of artifacts. Once we have queried for the size + # of one artifact in a TaskDatastore, caching this means that any + # queries on that same TaskDatastore will be quick (since we already + # have all the metadata) + self._task_metadata_caches = OrderedDict() @property def cache_dir(self): @@ -80,6 +99,27 @@ def get_log_legacy( self.create_file(path, log) return log + def get_legacy_log_size(self, ds_type, location, logtype, attempt, flow_name, run_id, step_name, task_id): + ds_cls = self._get_datastore_storage_impl(ds_type) + ds_root = ds_cls.path_join(*ds_cls.path_split(location)[:-5]) + ds = self._get_flow_datastore(ds_type, ds_root, flow_name) + + task_ds = ds.get_task_datastore( + run_id, step_name, task_id, attempt=attempt, + data_metadata={'objects': {}, 'info': {}}) + + return task_ds.get_legacy_log_size(logtype) + + def get_log_size(self, ds_type, ds_root, logtype, attempt, flow_name, run_id, step_name, task_id): + from metaflow.mflog import LOG_SOURCES + ds = self._get_flow_datastore(ds_type, ds_root, flow_name) + + task_ds = ds.get_task_datastore( + run_id, step_name, task_id, attempt=attempt, + data_metadata={'objects': {}, 'info': {}}) + + return task_ds.get_log_size(LOG_SOURCES, logtype) + def get_data(self, ds_type, flow_name, location, key): ds_cls = self._get_datastore_storage_impl(ds_type) ds_root = ds_cls.get_datastore_root_from_location(location, flow_name) @@ -87,6 +127,24 @@ def get_data(self, ds_type, flow_name, location, key): return next(ds.load_data([key], force_raw=True)) + def get_artifact_size_by_location(self, ds_type, location, attempt, flow_name, run_id, + step_name, task_id, name): + """Gets the size of the artifact content (in bytes) for the name at the location""" + ds_cls = self._get_datastore_storage_impl(ds_type) + ds_root = ds_cls.get_datastore_root_from_location(location, flow_name) + + return self.get_artifact_size( + ds_type, ds_root, attempt, flow_name, run_id, step_name, task_id, name) + + def get_artifact_size(self, ds_type, ds_root, attempt, flow_name, run_id, + step_name, task_id, name): + """Gets the size of the artifact content (in bytes) for the name""" + task_ds = self._get_task_datastore( + ds_type, ds_root, flow_name, run_id, step_name, task_id, attempt) + + _, size = next(task_ds.get_artifact_sizes([name])) + return size + def get_artifact_by_location( self, ds_type, location, data_metadata, flow_name, run_id, step_name, task_id, name): @@ -168,12 +226,12 @@ def read_file(self, path): def _index_objects(self): objects = [] if os.path.exists(self._cache_dir): - for flow_ds_type in os.listdir(self._cache_dir): - root = os.path.join(self._cache_dir, flow_ds_type) + for flow_ds_id in os.listdir(self._cache_dir): + root = os.path.join(self._cache_dir, flow_ds_id) if not os.path.isdir(root): continue for subdir in os.listdir(root): - root = os.path.join(self._cache_dir, flow_ds_type, subdir) + root = os.path.join(self._cache_dir, flow_ds_id, subdir) if not os.path.isdir(root): continue for obj in os.listdir(root): @@ -188,9 +246,14 @@ def _index_objects(self): self._objects = sorted(objects, reverse=False) @staticmethod - def _flow_ds_type(ds_type, ds_root, flow_name): + def _flow_ds_id(ds_type, ds_root, flow_name): return '.'.join([ds_type, ds_root, flow_name]) + @staticmethod + def _task_ds_id(ds_type, ds_root, flow_name, run_id, step_name, task_id, attempt): + return '.'.join( + [ds_type, ds_root, flow_name, run_id, step_name, task_id, str(attempt)]) + def _garbage_collect(self): now = time.time() while self._objects and self._total > self._max_size * 1024**2: @@ -224,10 +287,11 @@ def _get_datastore_storage_impl(ds_type): return storage_impl def _get_flow_datastore(self, ds_type, ds_root, flow_name): - cache_id = self._flow_ds_type(ds_type, ds_root, flow_name) + cache_id = self._flow_ds_id(ds_type, ds_root, flow_name) cached_flow_datastore = self._store_caches.get(cache_id) if cached_flow_datastore: + od_move_to_end(self._store_caches, cache_id) return cached_flow_datastore else: storage_impl = self._get_datastore_storage_impl(ds_type) @@ -240,8 +304,35 @@ def _get_flow_datastore(self, ds_type, ds_root, flow_name): cache_id, FileBlobCache(self, cache_id)) cached_flow_datastore.ca_store.set_blob_cache(blob_cache) self._store_caches[cache_id] = cached_flow_datastore + if len(self._store_caches) > CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT: + cache_id_to_remove, _ = self._store_caches.popitem(last=False) + del self._blob_caches[cache_id_to_remove] return cached_flow_datastore + def _get_task_datastore( + self, ds_type, ds_root, flow_name, run_id, step_name, task_id, attempt): + flow_ds = self._get_flow_datastore(ds_type, ds_root, flow_name) + cached_metadata = None + if attempt is not None: + cache_id = self._task_ds_id( + ds_type, ds_root, flow_name, run_id, step_name, task_id, attempt) + cached_metadata = self._task_metadata_caches.get(cache_id) + if cached_metadata: + od_move_to_end(self._task_metadata_caches, cache_id) + return flow_ds.get_task_datastore( + run_id, step_name, task_id, attempt=attempt, + data_metadata=cached_metadata) + # If we are here, we either have attempt=None or nothing in the cache + task_ds = flow_ds.get_task_datastore( + run_id, step_name, task_id, attempt=attempt) + cache_id = self._task_ds_id( + ds_type, ds_root, flow_name, run_id, step_name, task_id, task_ds.attempt + ) + self._task_metadata_caches[cache_id] = task_ds.ds_metadata + if len(self._task_metadata_caches) > CLIENT_CACHE_MAX_TASKDATASTORE_COUNT: + self._task_metadata_caches.popitem(last=False) + return task_ds + class FileBlobCache(BlobCache): def __init__(self, filecache, cache_id): diff --git a/metaflow/datastore/datastore_storage.py b/metaflow/datastore/datastore_storage.py index e9f501efdf6..393d79df0a6 100644 --- a/metaflow/datastore/datastore_storage.py +++ b/metaflow/datastore/datastore_storage.py @@ -155,6 +155,22 @@ def info_file(self, path): """ raise NotImplementedError + def size_file(self, path): + """ + Returns file size at the indicated 'path', or None if file can not be found. + + Parameters + ---------- + path : string + Path to the object + + Returns + ------- + Optional + int + """ + raise NotImplementedError + def list_content(self, paths): """ Lists the content of the datastore in the directory indicated by 'paths'. diff --git a/metaflow/datastore/local_storage.py b/metaflow/datastore/local_storage.py index f4ddfc8cf14..694c8b622b7 100644 --- a/metaflow/datastore/local_storage.py +++ b/metaflow/datastore/local_storage.py @@ -72,6 +72,16 @@ def info_file(self, path): return True, None return False, None + def size_file(self, path): + file_exists = self.is_file([path])[0] + if file_exists: + path = self.full_uri(path) + try: + return os.path.getsize(path) + except OSError: + return None + return None + def list_content(self, paths): results = [] for path in paths: diff --git a/metaflow/datastore/s3_storage.py b/metaflow/datastore/s3_storage.py index a4ee7fe1bf4..d61b923296d 100644 --- a/metaflow/datastore/s3_storage.py +++ b/metaflow/datastore/s3_storage.py @@ -45,6 +45,12 @@ def info_file(self, path): s3obj = s3.info(path, return_missing=True) return s3obj.exists, s3obj.metadata + def size_file(self, path): + with S3(s3root=self.datastore_root, + tmproot=os.getcwd(), external_client=self.s3_client) as s3: + s3obj = s3.info(path, return_missing=True) + return s3obj.size + def list_content(self, paths): strip_prefix_len = len(self.datastore_root.rstrip('/')) + 1 with S3(s3root=self.datastore_root, diff --git a/metaflow/datastore/task_datastore.py b/metaflow/datastore/task_datastore.py index b617aaf94ec..c123dfce399 100644 --- a/metaflow/datastore/task_datastore.py +++ b/metaflow/datastore/task_datastore.py @@ -126,12 +126,20 @@ def __init__(self, # datastore, the data may change). We make an exception to that # rule when allow_not_done is True which allows access to things # like logs even for tasks that did not write a done marker - self._attempt = None for i in range(metaflow_config.MAX_ATTEMPTS): check_meta = self._metadata_name_for_attempt( self.METADATA_ATTEMPT_SUFFIX, i) if self.has_metadata(check_meta, add_attempt=False): - self._attempt = i + max_attempt = i + if self._attempt is None: + self._attempt = max_attempt + elif self._attempt > max_attempt: + # In this case, the attempt does not exist so we can't load + # anything + self._objects = {} + self._info = {} + return + # Check if the latest attempt was completed successfully except # if we have allow_not_done data_obj = None @@ -164,6 +172,17 @@ def step_name(self): def task_id(self): return self._task_id + @property + def attempt(self): + return self._attempt + + @property + def ds_metadata(self): + return { + 'objects': self._objects.copy(), + 'info': self._info.copy() + } + @property def pathspec_index(self): idxstr = ','.join(map(str, (f.index for f in self['_foreach_stack']))) @@ -323,6 +342,48 @@ def load_artifacts(self, names): for sha, blob in self._ca_store.load_blobs(to_load): yield sha_to_names[sha], pickle.loads(blob) + @require_mode('r') + def get_artifact_sizes(self, names): + """ + Retrieves file sizes of artifacts defined in 'names' from their respective + stored file metadata. + + Usage restricted to only 'r' mode due to depending on the metadata being written + + Parameters + ---------- + names : List[string] + List of artifacts to retrieve + + Returns + ------- + Iterator[(string, int)] : + An iterator over sizes retrieved. + """ + for name in names: + info = self._info.get(name) + yield name, info.get('size', 0) + + @require_mode('r') + def get_legacy_log_size(self, stream): + name = self._metadata_name_for_attempt('%s.log' % stream) + path = self._storage_impl.path_join(self._path, name) + + return self._storage_impl.size_file(path) + + @require_mode('r') + def get_log_size(self, logsources, stream): + def _path(s): + # construct path for fetching of a single log source + _p = self._metadata_name_for_attempt( + self._get_log_location(s, stream)) + return self._storage_impl.path_join(self._path, _p) + + paths = list(map(_path, logsources)) + sizes = [self._storage_impl.size_file(p) for p in paths] + + return sum(size for size in sizes if size is not None) + @only_if_not_done @require_mode('w') def save_metadata(self, contents, allow_overwrite=True, add_attempt=True): diff --git a/metaflow/metadata/metadata.py b/metaflow/metadata/metadata.py index 871015cd441..9fea610bf53 100644 --- a/metaflow/metadata/metadata.py +++ b/metaflow/metadata/metadata.py @@ -1,5 +1,6 @@ import json import os +import re import time from collections import namedtuple from datetime import datetime @@ -14,6 +15,7 @@ MetaDatum = namedtuple('MetaDatum', 'field value type tags') +attempt_id_re = re.compile(r"attempt_id:([0-9]+)") class MetadataProviderMeta(type): def __new__(metaname, classname, bases, attrs): @@ -250,7 +252,8 @@ def stop_heartbeat(self): pass @classmethod - def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters, *args): + def _get_object_internal( + cls, obj_type, obj_order, sub_type, sub_order, filters, attempt, *args): ''' Return objects for the implementation of this class @@ -270,6 +273,13 @@ def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters, Dictionary with keys 'any_tags', 'tags' and 'system_tags'. If specified will return only objects that have the specified tags present. Filters are ANDed together so all tags must be present for the object to be returned. + attempt : int or None + If None, returns artifacts for latest *done* attempt and all metadata. Otherwise, + returns artifacts for that attempt (existent, done or not) and *all* metadata + NOTE: Unlike its external facing `get_object`, this function should + return *all* metadata; the base class will properly implement the + filter. For artifacts, this function should filter artifacts at + the backend level. Return ------ @@ -300,7 +310,7 @@ def add_sticky_tags(self, tags=None, sys_tags=None): self.sticky_sys_tags.update(sys_tags) @classmethod - def get_object(cls, obj_type, sub_type, filters, *args): + def get_object(cls, obj_type, sub_type, filters, attempt, *args): '''Returns the requested object depending on obj_type and sub_type obj_type can be one of 'root', 'flow', 'run', 'step', 'task', @@ -333,6 +343,15 @@ def get_object(cls, obj_type, sub_type, filters, *args): Dictionary with keys 'any_tags', 'tags' and 'system_tags'. If specified will return only objects that have the specified tags present. Filters are ANDed together so all tags must be present for the object to be returned. + attempt : int or None + If None, for metadata and artifacts: + - returns information about the latest attempt for artifacts + - returns all metadata across all attempts + Otherwise, returns information about metadata and artifacts for that + attempt only. + NOTE: For older versions of Metaflow (pre 2.4.0), the attempt for + metadata is not known; in that case, all metadata is returned (as + if None was passed in). Return ------ @@ -366,7 +385,23 @@ def get_object(cls, obj_type, sub_type, filters, *args): if sub_type == 'metadata' and obj_type != 'task': raise MetaflowInternalError(msg='Metadata can only be retrieved at the task level') - return cls._get_object_internal(obj_type, type_order, sub_type, sub_order, filters, *args) + if attempt is not None: + try: + attempt_int = int(attempt) + if attempt_int < 0: + raise ValueError("Attempt can only be positive") + except ValueError: + raise ValueError("Attempt can only be a positive integer") + else: + attempt_int = None + + pre_filter = cls._get_object_internal( + obj_type, type_order, sub_type, sub_order, filters, attempt_int, *args) + if attempt_int is None or sub_order != 6: + # If no attempt or not for metadata, just return as is + return pre_filter + return MetadataProvider._reconstruct_metadata_for_attempt( + pre_filter, attempt_int) def _all_obj_elements(self, tags=None, sys_tags=None): user = get_username() @@ -505,6 +540,39 @@ def _apply_filter(elts, filters): result = [] return starting_point + @staticmethod + def _reconstruct_metadata_for_attempt(all_metadata, attempt_id): + have_all_attempt_id = True + attempts_start = {} + post_filter = [] + for v in all_metadata: + if v['field_name'] == 'attempt': + attempts_start[int(v['value'])] = v['ts_epoch'] + all_tags = v.get('tags') + if all_tags is None: + all_tags = [] + for t in all_tags: + match_result = attempt_id_re.match(t) + if match_result: + if int(match_result.group(1)) == attempt_id: + post_filter.append(v) + break + else: + # We didn't encounter a match for attempt_id + have_all_attempt_id = False + + if not have_all_attempt_id: + # We reconstruct base on the attempts_start + start_ts = attempts_start.get(attempt_id, -1) + if start_ts < 0: + return [] # No metadata since the attempt hasn't started + # Doubt we will be using Python in year 3000 + end_ts = attempts_start.get(attempt_id + 1, 32503680000000) + post_filter = [v for v in all_metadata + if v['ts_epoch'] >= start_ts and v['ts_epoch'] < end_ts] + + return post_filter + def __init__(self, environment, flow, event_logger, monitor): self._task_id_seq = -1 self.sticky_tags = set() diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py index a1a28db9327..0a4c77bbf40 100644 --- a/metaflow/metaflow_config.py +++ b/metaflow/metaflow_config.py @@ -87,7 +87,14 @@ def from_conf(name, default=None): # Path to the client cache CLIENT_CACHE_PATH = from_conf('METAFLOW_CLIENT_CACHE_PATH', '/tmp/metaflow_client') # Maximum size (in bytes) of the cache -CLIENT_CACHE_MAX_SIZE = from_conf('METAFLOW_CLIENT_CACHE_MAX_SIZE', 10000) +CLIENT_CACHE_MAX_SIZE = int(from_conf('METAFLOW_CLIENT_CACHE_MAX_SIZE', 10000)) +# Maximum number of cached Flow and TaskDatastores in the cache +CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT = int(from_conf( + 'METAFLOW_CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT', 50)) +CLIENT_CACHE_MAX_TASKDATASTORE_COUNT = int(from_conf( + 'METAFLOW_CLIENT_CACHE_MAX_TASKDATASTORE_COUNT', + CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT * 100)) + ### # Metadata configuration diff --git a/metaflow/plugins/metadata/local.py b/metaflow/plugins/metadata/local.py index def6bc0f808..8a96ed066d1 100644 --- a/metaflow/plugins/metadata/local.py +++ b/metaflow/plugins/metadata/local.py @@ -97,7 +97,8 @@ def register_metadata(self, run_id, step_name, task_id, metadata): self._save_meta(meta_dir, metadict) @classmethod - def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters, *args): + def _get_object_internal( + cls, obj_type, obj_order, sub_type, sub_order, filters, attempt, *args): from metaflow.datastore.local_storage import LocalStorage if obj_type == 'artifact': # Artifacts are actually part of the tasks in the filesystem @@ -122,11 +123,15 @@ def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters, result = [] if meta_path is None: return result - attempt_done_files = os.path.join(meta_path, 'sysmeta_attempt-done_*') - attempts_done = sorted(glob.iglob(attempt_done_files)) - if attempts_done: - successful_attempt = int(LocalMetadataProvider._read_json_file( - attempts_done[-1])['value']) + + successful_attempt = attempt + if successful_attempt is None: + attempt_done_files = os.path.join(meta_path, 'sysmeta_attempt-done_*') + attempts_done = sorted(glob.iglob(attempt_done_files)) + if attempts_done: + successful_attempt = int(LocalMetadataProvider._read_json_file( + attempts_done[-1])['value']) + if successful_attempt is not None: which_artifact = '*' if len(args) >= sub_order: which_artifact = args[sub_order - 1] diff --git a/metaflow/plugins/metadata/service.py b/metaflow/plugins/metadata/service.py index 96d99d046a6..61c16d148ac 100644 --- a/metaflow/plugins/metadata/service.py +++ b/metaflow/plugins/metadata/service.py @@ -29,6 +29,8 @@ def __init__(self, msg, http_code=None, body=None): class ServiceMetadataProvider(MetadataProvider): TYPE = 'service' + _supports_attempt_gets = None + def __init__(self, environment, flow, event_logger, monitor): super(ServiceMetadataProvider, self).__init__(environment, flow, event_logger, monitor) self.url_task_template = os.path.join(METADATA_SERVICE_URL, @@ -159,10 +161,26 @@ def register_metadata(self, run_id, step_name, task_id, metadata): self._request(self._monitor, url, data) @classmethod - def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters=None, *args): - # Special handling of self, artifact, and metadata + def _get_object_internal( + cls, obj_type, obj_order, sub_type, sub_order, filters, attempt, *args): + if attempt is not None: + if cls._supports_attempt_gets is None: + version = cls._version(None) + cls._supports_attempt_gets = version is not None and \ + LooseVersion(version) >= LooseVersion('2.0.6') + if not cls._supports_attempt_gets: + raise ServiceException( + "Getting specific attempts of Tasks or Artifacts requires " + "the metaflow service to be at least version 2.0.6. Please " + "upgrade your service") + if sub_type == 'self': - url = ServiceMetadataProvider._obj_path(*args[:obj_order]) + if obj_type == 'artifact': + # Special case with the artifacts; we add the attempt + url = ServiceMetadataProvider._obj_path( + *args[:obj_order], attempt=attempt) + else: + url = ServiceMetadataProvider._obj_path(*args[:obj_order]) try: return MetadataProvider._apply_filter([cls._request(None, url)], filters)[0] except ServiceException as ex: @@ -175,10 +193,12 @@ def _get_object_internal(cls, obj_type, obj_order, sub_type, sub_order, filters= url = ServiceMetadataProvider._obj_path(*args[:obj_order]) else: url = '' - if sub_type != 'metadata': - url += '/%ss' % sub_type - else: + if sub_type == 'metadata': url += '/metadata' + elif sub_type == 'artifact' and obj_type == 'task' and attempt is not None: + url += '/attempt/%s/artifacts' % attempt + else: + url += '/%ss' % sub_type try: return MetadataProvider._apply_filter(cls._request(None, url), filters) except ServiceException as ex: @@ -207,16 +227,19 @@ def _new_task(self, @staticmethod def _obj_path( - flow_name, run_id=None, step_name=None, task_id=None, artifact_name=None): + flow_name, run_id=None, step_name=None, task_id=None, + artifact_name=None, attempt=None): object_path = '/flows/%s' % flow_name - if run_id: + if run_id is not None: object_path += '/runs/%s' % run_id - if step_name: + if step_name is not None: object_path += '/steps/%s' % step_name - if task_id: + if task_id is not None: object_path += '/tasks/%s' % task_id - if artifact_name: + if artifact_name is not None: object_path += '/artifacts/%s' % artifact_name + if attempt is not None: + object_path += '/attempt/%s' % attempt return object_path @staticmethod From 1f306072623735018994a6f3d2944647eec98cff Mon Sep 17 00:00:00 2001 From: Savin Date: Mon, 18 Oct 2021 15:27:50 -0700 Subject: [PATCH 073/176] Patch release for Metaflow (#775) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0a13b9aca61..a8c497ce090 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '2.4.0' +version = '2.4.1' setup(name='metaflow', version=version, From a22f96fdc8e8f03edc7111e3fc1187a1f8cc4f13 Mon Sep 17 00:00:00 2001 From: Sakari Ikonen <64256562+saikonen@users.noreply.github.com> Date: Thu, 21 Oct 2021 23:28:06 +0300 Subject: [PATCH 074/176] fix use of renamed _flow_ds_type (#779) --- metaflow/client/filecache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/client/filecache.py b/metaflow/client/filecache.py index 17b1a165974..63f60b30bd9 100644 --- a/metaflow/client/filecache.py +++ b/metaflow/client/filecache.py @@ -77,7 +77,7 @@ def get_log_legacy( ds_cls = self._get_datastore_storage_impl(ds_type) ds_root = ds_cls.path_join(*ds_cls.path_split(location)[:-5]) - cache_id = self._flow_ds_type(ds_type, ds_root, flow_name) + cache_id = self._flow_ds_id(ds_type, ds_root, flow_name) token = '%s.cached' % sha1(os.path.join( run_id, step_name, task_id, '%s_log' % logtype).\ From 444489aff6e4524a2cd085be3bfd4aca3e68f938 Mon Sep 17 00:00:00 2001 From: Romain Date: Thu, 21 Oct 2021 23:23:11 -0700 Subject: [PATCH 075/176] Improved error messages when a problem arises in the datastore (#781) --- metaflow/datastore/content_addressed_store.py | 7 ++++--- metaflow/datastore/datastore_storage.py | 4 ++-- metaflow/datastore/task_datastore.py | 12 +++++++----- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/metaflow/datastore/content_addressed_store.py b/metaflow/datastore/content_addressed_store.py index 931d33f53e9..45a985092cd 100644 --- a/metaflow/datastore/content_addressed_store.py +++ b/metaflow/datastore/content_addressed_store.py @@ -151,7 +151,7 @@ def load_blobs(self, keys, force_raw=False): version = meta.get("cas_version", -1) if version == -1: raise DataException( - "Could not extract encoding version for %s" + "Could not extract encoding version for '%s'" % path ) unpack_code = getattr( @@ -159,7 +159,7 @@ def load_blobs(self, keys, force_raw=False): ) if unpack_code is None: raise DataException( - "Unknown encoding version %d for %s -- " + "Unknown encoding version %d for '%s' -- " "the artifact is either corrupt or you " "need to update Metaflow to the latest " "version" % (version, path) @@ -167,7 +167,8 @@ def load_blobs(self, keys, force_raw=False): try: blob = unpack_code(f) except Exception as e: - raise DataException("Could not unpack data: %s" % e) + raise DataException( + "Could not unpack artifact '%s': %s" % (path, e)) if self._blob_cache: self._blob_cache.store_key(key, blob) diff --git a/metaflow/datastore/datastore_storage.py b/metaflow/datastore/datastore_storage.py index 393d79df0a6..01879e4c5d4 100644 --- a/metaflow/datastore/datastore_storage.py +++ b/metaflow/datastore/datastore_storage.py @@ -91,8 +91,8 @@ def get_datastore_root_from_location(cls, path, flow_name): m = cls.path_rexp.match(path) if not m or m.group('flow_name') != flow_name: raise DataException( - "Location %s does not correspond to a valid location for " - "flow %s." % (path, flow_name)) + "Location '%s' does not correspond to a valid location for " + "flow '%s'." % (path, flow_name)) return m.group('root') @classmethod diff --git a/metaflow/datastore/task_datastore.py b/metaflow/datastore/task_datastore.py index c123dfce399..3d350d4acf8 100644 --- a/metaflow/datastore/task_datastore.py +++ b/metaflow/datastore/task_datastore.py @@ -148,7 +148,8 @@ def __init__(self, data_obj = data_obj[self.METADATA_DATA_SUFFIX] elif self._attempt is None or not allow_not_done: raise DataException( - "Data was not found or not finished at %s" % self._path) + "No completed attempts of the task was found for task '%s'" + % self._path) if data_obj is not None: self._objects = data_obj.get('objects', {}) @@ -316,7 +317,8 @@ def load_artifacts(self, names): """ if not self._info: raise DataException( - "No info object available to retrieve artifacts") + "Datastore for task '%s' does not have the required metadata to " + "load artifacts" % self._path) to_load = [] sha_to_names = {} for name in names: @@ -330,7 +332,7 @@ def load_artifacts(self, names): encode_type = 'gzip+pickle-v2' if encode_type not in self._encodings: raise DataException( - "Python 3.4 or later is required to load %s" % name) + "Python 3.4 or later is required to load artifact '%s'" % name) else: sha = self._objects[name] sha_to_names[sha] = name @@ -766,8 +768,8 @@ def blob_iter(): elif is_stringish(value): yield path, to_fileobj(value) else: - raise DataException("Metadata '%s' has an invalid type: %s" % - (name, type(value))) + raise DataException("Metadata '%s' for task '%s' has an invalid type: %s" % + (name, self._path, type(value))) self._storage_impl.save_bytes(blob_iter(), overwrite=allow_overwrite) def _load_file(self, names, add_attempt=True): From 2c75494052b02435fd0ee3c0c85f65886daf9342 Mon Sep 17 00:00:00 2001 From: Romain Date: Thu, 21 Oct 2021 23:23:34 -0700 Subject: [PATCH 076/176] Fix an issue when using the client for a task that has started but no attempt (#780) has been recorded. --- metaflow/datastore/task_datastore.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metaflow/datastore/task_datastore.py b/metaflow/datastore/task_datastore.py index 3d350d4acf8..a5da8cb93fa 100644 --- a/metaflow/datastore/task_datastore.py +++ b/metaflow/datastore/task_datastore.py @@ -126,6 +126,7 @@ def __init__(self, # datastore, the data may change). We make an exception to that # rule when allow_not_done is True which allows access to things # like logs even for tasks that did not write a done marker + max_attempt = None for i in range(metaflow_config.MAX_ATTEMPTS): check_meta = self._metadata_name_for_attempt( self.METADATA_ATTEMPT_SUFFIX, i) @@ -133,7 +134,7 @@ def __init__(self, max_attempt = i if self._attempt is None: self._attempt = max_attempt - elif self._attempt > max_attempt: + elif max_attempt is None or self._attempt > max_attempt: # In this case, the attempt does not exist so we can't load # anything self._objects = {} From eee086aca996b9e2f122b119dd874cfa83159268 Mon Sep 17 00:00:00 2001 From: Savin Date: Mon, 25 Oct 2021 09:13:48 -0700 Subject: [PATCH 077/176] Revert "Close datastore after task_finished has been invoked (#757)" (#783) This reverts commit 8a54a422a24795639cab02f776f6c03f7d60c855. --- metaflow/task.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metaflow/task.py b/metaflow/task.py index b98a2c8c0ee..b49bf77b0aa 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -496,6 +496,10 @@ def run_step(self, output.save_metadata({'task_end': {}}) output.persist(self.flow) + # this writes a success marker indicating that the + # "transaction" is done + output.done() + # final decorator hook: The task results are now # queryable through the client API / datastore for deco in decorators: @@ -506,10 +510,6 @@ def run_step(self, retry_count, max_user_code_retries) - # this writes a success marker indicating that the - # "transaction" is done - output.done() - # terminate side cars logger.terminate() self.metadata.stop_heartbeat() From 95c5f83d14f2e2d3490efa599366e0172c801e1c Mon Sep 17 00:00:00 2001 From: Savin Date: Mon, 25 Oct 2021 21:40:41 -0700 Subject: [PATCH 078/176] Patch release for Metaflow (#784) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a8c497ce090..2e81d1266cd 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '2.4.1' +version = '2.4.2' setup(name='metaflow', version=version, From 5020ca813dcda128d6ed431a71d6117e0a3f7886 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 27 Oct 2021 23:18:35 -0700 Subject: [PATCH 079/176] Fix a race condition when querying for artifacts of a running task (#789) In some cases, requesting the artifacts through `.artifacts` of a running task may return an error. This patch addresses this issue --- metaflow/client/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/client/core.py b/metaflow/client/core.py index d482e6a7236..f237350269c 100644 --- a/metaflow/client/core.py +++ b/metaflow/client/core.py @@ -972,7 +972,7 @@ def artifacts(self): Container of all DataArtifacts produced by this task """ arts = list(self) - obj = namedtuple('MetaflowArtifacts', [art.id for art in self]) + obj = namedtuple('MetaflowArtifacts', [art.id for art in arts]) return obj._make(arts) @property From ac686b37a07e62418b8628baf49413b6fbe308ab Mon Sep 17 00:00:00 2001 From: Romain Date: Thu, 28 Oct 2021 13:47:08 -0700 Subject: [PATCH 080/176] Fix issue where a "soft" failure would not be retried in the presence of a catch (#776) A soft failure (like an exception) in a step having both catch and retry decorators would not actually cause the step to be first retried before catching the eventual last exception and instead the exception would be caught right away and propagated down. This fixes this issue and adds a test. --- metaflow/plugins/catch_decorator.py | 4 + test/core/tests/catch_retry.py | 129 ++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 test/core/tests/catch_retry.py diff --git a/metaflow/plugins/catch_decorator.py b/metaflow/plugins/catch_decorator.py index fc70d715d39..4950f3876a9 100644 --- a/metaflow/plugins/catch_decorator.py +++ b/metaflow/plugins/catch_decorator.py @@ -77,6 +77,10 @@ def task_exception(self, retry_count, max_user_code_retries): + # Only "catch" exceptions after all retries are exhausted + if retry_count < max_user_code_retries: + return False + if self.attributes['print_exception']: self._print_exception(step, flow) diff --git a/test/core/tests/catch_retry.py b/test/core/tests/catch_retry.py new file mode 100644 index 00000000000..031a30600c3 --- /dev/null +++ b/test/core/tests/catch_retry.py @@ -0,0 +1,129 @@ + +from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag +from metaflow import current + +class CatchRetryTest(MetaflowTest): + PRIORITY = 2 + + @tag('retry(times=3)') + @steps(0, ['start']) + def step_start(self): + import os, sys + self.test_attempt = current.retry_count + sys.stdout.write('stdout testing logs %d\n' % self.test_attempt) + sys.stderr.write('stderr testing logs %d\n' % self.test_attempt) + if self.test_attempt < 3: + self.invisible = True + raise TestRetry() + + # foreach splits don't support @catch but @retry should work + @tag('retry(times=2)') + @steps(0, ['foreach-split']) + def step_split(self): + import os + if current.retry_count == 2: + self.this_is_split = True + else: + raise TestRetry() + + @tag('retry(times=2)') + @steps(0, ['join']) + def step_join(self): + import os + if current.retry_count == 2: + self.test_attempt = inputs[0].test_attempt + else: + raise TestRetry() + + @tag('catch(var="end_ex", print_exception=False)') + @steps(0, ['end'], required=True) + def step_end(self): + from metaflow.exception import ExternalCommandFailed + # make sure we see the latest attempt version of the artifact + assert_equals(3, self.test_attempt) + # the test uses a non-trivial derived exception on purpose + # which is non-trivial to pickle correctly + self.here = True + raise ExternalCommandFailed('catch me!') + + @tag('catch(var="ex", print_exception=False)') + @tag('retry(times=2)') + @steps(1, ['all']) + def step_all(self): + # Die a soft death; this should retry and then catch in the end + self.retry_with_catch = current.retry_count + raise TestRetry() + + def check_results(self, flow, checker): + + checker.assert_log('start', 'stdout', 'stdout testing logs 3\n', exact_match=False) + checker.assert_log('start', 'stderr', 'stderr testing logs 3\n', exact_match=False) + + for step in flow: + + if step.name == 'start': + checker.assert_artifact('start', 'test_attempt', 3) + try: + for task in checker.artifact_dict('start', + 'invisible').values(): + if task: + raise Exception("'invisible' should not be visible "\ + "in 'start'") + except KeyError: + pass + elif step.name == 'end': + checker.assert_artifact('end', 'test_attempt', 3) + for task in checker.artifact_dict(step.name, 'end_ex').values(): + assert_equals('catch me!', str(task['end_ex'].exception)) + break + else: + raise Exception("No artifact 'end_ex' in step 'end'") + + elif flow._graph[step.name].type == 'foreach': + checker.assert_artifact(step.name, 'this_is_split', True) + + elif flow._graph[step.name].type == 'join': + checker.assert_artifact('end', 'test_attempt', 3) + + else: + for task in checker.artifact_dict(step.name, 'ex').values(): + extype = 'metaflow_test.TestRetry' + assert_equals(extype, str(task['ex'].type)) + break + else: + raise Exception("No artifact 'ex' in step '%s'" % step.name) + for task in checker.artifact_dict(step.name, 'retry_with_catch').values(): + assert_equals(task['retry_with_catch'], 2) + break + else: + raise Exception("No artifact 'retry_with_catch' in step '%s'" % step.name) + + run = checker.get_run() + if run: + for step in run: + if step.id == 'end': + continue + if flow._graph[step.id].type in ('foreach', 'join'): + # 1 normal run + 2 retries = 3 attempts + attempts = 3 + elif step.id == 'start': + attempts = 4 # 1 normal run + 3 retries = 4 attempts + else: + # 1 normal run + 2 retries = 3 attempts + attempts = 3 + for task in step: + data = task.data + got = sorted(m.value for m in task.metadata + if m.type == 'attempt') + assert_equals(list(map(str, range(attempts))), got) + + assert_equals(False, 'invisible' in run['start'].task.data) + assert_equals(3, run['start'].task.data.test_attempt) + end = run['end'].task + assert_equals(True, end.data.here) + assert_equals(3, end.data.test_attempt) + # task.exception is None since the exception was handled + assert_equals(None, end.exception) + assert_equals('catch me!', end.data.end_ex.exception) + assert_equals('metaflow.exception.ExternalCommandFailed', + end.data.end_ex.type) From cc829c4e84efc70da9a72cb714a8a342e073c44c Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 28 Oct 2021 22:48:44 +0200 Subject: [PATCH 081/176] PEP8: Avoid bare except (#717) --- metaflow/parameters.py | 2 +- metaflow/util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metaflow/parameters.py b/metaflow/parameters.py index 109bad3369a..7f0df8088ca 100644 --- a/metaflow/parameters.py +++ b/metaflow/parameters.py @@ -11,7 +11,7 @@ try: # Python2 strtype = basestring -except: +except NameError: # Python3 strtype = str diff --git a/metaflow/util.py b/metaflow/util.py index a577f7f6d09..360cf3cac77 100644 --- a/metaflow/util.py +++ b/metaflow/util.py @@ -39,7 +39,7 @@ def __str__(self): return self.path from pipes import quote as _quote -except: +except NameError: # python3 unicode_type = str bytes_type = bytes From 4a522017f7b5952e62f36a3e7217ed9d3c1ca540 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Thu, 28 Oct 2021 23:50:00 +0300 Subject: [PATCH 082/176] Upgrade pandas version in tutorials to 1.3.3 (#707) * Upgrade pandas version in tutorials to 1.3.3 * Update tutorial 06 to use pandas 1.3.3 --- metaflow/tutorials/02-statistics/README.md | 6 +++--- metaflow/tutorials/04-playlist-plus/playlist.py | 10 +++++----- metaflow/tutorials/06-statistics-redux/README.md | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/metaflow/tutorials/02-statistics/README.md b/metaflow/tutorials/02-statistics/README.md index bbf0b421f94..0ee1f8274ff 100644 --- a/metaflow/tutorials/02-statistics/README.md +++ b/metaflow/tutorials/02-statistics/README.md @@ -6,8 +6,8 @@ later examples to improve our playlist generator. You can optionally use the Metaflow client to eyeball the results in a Notebook, and make some simple plots using the Matplotlib library.** -Please note that Episode 04, a follow-on to this episode, requires Pandas version 0.24.2. -Please make sure that you install or upgrade/downgrade to Pandas 0.24.2. +Please note that Episode 04, a follow-on to this episode, requires Pandas version 1.3.3. +Please make sure that you install or upgrade/downgrade to Pandas 1.3.3. #### Showcasing: - Fan-out over a set of parameters using Metaflow foreach. @@ -15,7 +15,7 @@ Please make sure that you install or upgrade/downgrade to Pandas 0.24.2. - Plotting results in a Notebook. #### Before playing this episode: -1. ```python -m pip install pandas==0.24.2``` +1. ```python -m pip install pandas==1.3.3``` 2. ```python -m pip install notebook``` 3. ```python -m pip install matplotlib``` diff --git a/metaflow/tutorials/04-playlist-plus/playlist.py b/metaflow/tutorials/04-playlist-plus/playlist.py index f858940cddb..81a4e371110 100644 --- a/metaflow/tutorials/04-playlist-plus/playlist.py +++ b/metaflow/tutorials/04-playlist-plus/playlist.py @@ -44,7 +44,7 @@ class PlayListFlow(FlowSpec): "the playlist.", default=5) - @conda(libraries={'pandas' : '0.24.2'}) + @conda(libraries={'pandas' : '1.3.3'}) @step def start(self): """ @@ -52,7 +52,7 @@ def start(self): MovieStatsFlow and assign them as data artifacts in this flow. This step uses 'conda' to isolate the environment. This step will - always use pandas==0.24.2 regardless of what is installed on the + always use pandas==1.3.3 regardless of what is installed on the system. """ @@ -76,7 +76,7 @@ def start(self): # Compute our two recommendation types in parallel. self.next(self.bonus_movie, self.genre_movies) - @conda(libraries={'editdistance': '0.5.3', 'pandas' : '0.24.2'}) + @conda(libraries={'editdistance': '0.5.3', 'pandas' : '1.3.3'}) @step def bonus_movie(self): """ @@ -105,14 +105,14 @@ def _edit_distance(movie_title): self.next(self.join) - @conda(libraries={'pandas' : '0.24.2'}) + @conda(libraries={'pandas' : '1.3.3'}) @step def genre_movies(self): """ Select the top performing movies from the use specified genre. This step uses 'conda' to isolate the environment. This step will - always use pandas==0.24.2 regardless of what is installed on the + always use pandas==1.3.3 regardless of what is installed on the system. """ diff --git a/metaflow/tutorials/06-statistics-redux/README.md b/metaflow/tutorials/06-statistics-redux/README.md index 1c9e19a7cb0..ef394b46147 100644 --- a/metaflow/tutorials/06-statistics-redux/README.md +++ b/metaflow/tutorials/06-statistics-redux/README.md @@ -10,8 +10,8 @@ behavior with additional arguments, like '--max-workers'. For this example, computations. You can then access the data artifacts (even the local CSV file) from anywhere because the data is being stored in AWS S3. -This tutorial uses `pandas` which may not be available in your environment. -Use the 'conda' package manager with the `conda-forge` channel added to run +This tutorial uses `pandas` which may not be available in your environment. +Use the 'conda' package manager with the `conda-forge` channel added to run this tutorial in any environment** #### Showcasing: @@ -30,15 +30,15 @@ this tutorial in any environment** a. Download Miniconda at https://docs.conda.io/en/latest/miniconda.html b. ```conda config --add channels conda-forge``` 5. This tutorial requires access to compute and storage resources on AWS, which - can be configured by - a. Following the instructions at + can be configured by + a. Following the instructions at https://docs.metaflow.org/metaflow-on-aws/deploy-to-aws or - b. Requesting a sandbox at + b. Requesting a sandbox at https://docs.metaflow.org/metaflow-on-aws/metaflow-sandbox #### To play this episode: 1. ```cd metaflow-tutorials``` -2. ```python 02-statistics/stats.py --environment conda run --with batch --max-workers 4 --with conda:python=3.7,libraries="{pandas:0.24.2}"``` +2. ```python 02-statistics/stats.py --environment conda run --with batch --max-workers 4 --with conda:python=3.7,libraries="{pandas:1.3.3}"``` 3. ```jupyter-notebook 06-statistics-redux/stats.ipynb``` 4. Open 'stats.ipynb' in your remote Sagemaker notebook From 5186c6c5bba36d9e77077413ee2495dc79da3dca Mon Sep 17 00:00:00 2001 From: Savin Date: Thu, 28 Oct 2021 14:05:57 -0700 Subject: [PATCH 083/176] Setup black as formatter in GH actions (#771) * Setup linter in GH actions * apply black on all files * run tests on actions/black branch * revert test execution * Update test.yml * Update test.yml * Update test.yml * :art: Format Python code with psf/black (#772) Co-authored-by: savingoyal Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: savingoyal --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 25 +- metaflow/R.py | 84 +- metaflow/__init__.py | 140 +- metaflow/cli.py | 1223 +++++++++-------- metaflow/cli_args.py | 24 +- metaflow/client/__init__.py | 28 +- metaflow/client/core.py | 346 +++-- metaflow/client/filecache.py | 208 ++- metaflow/cmd_with_io.py | 10 +- metaflow/current.py | 30 +- metaflow/datastore/__init__.py | 3 +- metaflow/datastore/content_addressed_store.py | 18 +- metaflow/datastore/datastore_set.py | 27 +- metaflow/datastore/datastore_storage.py | 42 +- metaflow/datastore/exceptions.py | 5 +- metaflow/datastore/flow_datastore.py | 103 +- metaflow/datastore/inputs.py | 1 + metaflow/datastore/local_storage.py | 27 +- metaflow/datastore/s3_storage.py | 56 +- metaflow/datastore/task_datastore.py | 350 +++-- metaflow/datatools/__init__.py | 42 +- metaflow/datatools/s3.py | 499 ++++--- metaflow/datatools/s3op.py | 651 +++++---- metaflow/datatools/s3tail.py | 33 +- metaflow/datatools/s3util.py | 38 +- metaflow/debug.py | 10 +- metaflow/decorators.py | 260 ++-- metaflow/event_logger.py | 4 +- metaflow/exception.py | 78 +- metaflow/flowspec.py | 264 ++-- metaflow/graph.py | 116 +- metaflow/includefile.py | 195 +-- metaflow/lint.py | 225 +-- metaflow/main_cli.py | 905 ++++++------ metaflow/metadata/__init__.py | 2 +- metaflow/metadata/heartbeat.py | 27 +- metaflow/metadata/metadata.py | 291 ++-- metaflow/metadata/util.py | 14 +- metaflow/metaflow_config.py | 173 +-- metaflow/metaflow_environment.py | 76 +- metaflow/metaflow_profile.py | 5 +- metaflow/metaflow_version.py | 44 +- metaflow/mflog/__init__.py | 87 +- metaflow/mflog/mflog.py | 78 +- metaflow/mflog/save_logs.py | 54 +- metaflow/mflog/save_logs_periodically.py | 5 +- metaflow/mflog/tee.py | 10 +- metaflow/monitor.py | 65 +- metaflow/multicore_utils.py | 18 +- metaflow/package.py | 54 +- metaflow/parameters.py | 156 ++- metaflow/plugins/__init__.py | 142 +- metaflow/plugins/aws/aws_client.py | 34 +- metaflow/plugins/aws/aws_utils.py | 3 +- metaflow/plugins/aws/batch/batch.py | 144 +- metaflow/plugins/aws/batch/batch_cli.py | 55 +- metaflow/plugins/aws/batch/batch_client.py | 387 +++--- metaflow/plugins/aws/batch/batch_decorator.py | 283 ++-- metaflow/plugins/aws/eks/kubernetes.py | 72 +- metaflow/plugins/aws/eks/kubernetes_cli.py | 47 +- metaflow/plugins/aws/eks/kubernetes_client.py | 9 +- .../plugins/aws/eks/kubernetes_decorator.py | 138 +- .../aws/step_functions/dynamo_db_client.py | 86 +- .../aws/step_functions/event_bridge_client.py | 36 +- .../aws/step_functions/production_token.py | 26 +- .../aws/step_functions/schedule_decorator.py | 37 +- .../step_functions/set_batch_environment.py | 28 +- .../aws/step_functions/step_functions.py | 961 +++++++------ .../aws/step_functions/step_functions_cli.py | 646 +++++---- .../step_functions/step_functions_client.py | 110 +- .../step_functions_decorator.py | 103 +- metaflow/plugins/catch_decorator.py | 67 +- metaflow/plugins/conda/__init__.py | 6 +- metaflow/plugins/conda/batch_bootstrap.py | 40 +- metaflow/plugins/conda/conda.py | 173 +-- metaflow/plugins/conda/conda_environment.py | 54 +- .../plugins/conda/conda_flow_decorator.py | 26 +- .../plugins/conda/conda_step_decorator.py | 293 ++-- metaflow/plugins/debug_logger.py | 4 +- metaflow/plugins/debug_monitor.py | 16 +- metaflow/plugins/env_escape/__init__.py | 62 +- metaflow/plugins/env_escape/client.py | 33 +- metaflow/plugins/env_escape/client_modules.py | 36 +- .../emulate_test_lib/overrides.py | 2 + .../emulate_test_lib/server_mappings.py | 5 +- .../configurations/test_lib_impl/__init__.py | 2 +- .../plugins/env_escape/data_transferer.py | 29 +- metaflow/plugins/env_escape/server.py | 12 +- metaflow/plugins/env_escape/stub.py | 61 +- metaflow/plugins/environment_decorator.py | 11 +- metaflow/plugins/metadata/__init__.py | 2 +- metaflow/plugins/metadata/local.py | 188 ++- metaflow/plugins/metadata/service.py | 313 +++-- metaflow/plugins/package_cli.py | 65 +- metaflow/plugins/project_decorator.py | 125 +- metaflow/plugins/resources_decorator.py | 16 +- metaflow/plugins/retry_decorator.py | 16 +- .../test_unbounded_foreach_decorator.py | 104 +- metaflow/plugins/timeout_decorator.py | 78 +- metaflow/procpoll.py | 84 +- metaflow/pylint_wrapper.py | 12 +- metaflow/runtime.py | 702 +++++----- metaflow/sidecar.py | 61 +- metaflow/sidecar_messages.py | 11 +- metaflow/sidecar_worker.py | 3 +- metaflow/task.py | 494 ++++--- .../tutorials/00-helloworld/helloworld.py | 3 +- metaflow/tutorials/01-playlist/playlist.py | 61 +- metaflow/tutorials/02-statistics/stats.py | 36 +- .../tutorials/03-playlist-redux/playlist.py | 41 +- .../tutorials/04-playlist-plus/playlist.py | 57 +- metaflow/tutorials/05-helloaws/helloaws.py | 5 +- metaflow/unbounded_foreach.py | 7 +- metaflow/util.py | 99 +- setup.py | 39 +- .../exceptions/__init__.py | 7 +- .../metaflow_extensions/plugins/__init__.py | 4 +- .../plugins/flow_options.py | 16 +- .../plugins/nondecoplugin/__init__.py | 2 +- .../plugins/test_step_decorator.py | 6 +- .../metaflow_extensions/toplevel/toplevel.py | 4 +- test/core/metaflow_test/__init__.py | 44 +- test/core/metaflow_test/cli_check.py | 79 +- test/core/metaflow_test/formatter.py | 143 +- test/core/metaflow_test/metadata_check.py | 81 +- test/core/run_tests.py | 346 ++--- test/core/tests/basic_artifact.py | 15 +- test/core/tests/basic_foreach.py | 9 +- test/core/tests/basic_include.py | 46 +- test/core/tests/basic_log.py | 41 +- test/core/tests/basic_parameters.py | 39 +- test/core/tests/basic_tags.py | 48 +- test/core/tests/basic_unbounded_foreach.py | 22 +- test/core/tests/catch_retry.py | 106 +- test/core/tests/current_singleton.py | 34 +- test/core/tests/detect_segfault.py | 17 +- test/core/tests/dynamic_parameters.py | 22 +- test/core/tests/extensions.py | 14 +- test/core/tests/flow_options.py | 6 +- test/core/tests/large_artifact.py | 21 +- test/core/tests/large_mflog.py | 59 +- test/core/tests/lineage.py | 16 +- test/core/tests/merge_artifacts.py | 68 +- test/core/tests/merge_artifacts_include.py | 48 +- .../core/tests/merge_artifacts_propagation.py | 23 +- test/core/tests/nested_foreach.py | 11 +- test/core/tests/nested_unbounded_foreach.py | 36 +- test/core/tests/param_names.py | 12 +- test/core/tests/project_branch.py | 14 +- test/core/tests/project_production.py | 13 +- test/core/tests/resume_end_step.py | 25 +- test/core/tests/resume_foreach_inner.py | 51 +- test/core/tests/resume_foreach_join.py | 47 +- test/core/tests/resume_foreach_split.py | 51 +- test/core/tests/resume_start_step.py | 25 +- test/core/tests/s3_failure.py | 25 +- test/core/tests/tag_catch.py | 93 +- test/core/tests/task_exception.py | 14 +- test/core/tests/timeout_decorator.py | 16 +- test/core/tests/wide_foreach.py | 16 +- test/data/__init__.py | 5 +- test/data/s3/__init__.py | 1 - test/data/s3/s3_data.py | 390 ++++-- test/data/s3/test_s3.py | 297 ++-- test/env_escape/example.py | 23 +- test/unit/test_k8s_job_name_sanitizer.py | 23 +- test/unit/test_k8s_label_sanitizer.py | 18 +- 168 files changed, 9249 insertions(+), 7475 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c903616c649..de76e366223 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -4,7 +4,7 @@ on: types: [published] jobs: test: - name: ${{ matrix.lang }} tests on ${{ matrix.os }} + name: tests / ${{ matrix.lang }} tests on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 28ad3cf0404..80b759816cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,8 +7,31 @@ on: branches: - master jobs: + formatter: + name: runner / black + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Check files using the black formatter + uses: rickstaa/action-black@v1 + id: action_black + with: + black_args: "." + - name: Create Pull Request + if: steps.action_black.outputs.is_formatted == 'true' + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + title: "Format Python code with psf/black push" + commit-message: ":art: Format Python code with psf/black" + body: | + There appear to be some python formatting errors in #${{ github.event.number }} - ${{ github.sha }}. + This pull request uses the [psf/black](https://github.com/psf/black) formatter to fix these issues. + `black .` + base: ${{ github.head_ref }} + branch: actions/black test: - name: ${{ matrix.lang }} tests on ${{ matrix.os }} + name: tests / ${{ matrix.lang }} tests on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: diff --git a/metaflow/R.py b/metaflow/R.py index e78ba61d890..e24dbf5c2bc 100644 --- a/metaflow/R.py +++ b/metaflow/R.py @@ -12,67 +12,77 @@ R_VERSION = None R_VERSION_CODE = None + def call_r(func_name, args): R_FUNCTIONS[func_name](*args) + def get_r_func(func_name): return R_FUNCTIONS[func_name] + def package_paths(): if R_PACKAGE_PATHS is not None: - root = R_PACKAGE_PATHS['package'] - prefixlen = len('%s/' % root.rstrip('/')) - for path, dirs, files in os.walk(R_PACKAGE_PATHS['package']): - if '/.' in path: + root = R_PACKAGE_PATHS["package"] + prefixlen = len("%s/" % root.rstrip("/")) + for path, dirs, files in os.walk(R_PACKAGE_PATHS["package"]): + if "/." in path: continue for fname in files: - if fname[0] == '.': + if fname[0] == ".": continue p = os.path.join(path, fname) - yield p, os.path.join('metaflow-r', p[prefixlen:]) - flow = R_PACKAGE_PATHS['flow'] + yield p, os.path.join("metaflow-r", p[prefixlen:]) + flow = R_PACKAGE_PATHS["flow"] yield flow, os.path.basename(flow) + def entrypoint(): - return 'PYTHONPATH=/root/metaflow R_LIBS_SITE=`Rscript -e \'cat(paste(.libPaths(), collapse=\\":\\"))\'`:metaflow/ Rscript metaflow-r/run_batch.R --flowRDS=%s' % RDS_FILE_PATH + return ( + "PYTHONPATH=/root/metaflow R_LIBS_SITE=`Rscript -e 'cat(paste(.libPaths(), collapse=\\\":\\\"))'`:metaflow/ Rscript metaflow-r/run_batch.R --flowRDS=%s" + % RDS_FILE_PATH + ) + def use_r(): return R_PACKAGE_PATHS is not None + def container_image(): return R_CONTAINER_IMAGE + def metaflow_r_version(): return METAFLOW_R_VERSION + def r_version(): return R_VERSION + def r_version_code(): return R_VERSION_CODE + def working_dir(): - if use_r(): - return R_PACKAGE_PATHS['wd'] + if use_r(): + return R_PACKAGE_PATHS["wd"] return None -def run(flow_script, - r_functions, - rds_file, - metaflow_args, - full_cmdline, - r_paths, - r_container_image, - metaflow_r_version, - r_version, - r_version_code): - global R_FUNCTIONS, \ - R_PACKAGE_PATHS, \ - RDS_FILE_PATH, \ - R_CONTAINER_IMAGE, \ - METAFLOW_R_VERSION, \ - R_VERSION, \ - R_VERSION_CODE + +def run( + flow_script, + r_functions, + rds_file, + metaflow_args, + full_cmdline, + r_paths, + r_container_image, + metaflow_r_version, + r_version, + r_version_code, +): + global R_FUNCTIONS, R_PACKAGE_PATHS, RDS_FILE_PATH, R_CONTAINER_IMAGE, METAFLOW_R_VERSION, R_VERSION, R_VERSION_CODE R_FUNCTIONS = r_functions R_PACKAGE_PATHS = r_paths @@ -90,25 +100,29 @@ def run(flow_script, full_cmdline[0] = os.path.basename(full_cmdline[0]) with NamedTemporaryFile(prefix="metaflowR.", delete=False) as tmp: tmp.write(to_bytes(flow_script)) - module = imp.load_source('metaflowR', tmp.name) + module = imp.load_source("metaflowR", tmp.name) flow = module.FLOW(use_cli=False) - from . import exception - from . import cli + from . import exception + from . import cli + try: - cli.main(flow, - args=metaflow_args, - handle_exceptions=False, - entrypoint=full_cmdline[:-len(metaflow_args)]) + cli.main( + flow, + args=metaflow_args, + handle_exceptions=False, + entrypoint=full_cmdline[: -len(metaflow_args)], + ) except exception.MetaflowException as e: cli.print_metaflow_exception(e) os.remove(tmp.name) os._exit(1) except Exception as e: import sys + print(e) sys.stdout.flush() os.remove(tmp.name) os._exit(1) finally: - os.remove(tmp.name) \ No newline at end of file + os.remove(tmp.name) diff --git a/metaflow/__init__.py b/metaflow/__init__.py index ac138a1fb67..0ef30d353e4 100644 --- a/metaflow/__init__.py +++ b/metaflow/__init__.py @@ -73,15 +73,16 @@ def __init__(self, handled): def find_module(self, fullname, path=None): if fullname in self._tempexcluded: return None - if fullname in self._handled or \ - (fullname.endswith('._orig') and fullname[:-6] in self._handled): + if fullname in self._handled or ( + fullname.endswith("._orig") and fullname[:-6] in self._handled + ): return self - name_parts = fullname.split('.') - if len(name_parts) > 1 and name_parts[-1] != '_orig': + name_parts = fullname.split(".") + if len(name_parts) > 1 and name_parts[-1] != "_orig": # We check if we had an alias created for this module and if so, # we are going to load it to properly fully create aliases all # the way down. - parent_name = '.'.join(name_parts[:-1]) + parent_name = ".".join(name_parts[:-1]) if parent_name in self._alias_to_orig: return self return None @@ -89,18 +90,19 @@ def find_module(self, fullname, path=None): def load_module(self, fullname): if fullname in sys.modules: return sys.modules[fullname] - if not self._can_handle_orig_module() and fullname.endswith('._orig'): + if not self._can_handle_orig_module() and fullname.endswith("._orig"): # We return a nicer error message raise ImportError( "Attempting to load '%s' -- loading shadowed modules in Metaflow " - "Extensions are only supported in Python 3.4+" % fullname) + "Extensions are only supported in Python 3.4+" % fullname + ) to_import = self._handled.get(fullname, None) # If to_import is None, two cases: # - we are loading a ._orig module # - OR we are loading a submodule if to_import is None: - if fullname.endswith('._orig'): + if fullname.endswith("._orig"): try: # We exclude this module temporarily from what we handle to # revert back to the non-shadowing mode of import @@ -109,18 +111,18 @@ def load_module(self, fullname): finally: self._tempexcluded.remove(fullname) else: - name_parts = fullname.split('.') + name_parts = fullname.split(".") submodule = name_parts[-1] - parent_name = '.'.join(name_parts[:-1]) - to_import = '.'.join( - [self._alias_to_orig[parent_name], submodule]) + parent_name = ".".join(name_parts[:-1]) + to_import = ".".join([self._alias_to_orig[parent_name], submodule]) if isinstance(to_import, str): try: to_import_mod = importlib.import_module(to_import) except ImportError: raise ImportError( - "No module found '%s' (aliasing %s)" % (fullname, to_import)) + "No module found '%s' (aliasing %s)" % (fullname, to_import) + ) sys.modules[fullname] = to_import_mod self._alias_to_orig[fullname] = to_import_mod.__name__ elif isinstance(to_import, types.ModuleType): @@ -131,14 +133,15 @@ def load_module(self, fullname): m = importlib.util.module_from_spec(to_import) to_import.loader.exec_module(m) sys.modules[fullname] = m - elif to_import is None and fullname.endswith('._orig'): + elif to_import is None and fullname.endswith("._orig"): # This happens when trying to access a shadowed ._orig module # when actually, there is no shadowed module; print a nicer message # Condition is a bit overkill and most likely only checking to_import # would be OK. Being extra sure in case _LazyLoader is misused and # a None value is passed in. raise ImportError( - "Metaflow Extensions shadowed module '%s' does not exist" % fullname) + "Metaflow Extensions shadowed module '%s' does not exist" % fullname + ) else: raise ImportError return sys.modules[fullname] @@ -147,6 +150,7 @@ def load_module(self, fullname): def _can_handle_orig_module(): return sys.version_info[0] >= 3 and sys.version_info[1] >= 4 + # We load the module overrides *first* explicitly. Non overrides can be loaded # in toplevel as well but these can be loaded first if needed. Note that those # modules should be careful not to include anything in Metaflow at their top-level @@ -159,20 +163,30 @@ def _can_handle_orig_module(): # e.name is set to the name of the package that fails to load # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) - if not (isinstance(e, ModuleNotFoundError) and \ - e.name in ['metaflow_extensions', 'metaflow_extensions.toplevel', - 'metaflow_extensions.toplevel.module_overrides']): + if not ( + isinstance(e, ModuleNotFoundError) + and e.name + in [ + "metaflow_extensions", + "metaflow_extensions.toplevel", + "metaflow_extensions.toplevel.module_overrides", + ] + ): print( "Cannot load metaflow_extensions top-level configuration -- " - "if you want to ignore, uninstall metaflow_extensions package") + "if you want to ignore, uninstall metaflow_extensions package" + ) raise else: # We load only modules lazy_load_custom_modules = {} for n, o in extension_module.__dict__.items(): - if isinstance(o, types.ModuleType) and o.__package__ and \ - o.__package__.startswith('metaflow_extensions'): - lazy_load_custom_modules['metaflow.%s' % n] = o + if ( + isinstance(o, types.ModuleType) + and o.__package__ + and o.__package__.startswith("metaflow_extensions") + ): + lazy_load_custom_modules["metaflow.%s" % n] = o if lazy_load_custom_modules: # Prepend to make sure extensions package overrides things sys.meta_path = [_LazyLoader(lazy_load_custom_modules)] + sys.meta_path @@ -183,6 +197,7 @@ def _can_handle_orig_module(): from .flowspec import FlowSpec from .includefile import IncludeFile from .parameters import Parameter, JSONTypeClass + JSONType = JSONTypeClass() # current runtime singleton @@ -193,27 +208,29 @@ def _can_handle_orig_module(): # Decorators from .decorators import step, _import_plugin_decorators + # this auto-generates decorator functions from Decorator objects # in the top-level metaflow namespace _import_plugin_decorators(globals()) # Client -from .client import namespace,\ - get_namespace,\ - default_namespace,\ - metadata, \ - get_metadata, \ - default_metadata, \ - Metaflow,\ - Flow,\ - Run,\ - Step,\ - Task,\ - DataArtifact +from .client import ( + namespace, + get_namespace, + default_namespace, + metadata, + get_metadata, + default_metadata, + Metaflow, + Flow, + Run, + Step, + Task, + DataArtifact, +) # Utilities -from .multicore_utils import parallel_imap_unordered,\ - parallel_map +from .multicore_utils import parallel_imap_unordered, parallel_map from .metaflow_profile import profile # Now override everything other than modules @@ -226,12 +243,19 @@ def _can_handle_orig_module(): # e.name is set to the name of the package that fails to load # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) - if not (isinstance(e, ModuleNotFoundError) and \ - e.name in ['metaflow_extensions', 'metaflow_extensions.toplevel', - 'metaflow_extensions.toplevel.toplevel']): + if not ( + isinstance(e, ModuleNotFoundError) + and e.name + in [ + "metaflow_extensions", + "metaflow_extensions.toplevel", + "metaflow_extensions.toplevel.toplevel", + ] + ): print( "Cannot load metaflow_extensions top-level configuration -- " - "if you want to ignore, uninstall metaflow_extensions package") + "if you want to ignore, uninstall metaflow_extensions package" + ) raise else: # We load into globals whatever we have in extension_module @@ -239,38 +263,50 @@ def _can_handle_orig_module(): # *except* for ones that are part of metaflow_extensions (basically providing # an aliasing mechanism) lazy_load_custom_modules = {} - addl_modules = extension_module.__dict__.get('__mf_promote_submodules__') + addl_modules = extension_module.__dict__.get("__mf_promote_submodules__") if addl_modules: # We make an alias for these modules which the metaflow_extensions author # wants to expose but that may not be loaded yet lazy_load_custom_modules = { - 'metaflow.%s' % k: 'metaflow_extensions.%s' % k for k in addl_modules} + "metaflow.%s" % k: "metaflow_extensions.%s" % k for k in addl_modules + } for n, o in extension_module.__dict__.items(): - if not n.startswith('__') and not isinstance(o, types.ModuleType): + if not n.startswith("__") and not isinstance(o, types.ModuleType): globals()[n] = o - elif isinstance(o, types.ModuleType) and o.__package__ and \ - o.__package__.startswith('metaflow_extensions'): - lazy_load_custom_modules['metaflow.%s' % n] = o + elif ( + isinstance(o, types.ModuleType) + and o.__package__ + and o.__package__.startswith("metaflow_extensions") + ): + lazy_load_custom_modules["metaflow.%s" % n] = o if lazy_load_custom_modules: # Prepend to make sure custom package overrides things sys.meta_path = [_LazyLoader(lazy_load_custom_modules)] + sys.meta_path - __version_addl__ = getattr(extension_module, '__mf_extensions__', '') + __version_addl__ = getattr(extension_module, "__mf_extensions__", "") if extension_module.__version__: - __version_addl__ = '%s(%s)' % (__version_addl__, extension_module.__version__) + __version_addl__ = "%s(%s)" % (__version_addl__, extension_module.__version__) # Erase all temporary names to avoid leaking things -for _n in ['ver', 'n', 'o', 'e', 'lazy_load_custom_modules', - 'extension_module', 'addl_modules']: +for _n in [ + "ver", + "n", + "o", + "e", + "lazy_load_custom_modules", + "extension_module", + "addl_modules", +]: try: del globals()[_n] except KeyError: pass -del globals()['_n'] +del globals()["_n"] import pkg_resources + try: - __version__ = pkg_resources.get_distribution('metaflow').version + __version__ = pkg_resources.get_distribution("metaflow").version except: # this happens on remote environments since the job package # does not have a version diff --git a/metaflow/cli.py b/metaflow/cli.py index 7f48b8117e9..cac4c75737c 100644 --- a/metaflow/cli.py +++ b/metaflow/cli.py @@ -15,8 +15,13 @@ from . import namespace from . import current from .cli_args import cli_args -from .util import resolve_identity, decompress_list, write_latest_run_id, \ - get_latest_run_id, to_unicode +from .util import ( + resolve_identity, + decompress_list, + write_latest_run_id, + get_latest_run_id, + to_unicode, +) from .task import MetaflowTask from .exception import CommandException, MetaflowException from .graph import FlowGraph @@ -24,9 +29,20 @@ from .runtime import NativeRuntime from .package import MetaflowPackage -from .plugins import ENVIRONMENTS, LOGGING_SIDECARS, METADATA_PROVIDERS, MONITOR_SIDECARS -from .metaflow_config import DEFAULT_DATASTORE, DEFAULT_ENVIRONMENT, DEFAULT_EVENT_LOGGER, \ - DEFAULT_METADATA, DEFAULT_MONITOR, DEFAULT_PACKAGE_SUFFIXES +from .plugins import ( + ENVIRONMENTS, + LOGGING_SIDECARS, + METADATA_PROVIDERS, + MONITOR_SIDECARS, +) +from .metaflow_config import ( + DEFAULT_DATASTORE, + DEFAULT_ENVIRONMENT, + DEFAULT_EVENT_LOGGER, + DEFAULT_METADATA, + DEFAULT_MONITOR, + DEFAULT_PACKAGE_SUFFIXES, +) from .metaflow_environment import MetaflowEnvironment from .pylint_wrapper import PyLint from .event_logger import EventLogger @@ -36,13 +52,13 @@ from .unbounded_foreach import UBF_CONTROL, UBF_TASK -ERASE_TO_EOL = '\033[K' -HIGHLIGHT = 'red' -INDENT = ' ' * 4 +ERASE_TO_EOL = "\033[K" +HIGHLIGHT = "red" +INDENT = " " * 4 -LOGGER_TIMESTAMP = 'magenta' -LOGGER_COLOR = 'green' -LOGGER_BAD_COLOR = 'red' +LOGGER_TIMESTAMP = "magenta" +LOGGER_COLOR = "green" +LOGGER_BAD_COLOR = "red" try: # Python 2 @@ -57,62 +73,56 @@ def echo_dev_null(*args, **kwargs): def echo_always(line, **kwargs): - kwargs['err'] = kwargs.get('err', True) - if kwargs.pop('indent', None): - line = '\n'.join(INDENT + x for x in line.splitlines()) - if 'nl' not in kwargs or kwargs['nl']: + kwargs["err"] = kwargs.get("err", True) + if kwargs.pop("indent", None): + line = "\n".join(INDENT + x for x in line.splitlines()) + if "nl" not in kwargs or kwargs["nl"]: line += ERASE_TO_EOL - top = kwargs.pop('padding_top', None) - bottom = kwargs.pop('padding_bottom', None) - highlight = kwargs.pop('highlight', HIGHLIGHT) + top = kwargs.pop("padding_top", None) + bottom = kwargs.pop("padding_bottom", None) + highlight = kwargs.pop("highlight", HIGHLIGHT) if top: click.secho(ERASE_TO_EOL, **kwargs) - hl_bold = kwargs.pop('highlight_bold', True) - nl = kwargs.pop('nl', True) - fg = kwargs.pop('fg', None) - bold = kwargs.pop('bold', False) - kwargs['nl'] = False + hl_bold = kwargs.pop("highlight_bold", True) + nl = kwargs.pop("nl", True) + fg = kwargs.pop("fg", None) + bold = kwargs.pop("bold", False) + kwargs["nl"] = False hl = True - nobold = kwargs.pop('no_bold', False) + nobold = kwargs.pop("no_bold", False) if nobold: click.secho(line, **kwargs) else: - for span in line.split('*'): + for span in line.split("*"): if hl: hl = False - kwargs['fg'] = fg - kwargs['bold'] = bold + kwargs["fg"] = fg + kwargs["bold"] = bold click.secho(span, **kwargs) else: hl = True - kwargs['fg'] = highlight - kwargs['bold'] = hl_bold + kwargs["fg"] = highlight + kwargs["bold"] = hl_bold click.secho(span, **kwargs) if nl: - kwargs['nl'] = True - click.secho('', **kwargs) + kwargs["nl"] = True + click.secho("", **kwargs) if bottom: click.secho(ERASE_TO_EOL, **kwargs) -def logger(body='', - system_msg=False, - head='', - bad=False, - timestamp=True): +def logger(body="", system_msg=False, head="", bad=False, timestamp=True): if timestamp: if timestamp is True: dt = datetime.now() else: dt = timestamp - tstamp = dt.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3] - click.secho(tstamp + ' ', fg=LOGGER_TIMESTAMP, nl=False) + tstamp = dt.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + click.secho(tstamp + " ", fg=LOGGER_TIMESTAMP, nl=False) if head: click.secho(head, fg=LOGGER_COLOR, nl=False) - click.secho(body, - bold=system_msg, - fg=LOGGER_BAD_COLOR if bad else None) + click.secho(body, bold=system_msg, fg=LOGGER_BAD_COLOR if bad else None) @click.group() @@ -120,214 +130,242 @@ def cli(ctx): pass -@cli.command(help='Check that the flow is valid (default).') -@click.option('--warnings/--no-warnings', - default=False, - show_default=True, - help='Show all Pylint warnings, not just errors.') +@cli.command(help="Check that the flow is valid (default).") +@click.option( + "--warnings/--no-warnings", + default=False, + show_default=True, + help="Show all Pylint warnings, not just errors.", +) @click.pass_obj def check(obj, warnings=False): _check(obj.graph, obj.flow, obj.environment, pylint=obj.pylint, warnings=warnings) fname = inspect.getfile(obj.flow.__class__) - echo("\n*'{cmd} show'* shows a description of this flow.\n" - "*'{cmd} run'* runs the flow locally.\n" - "*'{cmd} help'* shows all available commands and options.\n" - .format(cmd=fname), highlight='magenta', highlight_bold=False) + echo( + "\n*'{cmd} show'* shows a description of this flow.\n" + "*'{cmd} run'* runs the flow locally.\n" + "*'{cmd} help'* shows all available commands and options.\n".format(cmd=fname), + highlight="magenta", + highlight_bold=False, + ) -@cli.command(help='Show structure of the flow.') +@cli.command(help="Show structure of the flow.") @click.pass_obj def show(obj): - echo_always('\n%s' % obj.graph.doc) + echo_always("\n%s" % obj.graph.doc) for _, node in sorted((n.func_lineno, n) for n in obj.graph): - echo_always('\nStep *%s*' % node.name, err=False) - echo_always(node.doc if node.doc else '?', indent=True, err=False) - if node.type != 'end': - echo_always('*=>* %s' % ', '.join('*%s*' % n for n in node.out_funcs), - indent=True, - highlight='magenta', - highlight_bold=False, - err=False) - echo_always('') - - -@cli.command(help='Show all available commands.') + echo_always("\nStep *%s*" % node.name, err=False) + echo_always(node.doc if node.doc else "?", indent=True, err=False) + if node.type != "end": + echo_always( + "*=>* %s" % ", ".join("*%s*" % n for n in node.out_funcs), + indent=True, + highlight="magenta", + highlight_bold=False, + err=False, + ) + echo_always("") + + +@cli.command(help="Show all available commands.") @click.pass_context def help(ctx): print(ctx.parent.get_help()) -@cli.command(help='Output internal state of the flow graph.') +@cli.command(help="Output internal state of the flow graph.") @click.pass_obj def output_raw(obj): - echo('Internal representation of the flow:', - fg='magenta', - bold=False) + echo("Internal representation of the flow:", fg="magenta", bold=False) echo_always(str(obj.graph), err=False) -@cli.command(help='Visualize the flow with Graphviz.') +@cli.command(help="Visualize the flow with Graphviz.") @click.pass_obj def output_dot(obj): - echo('Visualizing the flow as a GraphViz graph', - fg='magenta', - bold=False) - echo("Try piping the output to 'dot -Tpng -o graph.png' to produce " - "an actual image.", indent=True) + echo("Visualizing the flow as a GraphViz graph", fg="magenta", bold=False) + echo( + "Try piping the output to 'dot -Tpng -o graph.png' to produce " + "an actual image.", + indent=True, + ) echo_always(obj.graph.output_dot(), err=False) -@cli.command(help='Get data artifacts of a task or all tasks in a step. ' - 'The format for input-path is either / or ' - '//.') -@click.argument('input-path') -@click.option('--private/--no-private', - default=False, - show_default=True, - help='Show also private attributes.') -@click.option('--max-value-size', - default=1000, - show_default=True, - type=int, - help='Show only values that are smaller than this number. ' - 'Set to 0 to see only keys.') -@click.option('--include', - type=str, - default='', - help='Include only artifacts in the given comma-separated list.') -@click.option('--file', - type=str, - default=None, - help='Serialize artifacts in the given file.') +@cli.command( + help="Get data artifacts of a task or all tasks in a step. " + "The format for input-path is either / or " + "//." +) +@click.argument("input-path") +@click.option( + "--private/--no-private", + default=False, + show_default=True, + help="Show also private attributes.", +) +@click.option( + "--max-value-size", + default=1000, + show_default=True, + type=int, + help="Show only values that are smaller than this number. " + "Set to 0 to see only keys.", +) +@click.option( + "--include", + type=str, + default="", + help="Include only artifacts in the given comma-separated list.", +) +@click.option( + "--file", type=str, default=None, help="Serialize artifacts in the given file." +) @click.pass_obj -def dump(obj, - input_path, - private=None, - max_value_size=None, - include=None, - file=None): +def dump(obj, input_path, private=None, max_value_size=None, include=None, file=None): output = {} - kwargs = {'show_private': private, - 'max_value_size': max_value_size, - 'include': {t for t in include.split(',') if t}} + kwargs = { + "show_private": private, + "max_value_size": max_value_size, + "include": {t for t in include.split(",") if t}, + } # Pathspec can either be run_id/step_name or run_id/step_name/task_id. - parts = input_path.split('/') + parts = input_path.split("/") if len(parts) == 2: run_id, step_name = parts task_id = None elif len(parts) == 3: run_id, step_name, task_id = parts else: - raise CommandException("input_path should either be run_id/step_name" - "or run_id/step_name/task_id") + raise CommandException( + "input_path should either be run_id/step_name" "or run_id/step_name/task_id" + ) datastore_set = TaskDataStoreSet( - obj.flow_datastore, - run_id, - steps=[step_name], - prefetch_data_artifacts=kwargs.get('include')) + obj.flow_datastore, + run_id, + steps=[step_name], + prefetch_data_artifacts=kwargs.get("include"), + ) if task_id: ds_list = [datastore_set.get_with_pathspec(input_path)] else: - ds_list = list(datastore_set) # get all tasks + ds_list = list(datastore_set) # get all tasks for ds in ds_list: - echo('Dumping output of run_id=*{run_id}* ' - 'step=*{step}* task_id=*{task_id}*'.format(run_id=ds.run_id, - step=ds.step_name, - task_id=ds.task_id), - fg='magenta') + echo( + "Dumping output of run_id=*{run_id}* " + "step=*{step}* task_id=*{task_id}*".format( + run_id=ds.run_id, step=ds.step_name, task_id=ds.task_id + ), + fg="magenta", + ) if file is None: - echo_always(ds.format(**kwargs), - highlight='green', - highlight_bold=False, - err=False) + echo_always( + ds.format(**kwargs), highlight="green", highlight_bold=False, err=False + ) else: output[ds.pathspec] = ds.to_dict(**kwargs) if file is not None: - with open(file, 'wb') as f: + with open(file, "wb") as f: pickle.dump(output, f, protocol=pickle.HIGHEST_PROTOCOL) - echo('Artifacts written to *%s*' % file) - - -@cli.command(help='Show stdout/stderr produced by a task or all tasks in a step. ' - 'The format for input-path is either / or ' - '//.') -@click.argument('input-path') -@click.option('--stdout/--no-stdout', - default=False, - show_default=True, - help='Show stdout of the task.') -@click.option('--stderr/--no-stderr', - default=False, - show_default=True, - help='Show stderr of the task.') -@click.option('--both/--no-both', - default=True, - show_default=True, - help='Show both stdout and stderr of the task.') -@click.option('--timestamps/--no-timestamps', - default=False, - show_default=True, - help='Show timestamps.') + echo("Artifacts written to *%s*" % file) + + +@cli.command( + help="Show stdout/stderr produced by a task or all tasks in a step. " + "The format for input-path is either / or " + "//." +) +@click.argument("input-path") +@click.option( + "--stdout/--no-stdout", + default=False, + show_default=True, + help="Show stdout of the task.", +) +@click.option( + "--stderr/--no-stderr", + default=False, + show_default=True, + help="Show stderr of the task.", +) +@click.option( + "--both/--no-both", + default=True, + show_default=True, + help="Show both stdout and stderr of the task.", +) +@click.option( + "--timestamps/--no-timestamps", + default=False, + show_default=True, + help="Show timestamps.", +) @click.pass_obj -def logs(obj, - input_path, - stdout=None, - stderr=None, - both=None, - timestamps=False): +def logs(obj, input_path, stdout=None, stderr=None, both=None, timestamps=False): types = set() if stdout: - types.add('stdout') + types.add("stdout") both = False if stderr: - types.add('stderr') + types.add("stderr") both = False if both: - types.update(('stdout', 'stderr')) + types.update(("stdout", "stderr")) streams = list(sorted(types, reverse=True)) # Pathspec can either be run_id/step_name or run_id/step_name/task_id. - parts = input_path.split('/') + parts = input_path.split("/") if len(parts) == 2: run_id, step_name = parts task_id = None elif len(parts) == 3: run_id, step_name, task_id = parts else: - raise CommandException("input_path should either be run_id/step_name " - "or run_id/step_name/task_id") + raise CommandException( + "input_path should either be run_id/step_name " + "or run_id/step_name/task_id" + ) datastore_set = TaskDataStoreSet( - obj.flow_datastore, run_id, steps=[step_name], allow_not_done=True) + obj.flow_datastore, run_id, steps=[step_name], allow_not_done=True + ) if task_id: - ds_list = [obj.datastore(obj.flow.name, - run_id=run_id, - step_name=step_name, - task_id=task_id, - mode='r', - allow_unsuccessful=True)] + ds_list = [ + obj.datastore( + obj.flow.name, + run_id=run_id, + step_name=step_name, + task_id=task_id, + mode="r", + allow_unsuccessful=True, + ) + ] else: - ds_list = list(datastore_set) # get all tasks + ds_list = list(datastore_set) # get all tasks if ds_list: + def echo_unicode(line, **kwargs): - click.secho(line.decode('UTF-8', errors='replace'), **kwargs) + click.secho(line.decode("UTF-8", errors="replace"), **kwargs) # old style logs are non mflog-style logs maybe_old_style = True for ds in ds_list: - echo('Dumping logs of run_id=*{run_id}* ' - 'step=*{step}* task_id=*{task_id}*'.format(run_id=ds.run_id, - step=ds.step_name, - task_id=ds.task_id), - fg='magenta') + echo( + "Dumping logs of run_id=*{run_id}* " + "step=*{step}* task_id=*{task_id}*".format( + run_id=ds.run_id, step=ds.step_name, task_id=ds.task_id + ), + fg="magenta", + ) for stream in streams: echo(stream, bold=True) @@ -337,10 +375,8 @@ def echo_unicode(line, **kwargs): for line in mflog.merge_logs([blob for _, blob in logs]): if timestamps: ts = mflog.utc_to_local(line.utc_tstamp) - tstamp = ts.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3] - click.secho(tstamp + ' ', - fg=LOGGER_TIMESTAMP, - nl=False) + tstamp = ts.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + click.secho(tstamp + " ", fg=LOGGER_TIMESTAMP, nl=False) echo_unicode(line.msg) maybe_old_style = False elif maybe_old_style: @@ -350,87 +386,114 @@ def echo_unicode(line, **kwargs): # nothing is found log = ds.load_log_legacy(stream) if log and timestamps: - raise CommandException("We can't show --timestamps for " - "old runs. Sorry!") + raise CommandException( + "We can't show --timestamps for " "old runs. Sorry!" + ) echo_unicode(log, nl=False) else: - raise CommandException("No Tasks found at the given path -- " - "either none exist or none have started yet") + raise CommandException( + "No Tasks found at the given path -- " + "either none exist or none have started yet" + ) + # TODO - move step and init under a separate 'internal' subcommand + @cli.command(help="Internal command to execute a single task.") -@click.argument('step-name') -@click.option('--run-id', - default=None, - required=True, - help='ID for one execution of all steps in the flow.') -@click.option('--task-id', - default=None, - required=True, - show_default=True, - help='ID for this instance of the step.') -@click.option('--input-paths', - help='A comma-separated list of pathspecs ' - 'specifying inputs for this step.') -@click.option('--split-index', - type=int, - default=None, - show_default=True, - help='Index of this foreach split.') -@click.option('--tag', - 'opt_tag', - multiple=True, - default=None, - help="Annotate this run with the given tag. You can specify " - "this option multiple times to attach multiple tags in " - "the task.") -@click.option('--namespace', - 'opt_namespace', - default=None, - help="Change namespace from the default (your username) to " - "the specified tag.") -@click.option('--retry-count', - default=0, - help="How many times we have attempted to run this task.") -@click.option('--max-user-code-retries', - default=0, - help="How many times we should attempt running the user code.") -@click.option('--clone-only', - default=None, - help="Pathspec of the origin task for this task to clone. Do " - "not execute anything.") -@click.option('--clone-run-id', - default=None, - help="Run id of the origin flow, if this task is part of a flow " - "being resumed.") -@click.option('--with', - 'decospecs', - multiple=True, - help="Add a decorator to this task. You can specify this " - "option multiple times to attach multiple decorators " - "to this task.") -@click.option('--ubf-context', - default='none', - type=click.Choice(['none', UBF_CONTROL, UBF_TASK]), - help="Provides additional context if this task is of type " - "unbounded foreach.") +@click.argument("step-name") +@click.option( + "--run-id", + default=None, + required=True, + help="ID for one execution of all steps in the flow.", +) +@click.option( + "--task-id", + default=None, + required=True, + show_default=True, + help="ID for this instance of the step.", +) +@click.option( + "--input-paths", + help="A comma-separated list of pathspecs " "specifying inputs for this step.", +) +@click.option( + "--split-index", + type=int, + default=None, + show_default=True, + help="Index of this foreach split.", +) +@click.option( + "--tag", + "opt_tag", + multiple=True, + default=None, + help="Annotate this run with the given tag. You can specify " + "this option multiple times to attach multiple tags in " + "the task.", +) +@click.option( + "--namespace", + "opt_namespace", + default=None, + help="Change namespace from the default (your username) to " "the specified tag.", +) +@click.option( + "--retry-count", + default=0, + help="How many times we have attempted to run this task.", +) +@click.option( + "--max-user-code-retries", + default=0, + help="How many times we should attempt running the user code.", +) +@click.option( + "--clone-only", + default=None, + help="Pathspec of the origin task for this task to clone. Do " + "not execute anything.", +) +@click.option( + "--clone-run-id", + default=None, + help="Run id of the origin flow, if this task is part of a flow " "being resumed.", +) +@click.option( + "--with", + "decospecs", + multiple=True, + help="Add a decorator to this task. You can specify this " + "option multiple times to attach multiple decorators " + "to this task.", +) +@click.option( + "--ubf-context", + default="none", + type=click.Choice(["none", UBF_CONTROL, UBF_TASK]), + help="Provides additional context if this task is of type " "unbounded foreach.", +) @click.pass_context -def step(ctx, - step_name, - opt_tag=None, - run_id=None, - task_id=None, - input_paths=None, - split_index=None, - opt_namespace=None, - retry_count=None, - max_user_code_retries=None, - clone_only=None, - clone_run_id=None, - decospecs=None, - ubf_context='none'): - if ubf_context == 'none': +def step( + ctx, + step_name, + opt_tag=None, + run_id=None, + task_id=None, + input_paths=None, + split_index=None, + opt_namespace=None, + retry_count=None, + max_user_code_retries=None, + clone_only=None, + clone_run_id=None, + decospecs=None, + ubf_context="none", +): + if ubf_context == "none": ubf_context = None if opt_namespace is not None: namespace(opt_namespace or None) @@ -442,64 +505,68 @@ def step(ctx, raise CommandException("Step *%s* doesn't exist." % step_name) if not func.is_step: raise CommandException("Function *%s* is not a step." % step_name) - echo('Executing a step, *%s*' % step_name, - fg='magenta', - bold=False) + echo("Executing a step, *%s*" % step_name, fg="magenta", bold=False) if decospecs: decorators._attach_decorators_to_step(func, decospecs) step_kwargs = ctx.params # Remove argument `step_name` from `step_kwargs`. - step_kwargs.pop('step_name', None) + step_kwargs.pop("step_name", None) # Remove `opt_*` prefix from (some) option keys. - step_kwargs = dict([(k[4:], v) if k.startswith('opt_') else (k, v) - for k, v in step_kwargs.items()]) + step_kwargs = dict( + [(k[4:], v) if k.startswith("opt_") else (k, v) for k, v in step_kwargs.items()] + ) cli_args._set_step_kwargs(step_kwargs) ctx.obj.metadata.add_sticky_tags(tags=opt_tag) paths = decompress_list(input_paths) if input_paths else [] - task = MetaflowTask(ctx.obj.flow, - ctx.obj.flow_datastore, - ctx.obj.metadata, - ctx.obj.environment, - ctx.obj.echo, - ctx.obj.event_logger, - ctx.obj.monitor, - ubf_context) + task = MetaflowTask( + ctx.obj.flow, + ctx.obj.flow_datastore, + ctx.obj.metadata, + ctx.obj.environment, + ctx.obj.echo, + ctx.obj.event_logger, + ctx.obj.monitor, + ubf_context, + ) if clone_only: - task.clone_only(step_name, - run_id, - task_id, - clone_only) + task.clone_only(step_name, run_id, task_id, clone_only) else: - task.run_step(step_name, - run_id, - task_id, - clone_run_id, - paths, - split_index, - retry_count, - max_user_code_retries) + task.run_step( + step_name, + run_id, + task_id, + clone_run_id, + paths, + split_index, + retry_count, + max_user_code_retries, + ) + + echo("Success", fg="green", bold=True, indent=True) - echo('Success', fg='green', bold=True, indent=True) @parameters.add_custom_parameters(deploy_mode=False) @cli.command(help="Internal command to initialize a run.") -@click.option('--run-id', - default=None, - required=True, - help='ID for one execution of all steps in the flow.') -@click.option('--task-id', - default=None, - required=True, - help='ID for this instance of the step.') -@click.option('--tag', - 'tags', - multiple=True, - default=None, - help="Tags for this instance of the step.") +@click.option( + "--run-id", + default=None, + required=True, + help="ID for one execution of all steps in the flow.", +) +@click.option( + "--task-id", default=None, required=True, help="ID for this instance of the step." +) +@click.option( + "--tag", + "tags", + multiple=True, + default=None, + help="Tags for this instance of the step.", +) @click.pass_obj def init(obj, run_id=None, task_id=None, tags=None, **kwargs): # init is a separate command instead of an option in 'step' @@ -512,86 +579,107 @@ def init(obj, run_id=None, task_id=None, tags=None, **kwargs): obj.metadata.add_sticky_tags(tags=tags) - runtime = NativeRuntime(obj.flow, - obj.graph, - obj.flow_datastore, - obj.metadata, - obj.environment, - obj.package, - obj.logger, - obj.entrypoint, - obj.event_logger, - obj.monitor, - run_id=run_id) + runtime = NativeRuntime( + obj.flow, + obj.graph, + obj.flow_datastore, + obj.metadata, + obj.environment, + obj.package, + obj.logger, + obj.entrypoint, + obj.event_logger, + obj.monitor, + run_id=run_id, + ) parameters.set_parameters(obj.flow, kwargs) runtime.persist_parameters(task_id=task_id) + def common_run_options(func): - @click.option('--tag', - 'tags', - multiple=True, - default=None, - help="Annotate this run with the given tag. You can specify " - "this option multiple times to attach multiple tags in " - "the run.") - @click.option('--max-workers', - default=16, - show_default=True, - help='Maximum number of parallel processes.') - @click.option('--max-num-splits', - default=100, - show_default=True, - help='Maximum number of splits allowed in a foreach. This ' - 'is a safety check preventing bugs from triggering ' - 'thousands of steps inadvertently.') - @click.option('--max-log-size', - default=10, - show_default=True, - help='Maximum size of stdout and stderr captured in ' - 'megabytes. If a step outputs more than this to ' - 'stdout/stderr, its output will be truncated.') - @click.option('--with', - 'decospecs', - multiple=True, - help="Add a decorator to all steps. You can specify this " - "option multiple times to attach multiple decorators " - "in steps.") - @click.option('--run-id-file', - default=None, - show_default=True, - type=str, - help="Write the ID of this run to the file specified.") + @click.option( + "--tag", + "tags", + multiple=True, + default=None, + help="Annotate this run with the given tag. You can specify " + "this option multiple times to attach multiple tags in " + "the run.", + ) + @click.option( + "--max-workers", + default=16, + show_default=True, + help="Maximum number of parallel processes.", + ) + @click.option( + "--max-num-splits", + default=100, + show_default=True, + help="Maximum number of splits allowed in a foreach. This " + "is a safety check preventing bugs from triggering " + "thousands of steps inadvertently.", + ) + @click.option( + "--max-log-size", + default=10, + show_default=True, + help="Maximum size of stdout and stderr captured in " + "megabytes. If a step outputs more than this to " + "stdout/stderr, its output will be truncated.", + ) + @click.option( + "--with", + "decospecs", + multiple=True, + help="Add a decorator to all steps. You can specify this " + "option multiple times to attach multiple decorators " + "in steps.", + ) + @click.option( + "--run-id-file", + default=None, + show_default=True, + type=str, + help="Write the ID of this run to the file specified.", + ) @wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) + return wrapper -@click.option('--origin-run-id', - default=None, - help="ID of the run that should be resumed. By default, the " - "last run executed locally.") -@click.argument('step-to-rerun', - required=False) -@cli.command(help='Resume execution of a previous run of this flow.') +@click.option( + "--origin-run-id", + default=None, + help="ID of the run that should be resumed. By default, the " + "last run executed locally.", +) +@click.argument("step-to-rerun", required=False) +@cli.command(help="Resume execution of a previous run of this flow.") @common_run_options @click.pass_obj -def resume(obj, - tags=None, - step_to_rerun=None, - origin_run_id=None, - max_workers=None, - max_num_splits=None, - max_log_size=None, - decospecs=None, - run_id_file=None): +def resume( + obj, + tags=None, + step_to_rerun=None, + origin_run_id=None, + max_workers=None, + max_num_splits=None, + max_log_size=None, + decospecs=None, + run_id_file=None, +): before_run(obj, tags, decospecs + obj.environment.decospecs()) if origin_run_id is None: origin_run_id = get_latest_run_id(obj.echo, obj.flow.name) if origin_run_id is None: - raise CommandException("A previous run id was not found. Specify --origin-run-id.") + raise CommandException( + "A previous run id was not found. Specify --origin-run-id." + ) if step_to_rerun is None: clone_steps = set() @@ -600,27 +688,29 @@ def resume(obj, if step_to_rerun not in obj.graph.nodes: raise CommandException( "invalid step name {0} specified, must be step present in " - "current form of execution graph. Valid step names include: {1}" - .format( - step_to_rerun, - ",".join(list(obj.graph.nodes.keys())))) + "current form of execution graph. Valid step names include: {1}".format( + step_to_rerun, ",".join(list(obj.graph.nodes.keys())) + ) + ) clone_steps = {step_to_rerun} - runtime = NativeRuntime(obj.flow, - obj.graph, - obj.flow_datastore, - obj.metadata, - obj.environment, - obj.package, - obj.logger, - obj.entrypoint, - obj.event_logger, - obj.monitor, - clone_run_id=origin_run_id, - clone_steps=clone_steps, - max_workers=max_workers, - max_num_splits=max_num_splits, - max_log_size=max_log_size * 1024 * 1024) + runtime = NativeRuntime( + obj.flow, + obj.graph, + obj.flow_datastore, + obj.metadata, + obj.environment, + obj.package, + obj.logger, + obj.entrypoint, + obj.event_logger, + obj.monitor, + clone_run_id=origin_run_id, + clone_steps=clone_steps, + max_workers=max_workers, + max_num_splits=max_num_splits, + max_log_size=max_log_size * 1024 * 1024, + ) runtime.persist_parameters() runtime.execute() @@ -628,45 +718,51 @@ def resume(obj, @parameters.add_custom_parameters(deploy_mode=True) -@cli.command(help='Run the workflow locally.') +@cli.command(help="Run the workflow locally.") @common_run_options -@click.option('--namespace', - 'user_namespace', - default=None, - help="Change namespace from the default (your username) to " - "the specified tag. Note that this option does not alter " - "tags assigned to the objects produced by this run, just " - "what existing objects are visible in the client API. You " - "can enable the global namespace with an empty string." - "--namespace=") +@click.option( + "--namespace", + "user_namespace", + default=None, + help="Change namespace from the default (your username) to " + "the specified tag. Note that this option does not alter " + "tags assigned to the objects produced by this run, just " + "what existing objects are visible in the client API. You " + "can enable the global namespace with an empty string." + "--namespace=", +) @click.pass_obj -def run(obj, - tags=None, - max_workers=None, - max_num_splits=None, - max_log_size=None, - decospecs=None, - run_id_file=None, - user_namespace=None, - **kwargs): +def run( + obj, + tags=None, + max_workers=None, + max_num_splits=None, + max_log_size=None, + decospecs=None, + run_id_file=None, + user_namespace=None, + **kwargs +): if user_namespace is not None: namespace(user_namespace or None) before_run(obj, tags, decospecs + obj.environment.decospecs()) - runtime = NativeRuntime(obj.flow, - obj.graph, - obj.flow_datastore, - obj.metadata, - obj.environment, - obj.package, - obj.logger, - obj.entrypoint, - obj.event_logger, - obj.monitor, - max_workers=max_workers, - max_num_splits=max_num_splits, - max_log_size=max_log_size * 1024 * 1024) + runtime = NativeRuntime( + obj.flow, + obj.graph, + obj.flow_datastore, + obj.metadata, + obj.environment, + obj.package, + obj.logger, + obj.entrypoint, + obj.event_logger, + obj.monitor, + max_workers=max_workers, + max_num_splits=max_num_splits, + max_log_size=max_log_size * 1024 * 1024, + ) write_latest_run_id(obj, runtime.run_id) write_run_id(run_id_file, runtime.run_id) @@ -677,7 +773,7 @@ def run(obj, def write_run_id(run_id_file, run_id): if run_id_file is not None: - with open(run_id_file, 'w') as f: + with open(run_id_file, "w") as f: f.write(str(run_id)) @@ -695,94 +791,117 @@ def before_run(obj, tags, decospecs): decorators._attach_decorators(obj.flow, decospecs) obj.graph = FlowGraph(obj.flow.__class__) obj.check(obj.graph, obj.flow, obj.environment, pylint=obj.pylint) - #obj.environment.init_environment(obj.logger) + # obj.environment.init_environment(obj.logger) decorators._init_step_decorators( - obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger) + obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger + ) obj.metadata.add_sticky_tags(tags=tags) # Package working directory only once per run. # We explicitly avoid doing this in `start` since it is invoked for every # step in the run. - obj.package = MetaflowPackage(obj.flow, - obj.environment, - obj.echo, - obj.package_suffixes) + obj.package = MetaflowPackage( + obj.flow, obj.environment, obj.echo, obj.package_suffixes + ) -@cli.command(help='Print the Metaflow version') +@cli.command(help="Print the Metaflow version") @click.pass_obj def version(obj): echo_always(obj.version) + @decorators.add_decorator_options -@click.command(cls=click.CommandCollection, - sources=[cli] + plugins.get_plugin_cli(), - invoke_without_command=True) -@click.option('--quiet/--not-quiet', - show_default=True, - default=False, - help='Suppress unnecessary messages') -@click.option('--metadata', - default=DEFAULT_METADATA, - show_default=True, - type=click.Choice([m.TYPE for m in METADATA_PROVIDERS]), - help='Metadata service type') -@click.option('--environment', - default=DEFAULT_ENVIRONMENT, - show_default=True, - type=click.Choice(['local'] + [m.TYPE for m in ENVIRONMENTS]), - help='Execution environment type') -@click.option('--datastore', - default=DEFAULT_DATASTORE, - show_default=True, - type=click.Choice(DATASTORES), - help='Data backend type') -@click.option('--datastore-root', - help='Root path for datastore') -@click.option('--package-suffixes', - help='A comma-separated list of file suffixes to include ' - 'in the code package.', - default=DEFAULT_PACKAGE_SUFFIXES, - show_default=True) -@click.option('--with', - 'decospecs', - multiple=True, - help="Add a decorator to all steps. You can specify this option " - "multiple times to attach multiple decorators in steps.") -@click.option('--pylint/--no-pylint', - default=True, - show_default=True, - help='Run Pylint on the flow if pylint is installed.') -@click.option('--coverage', - is_flag=True, - default=False, - show_default=True, - help='Measure code coverage using coverage.py.') -@click.option('--event-logger', - default=DEFAULT_EVENT_LOGGER, - show_default=True, - type=click.Choice(LOGGING_SIDECARS), - help='type of event logger used') -@click.option('--monitor', - default=DEFAULT_MONITOR, - show_default=True, - type=click.Choice(MONITOR_SIDECARS), - help='Monitoring backend type') +@click.command( + cls=click.CommandCollection, + sources=[cli] + plugins.get_plugin_cli(), + invoke_without_command=True, +) +@click.option( + "--quiet/--not-quiet", + show_default=True, + default=False, + help="Suppress unnecessary messages", +) +@click.option( + "--metadata", + default=DEFAULT_METADATA, + show_default=True, + type=click.Choice([m.TYPE for m in METADATA_PROVIDERS]), + help="Metadata service type", +) +@click.option( + "--environment", + default=DEFAULT_ENVIRONMENT, + show_default=True, + type=click.Choice(["local"] + [m.TYPE for m in ENVIRONMENTS]), + help="Execution environment type", +) +@click.option( + "--datastore", + default=DEFAULT_DATASTORE, + show_default=True, + type=click.Choice(DATASTORES), + help="Data backend type", +) +@click.option("--datastore-root", help="Root path for datastore") +@click.option( + "--package-suffixes", + help="A comma-separated list of file suffixes to include " "in the code package.", + default=DEFAULT_PACKAGE_SUFFIXES, + show_default=True, +) +@click.option( + "--with", + "decospecs", + multiple=True, + help="Add a decorator to all steps. You can specify this option " + "multiple times to attach multiple decorators in steps.", +) +@click.option( + "--pylint/--no-pylint", + default=True, + show_default=True, + help="Run Pylint on the flow if pylint is installed.", +) +@click.option( + "--coverage", + is_flag=True, + default=False, + show_default=True, + help="Measure code coverage using coverage.py.", +) +@click.option( + "--event-logger", + default=DEFAULT_EVENT_LOGGER, + show_default=True, + type=click.Choice(LOGGING_SIDECARS), + help="type of event logger used", +) +@click.option( + "--monitor", + default=DEFAULT_MONITOR, + show_default=True, + type=click.Choice(MONITOR_SIDECARS), + help="Monitoring backend type", +) @click.pass_context -def start(ctx, - quiet=False, - metadata=None, - environment=None, - datastore=None, - datastore_root=None, - decospecs=None, - package_suffixes=None, - pylint=None, - coverage=None, - event_logger=None, - monitor=None, - **deco_options): +def start( + ctx, + quiet=False, + metadata=None, + environment=None, + datastore=None, + datastore_root=None, + decospecs=None, + package_suffixes=None, + pylint=None, + coverage=None, + event_logger=None, + monitor=None, + **deco_options +): global echo if quiet: echo = echo_dev_null @@ -794,17 +913,20 @@ def start(ctx, if use_r(): version = metaflow_r_version() - echo('Metaflow %s' % version, fg='magenta', bold=True, nl=False) - echo(" executing *%s*" % ctx.obj.flow.name, fg='magenta', nl=False) - echo(" for *%s*" % resolve_identity(), fg='magenta') + echo("Metaflow %s" % version, fg="magenta", bold=True, nl=False) + echo(" executing *%s*" % ctx.obj.flow.name, fg="magenta", nl=False) + echo(" for *%s*" % resolve_identity(), fg="magenta") if coverage: from coverage import Coverage + no_covrc = "COVERAGE_RCFILE" not in os.environ - cov = Coverage(data_suffix=True, - auto_data=True, - source=['metaflow'] if no_covrc else None, - branch=True if no_covrc else None) + cov = Coverage( + data_suffix=True, + auto_data=True, + source=["metaflow"] if no_covrc else None, + branch=True if no_covrc else None, + ) cov.start() cli_args._set_top_kwargs(ctx.params) @@ -815,33 +937,34 @@ def start(ctx, ctx.obj.check = _check ctx.obj.pylint = pylint ctx.obj.top_cli = cli - ctx.obj.package_suffixes = package_suffixes.split(',') + ctx.obj.package_suffixes = package_suffixes.split(",") ctx.obj.reconstruct_cli = _reconstruct_cli ctx.obj.event_logger = EventLogger(event_logger) - ctx.obj.environment = [e for e in ENVIRONMENTS + [MetaflowEnvironment] - if e.TYPE == environment][0](ctx.obj.flow) + ctx.obj.environment = [ + e for e in ENVIRONMENTS + [MetaflowEnvironment] if e.TYPE == environment + ][0](ctx.obj.flow) ctx.obj.environment.validate_environment(echo) ctx.obj.monitor = Monitor(monitor, ctx.obj.environment, ctx.obj.flow.name) ctx.obj.monitor.start() - ctx.obj.metadata = [m for m in METADATA_PROVIDERS - if m.TYPE == metadata][0](ctx.obj.environment, - ctx.obj.flow, - ctx.obj.event_logger, - ctx.obj.monitor) + ctx.obj.metadata = [m for m in METADATA_PROVIDERS if m.TYPE == metadata][0]( + ctx.obj.environment, ctx.obj.flow, ctx.obj.event_logger, ctx.obj.monitor + ) ctx.obj.datastore_impl = DATASTORES[datastore] if datastore_root is None: - datastore_root = \ - ctx.obj.datastore_impl.get_datastore_root_from_config(ctx.obj.echo) + datastore_root = ctx.obj.datastore_impl.get_datastore_root_from_config( + ctx.obj.echo + ) if datastore_root is None: raise CommandException( "Could not find the location of the datastore -- did you correctly set the " - "METAFLOW_DATASTORE_SYSROOT_%s environment variable?" % datastore.upper()) + "METAFLOW_DATASTORE_SYSROOT_%s environment variable?" % datastore.upper() + ) ctx.obj.datastore_impl.datastore_root = datastore_root @@ -851,36 +974,43 @@ def start(ctx, ctx.obj.environment, ctx.obj.metadata, ctx.obj.event_logger, - ctx.obj.monitor) + ctx.obj.monitor, + ) # It is important to initialize flow decorators early as some of the # things they provide may be used by some of the objects initialize after. - decorators._init_flow_decorators(ctx.obj.flow, - ctx.obj.graph, - ctx.obj.environment, - ctx.obj.flow_datastore, - ctx.obj.metadata, - ctx.obj.logger, - echo, - deco_options) + decorators._init_flow_decorators( + ctx.obj.flow, + ctx.obj.graph, + ctx.obj.environment, + ctx.obj.flow_datastore, + ctx.obj.metadata, + ctx.obj.logger, + echo, + deco_options, + ) if decospecs: decorators._attach_decorators(ctx.obj.flow, decospecs) # initialize current and parameter context for deploy-time parameters current._set_env(flow_name=ctx.obj.flow.name, is_running=False) - parameters.set_parameter_context(ctx.obj.flow.name, - ctx.obj.echo, - ctx.obj.flow_datastore) + parameters.set_parameter_context( + ctx.obj.flow.name, ctx.obj.echo, ctx.obj.flow_datastore + ) - if ctx.invoked_subcommand not in ('run', 'resume'): + if ctx.invoked_subcommand not in ("run", "resume"): # run/resume are special cases because they can add more decorators with --with, # so they have to take care of themselves. - decorators._attach_decorators( - ctx.obj.flow, ctx.obj.environment.decospecs()) + decorators._attach_decorators(ctx.obj.flow, ctx.obj.environment.decospecs()) decorators._init_step_decorators( - ctx.obj.flow, ctx.obj.graph, ctx.obj.environment, ctx.obj.flow_datastore, ctx.obj.logger) - #TODO (savin): Enable lazy instantiation of package + ctx.obj.flow, + ctx.obj.graph, + ctx.obj.environment, + ctx.obj.flow_datastore, + ctx.obj.logger, + ) + # TODO (savin): Enable lazy instantiation of package ctx.obj.package = None if ctx.invoked_subcommand is None: ctx.invoke(check) @@ -889,66 +1019,65 @@ def start(ctx, def _reconstruct_cli(params): for k, v in params.items(): if v: - if k == 'decospecs': - k = 'with' - k = k.replace('_', '-') + if k == "decospecs": + k = "with" + k = k.replace("_", "-") if not isinstance(v, tuple): v = [v] for value in v: - yield '--%s' % k + yield "--%s" % k if not isinstance(value, bool): yield str(value) def _check(graph, flow, environment, pylint=True, warnings=False, **kwargs): - echo("Validating your flow...", fg='magenta', bold=False) + echo("Validating your flow...", fg="magenta", bold=False) linter = lint.linter # TODO set linter settings linter.run_checks(graph, **kwargs) - echo('The graph looks good!', fg='green', bold=True, indent=True) + echo("The graph looks good!", fg="green", bold=True, indent=True) if pylint: - echo("Running pylint...", fg='magenta', bold=False) + echo("Running pylint...", fg="magenta", bold=False) fname = inspect.getfile(flow.__class__) pylint = PyLint(fname) if pylint.has_pylint(): pylint_is_happy, pylint_exception_msg = pylint.run( - warnings=warnings, - pylint_config=environment.pylint_config(), - logger=echo_always) + warnings=warnings, + pylint_config=environment.pylint_config(), + logger=echo_always, + ) if pylint_is_happy: - echo('Pylint is happy!', - fg='green', - bold=True, - indent=True) + echo("Pylint is happy!", fg="green", bold=True, indent=True) else: - echo("Pylint couldn't analyze your code.\n\tPylint exception: %s" - % pylint_exception_msg, - fg='red', - bold=True, - indent=True) - echo("Skipping Pylint checks.", - fg='red', - bold=True, - indent=True) + echo( + "Pylint couldn't analyze your code.\n\tPylint exception: %s" + % pylint_exception_msg, + fg="red", + bold=True, + indent=True, + ) + echo("Skipping Pylint checks.", fg="red", bold=True, indent=True) else: - echo("Pylint not found, so extra checks are disabled.", - fg='green', - indent=True, - bold=False) + echo( + "Pylint not found, so extra checks are disabled.", + fg="green", + indent=True, + bold=False, + ) def print_metaflow_exception(ex): echo_always(ex.headline, indent=True, nl=False, bold=True) if ex.line_no is None: - echo_always(':') + echo_always(":") else: - echo_always(' on line %d:' % ex.line_no, bold=True) + echo_always(" on line %d:" % ex.line_no, bold=True) echo_always(ex.message, indent=True, bold=False, padding_bottom=True) def print_unknown_exception(ex): - echo_always('Internal error', indent=True, bold=True) + echo_always("Internal error", indent=True, bold=True) echo_always(traceback.format_exc(), highlight=None, highlight_bold=False) @@ -956,12 +1085,14 @@ class CliState(object): def __init__(self, flow): self.flow = flow + def main(flow, args=None, handle_exceptions=True, entrypoint=None): # Ignore warning(s) and prevent spamming the end-user. # TODO: This serves as a short term workaround for RuntimeWarning(s) thrown # in py3.8 related to log buffering (bufsize=1). import warnings - warnings.filterwarnings('ignore') + + warnings.filterwarnings("ignore") if entrypoint is None: entrypoint = [sys.executable, sys.argv[0]] @@ -970,12 +1101,10 @@ def main(flow, args=None, handle_exceptions=True, entrypoint=None): try: if args is None: - start(auto_envvar_prefix='METAFLOW', obj=state) + start(auto_envvar_prefix="METAFLOW", obj=state) else: try: - start.main(args=args, - obj=state, - auto_envvar_prefix='METAFLOW') + start.main(args=args, obj=state, auto_envvar_prefix="METAFLOW") except SystemExit as e: return e.code except MetaflowException as x: diff --git a/metaflow/cli_args.py b/metaflow/cli_args.py index 5f8654300e6..c378de7c1cf 100644 --- a/metaflow/cli_args.py +++ b/metaflow/cli_args.py @@ -1,5 +1,5 @@ # This class provides a global singleton `cli_args` which stores the `top` and -# `step` level options for the metaflow CLI. This allows decorators to have +# `step` level options for the metaflow CLI. This allows decorators to have # access to the CLI options instead of relying (solely) on the click context. # TODO: We have two CLIArgs: # - this one, which captures the top level and step-level options passed to the @@ -35,13 +35,10 @@ def top_kwargs(self): def step_kwargs(self): return self._step_kwargs - def step_command(self, - executable, - script, - step_name, - top_kwargs=None, - step_kwargs=None): - cmd = [executable, '-u', script] + def step_command( + self, executable, script, step_name, top_kwargs=None, step_kwargs=None + ): + cmd = [executable, "-u", script] if top_kwargs is None: top_kwargs = self._top_kwargs if step_kwargs is None: @@ -49,7 +46,7 @@ def step_command(self, top_args_list = list(self._options(top_kwargs)) cmd.extend(top_args_list) - cmd.extend(['step', step_name]) + cmd.extend(["step", step_name]) step_args_list = list(self._options(step_kwargs)) cmd.extend(step_args_list) @@ -61,13 +58,14 @@ def _options(mapping): if v: # we need special handling for 'with' since it is a reserved # keyword in Python, so we call it 'decospecs' in click args - if k == 'decospecs': - k = 'with' - k = k.replace('_', '-') + if k == "decospecs": + k = "with" + k = k.replace("_", "-") v = v if isinstance(v, (list, tuple, set)) else [v] for value in v: - yield '--%s' % k + yield "--%s" % k if not isinstance(value, bool): yield to_unicode(value) + cli_args = CLIArgs() diff --git a/metaflow/client/__init__.py b/metaflow/client/__init__.py index d2abefb1c5b..a06fbd290ba 100644 --- a/metaflow/client/__init__.py +++ b/metaflow/client/__init__.py @@ -1,13 +1,15 @@ -#core client classes -from .core import namespace,\ - get_namespace,\ - default_namespace,\ - metadata, \ - get_metadata, \ - default_metadata, \ - Metaflow,\ - Flow,\ - Run,\ - Step,\ - Task,\ - DataArtifact +# core client classes +from .core import ( + namespace, + get_namespace, + default_namespace, + metadata, + get_metadata, + default_metadata, + Metaflow, + Flow, + Run, + Step, + Task, + DataArtifact, +) diff --git a/metaflow/client/core.py b/metaflow/client/core.py index f237350269c..f516b61e4fe 100644 --- a/metaflow/client/core.py +++ b/metaflow/client/core.py @@ -8,9 +8,11 @@ from itertools import chain from metaflow.metaflow_environment import MetaflowEnvironment -from metaflow.exception import MetaflowNotFound,\ - MetaflowNamespaceMismatch,\ - MetaflowInternalError +from metaflow.exception import ( + MetaflowNotFound, + MetaflowNamespaceMismatch, + MetaflowInternalError, +) from metaflow.metaflow_config import DEFAULT_METADATA, MAX_ATTEMPTS from metaflow.plugins import ENVIRONMENTS, METADATA_PROVIDERS @@ -29,11 +31,7 @@ # populated at the bottom of this file _CLASSES = {} -Metadata = namedtuple('Metadata', ['name', - 'value', - 'created_at', - 'type', - 'task']) +Metadata = namedtuple("Metadata", ["name", "value", "created_at", "type", "task"]) filecache = None current_namespace = False @@ -63,7 +61,7 @@ def metadata(ms): get_metadata()) """ global current_metadata - infos = ms.split('@', 1) + infos = ms.split("@", 1) types = [m.TYPE for m in METADATA_PROVIDERS] if infos[0] in types: current_metadata = [m for m in METADATA_PROVIDERS if m.TYPE == infos[0]][0] @@ -71,15 +69,17 @@ def metadata(ms): current_metadata.INFO = infos[1] else: # Deduce from ms; if starts with http, use service or else use local - if ms.startswith('http'): - metadata_type = 'service' + if ms.startswith("http"): + metadata_type = "service" else: - metadata_type = 'local' + metadata_type = "local" res = [m for m in METADATA_PROVIDERS if m.TYPE == metadata_type] if not res: print( "Cannot find a '%s' metadata provider -- " - "try specifying one explicitly using @", metadata_type) + "try specifying one explicitly using @", + metadata_type, + ) return get_metadata() current_metadata = res[0] current_metadata.INFO = ms @@ -105,7 +105,7 @@ def get_metadata(): """ if current_metadata is False: default_metadata() - return '%s@%s' % (current_metadata.TYPE, current_metadata.INFO) + return "%s@%s" % (current_metadata.TYPE, current_metadata.INFO) def default_metadata(): @@ -126,6 +126,7 @@ def default_metadata(): current_metadata = default[0] else: from metaflow.plugins.metadata import LocalMetadataProvider + current_metadata = LocalMetadataProvider return get_metadata() @@ -247,7 +248,7 @@ def __iter__(self): # filtering on namespace on flows means finding at least one # run in this namespace. This is_in_namespace() function # does this properly in this case - all_flows = self.metadata.get_object('root', 'flow', None, None) + all_flows = self.metadata.get_object("root", "flow", None, None) all_flows = all_flows if all_flows else [] for flow in all_flows: try: @@ -257,7 +258,7 @@ def __iter__(self): continue def __str__(self): - return 'Metaflow()' + return "Metaflow()" def __getitem__(self, id): """ @@ -311,25 +312,29 @@ class MetaflowObject(object): path_components : List[string] Components of the pathspec """ - _NAME = 'base' + + _NAME = "base" _CHILD_CLASS = None _PARENT_CLASS = None - def __init__(self, - pathspec=None, - attempt=None, - _object=None, - _parent=None, - _namespace_check=True): + def __init__( + self, + pathspec=None, + attempt=None, + _object=None, + _parent=None, + _namespace_check=True, + ): self._metaflow = Metaflow() self._parent = _parent self._path_components = None self._attempt = attempt if self._attempt is not None: - if self._NAME not in ['task', 'artifact']: + if self._NAME not in ["task", "artifact"]: raise MetaflowNotFound( - "Attempts can only be specified for Task or DataArtifact") + "Attempts can only be specified for Task or DataArtifact" + ) try: self._attempt = int(self._attempt) except ValueError: @@ -339,13 +344,14 @@ def __init__(self, raise MetaflowNotFound("Attempt can only be non-negative") elif self._attempt >= MAX_ATTEMPTS: raise MetaflowNotFound( - "Attempt can only be smaller than %d" % MAX_ATTEMPTS) + "Attempt can only be smaller than %d" % MAX_ATTEMPTS + ) # NOTE: It is possible that no attempt exists but we can't # distinguish between "attempt will happen" and "no such # attempt exists". if pathspec: - ids = pathspec.split('/') + ids = pathspec.split("/") self.id = ids[-1] self._pathspec = pathspec @@ -354,28 +360,30 @@ def __init__(self, self._object = _object self._pathspec = pathspec - if self._NAME in ('flow', 'task'): - self.id = str(self._object[self._NAME + '_id']) - elif self._NAME == 'run': - self.id = str(self._object['run_number']) - elif self._NAME == 'step': - self.id = str(self._object['step_name']) - elif self._NAME == 'artifact': - self.id = str(self._object['name']) + if self._NAME in ("flow", "task"): + self.id = str(self._object[self._NAME + "_id"]) + elif self._NAME == "run": + self.id = str(self._object["run_number"]) + elif self._NAME == "step": + self.id = str(self._object["step_name"]) + elif self._NAME == "artifact": + self.id = str(self._object["name"]) else: raise MetaflowInternalError(msg="Unknown type: %s" % self._NAME) - self._created_at = datetime.fromtimestamp(self._object['ts_epoch']/1000.0) + self._created_at = datetime.fromtimestamp(self._object["ts_epoch"] / 1000.0) - self._tags = frozenset(chain(self._object.get('system_tags') or [], - self._object.get('tags') or [])) + self._tags = frozenset( + chain(self._object.get("system_tags") or [], self._object.get("tags") or []) + ) if _namespace_check and not self.is_in_namespace(): raise MetaflowNamespaceMismatch(current_namespace) def _get_object(self, *path_components): result = self._metaflow.metadata.get_object( - self._NAME, 'self', None, self._attempt, *path_components) + self._NAME, "self", None, self._attempt, *path_components + ) if not result: raise MetaflowNotFound("%s does not exist" % self) return result @@ -393,17 +401,28 @@ def __iter__(self): """ query_filter = {} if current_namespace: - query_filter = {'any_tags': current_namespace} + query_filter = {"any_tags": current_namespace} unfiltered_children = self._metaflow.metadata.get_object( - self._NAME, _CLASSES[self._CHILD_CLASS]._NAME, query_filter, - self._attempt, *self.path_components) + self._NAME, + _CLASSES[self._CHILD_CLASS]._NAME, + query_filter, + self._attempt, + *self.path_components + ) unfiltered_children = unfiltered_children if unfiltered_children else [] children = filter( lambda x: self._iter_filter(x), - (_CLASSES[self._CHILD_CLASS](attempt=self._attempt, - _object=obj, _parent=self, _namespace_check=False) - for obj in unfiltered_children)) + ( + _CLASSES[self._CHILD_CLASS]( + attempt=self._attempt, + _object=obj, + _parent=self, + _namespace_check=False, + ) + for obj in unfiltered_children + ), + ) if children: return iter(sorted(children, reverse=True, key=lambda x: x.created_at)) @@ -426,7 +445,7 @@ def _filtered_children(self, *tags): @classmethod def _url_token(cls): - return '%ss' % cls._NAME + return "%ss" % cls._NAME def is_in_namespace(self): """ @@ -439,16 +458,18 @@ def is_in_namespace(self): bool Whether or not the object is in the current namespace """ - if self._NAME == 'flow': + if self._NAME == "flow": return any(True for _ in self) else: - return current_namespace is None or\ - current_namespace in self._tags + return current_namespace is None or current_namespace in self._tags def __str__(self): if self._attempt is not None: return "%s('%s', attempt=%d)" % ( - self.__class__.__name__, self.pathspec, self._attempt) + self.__class__.__name__, + self.pathspec, + self._attempt, + ) return "%s('%s')" % (self.__class__.__name__, self.pathspec) def __repr__(self): @@ -460,7 +481,8 @@ def _get_child(self, id): result.append(p) result.append(id) return self._metaflow.metadata.get_object( - _CLASSES[self._CHILD_CLASS]._NAME, 'self', None, self._attempt, *result) + _CLASSES[self._CHILD_CLASS]._NAME, "self", None, self._attempt, *result + ) def __getitem__(self, id): """ @@ -483,8 +505,9 @@ def __getitem__(self, id): """ obj = self._get_child(id) if obj: - return _CLASSES[self._CHILD_CLASS](attempt=self._attempt, - _object=obj, _parent=self) + return _CLASSES[self._CHILD_CLASS]( + attempt=self._attempt, _object=obj, _parent=self + ) else: raise KeyError(id) @@ -544,20 +567,21 @@ def parent(self): MetaflowObject The parent of this object """ - if self._NAME == 'flow': + if self._NAME == "flow": return None # Compute parent from pathspec and cache it. if self._parent is None: pathspec = self.pathspec - parent_pathspec = pathspec[:pathspec.rfind('/')] + parent_pathspec = pathspec[: pathspec.rfind("/")] # Only artifacts and tasks have attempts right now so we get the # right parent if we are an artifact. - attempt_to_pass = self._attempt if self._NAME == 'artifact' else None + attempt_to_pass = self._attempt if self._NAME == "artifact" else None # We can skip the namespace check because if self._NAME = 'run', # the parent object is guaranteed to be in namespace. # Otherwise the check is moot for Flow since parent is singular. self._parent = _CLASSES[self._PARENT_CLASS]( - parent_pathspec, attempt=attempt_to_pass, _namespace_check=False) + parent_pathspec, attempt=attempt_to_pass, _namespace_check=False + ) return self._parent @property @@ -594,7 +618,7 @@ def path_components(self): Individual components of the pathspec """ if self._path_components is None: - ids = self.pathspec.split('/') + ids = self.pathspec.split("/") self._path_components = ids return list(self._path_components) @@ -610,7 +634,7 @@ def __contains__(self, var): return var in self._artifacts def __str__(self): - return '' % ', '.join(self._artifacts) + return "" % ", ".join(self._artifacts) def __repr__(self): return str(self) @@ -641,20 +665,21 @@ def __init__(self, flow_name, code_package): self._flow_name = flow_name info = json.loads(code_package) - self._path = info['location'] - self._ds_type = info['ds_type'] - self._sha = info['sha'] + self._path = info["location"] + self._ds_type = info["ds_type"] + self._sha = info["sha"] if filecache is None: filecache = FileCache() _, blobdata = filecache.get_data( - self._ds_type, self._flow_name, self._path, self._sha) + self._ds_type, self._flow_name, self._path, self._sha + ) code_obj = BytesIO(blobdata) - self._tar = tarfile.open(fileobj=code_obj, mode='r:gz') + self._tar = tarfile.open(fileobj=code_obj, mode="r:gz") # The JSON module in Python3 deals with Unicode. Tar gives bytes. - info_str = self._tar.extractfile('INFO').read().decode('utf-8') + info_str = self._tar.extractfile("INFO").read().decode("utf-8") self._info = json.loads(info_str) - self._flowspec = self._tar.extractfile(self._info['script']).read() + self._flowspec = self._tar.extractfile(self._info["script"]).read() @property def path(self): @@ -705,7 +730,7 @@ def tarball(self): return self._tar def __str__(self): - return '' % self._info['script'] + return "" % self._info["script"] class DataArtifact(MetaflowObject): @@ -722,8 +747,8 @@ class DataArtifact(MetaflowObject): Alias for created_at """ - _NAME = 'artifact' - _PARENT_CLASS = 'task' + _NAME = "artifact" + _PARENT_CLASS = "task" _CHILD_CLASS = None @property @@ -738,8 +763,8 @@ def data(self): """ global filecache - ds_type = self._object['ds_type'] - location = self._object['location'] + ds_type = self._object["ds_type"] + location = self._object["location"] components = self.path_components if filecache is None: # TODO: Pass proper environment to properly extract artifacts @@ -749,17 +774,22 @@ def data(self): # TODO: We can store more information in the metadata, particularly # to determine if we need an environment to unpickle the artifact. meta = { - 'objects': {self._object['name']: self._object['sha']}, - 'info': {self._object['name']: { - 'size': 0, 'type': None, 'encoding': self._object['content_type']}} + "objects": {self._object["name"]: self._object["sha"]}, + "info": { + self._object["name"]: { + "size": 0, + "type": None, + "encoding": self._object["content_type"], + } + }, } - if location.startswith(':root:'): - return filecache.get_artifact( - ds_type, location[6:], meta, *components) + if location.startswith(":root:"): + return filecache.get_artifact(ds_type, location[6:], meta, *components) else: # Older artifacts have a location information which we can use. return filecache.get_artifact_by_location( - ds_type, location, meta, *components) + ds_type, location, meta, *components + ) @property def size(self): @@ -774,17 +804,21 @@ def size(self): """ global filecache - ds_type = self._object['ds_type'] - location = self._object['location'] + ds_type = self._object["ds_type"] + location = self._object["location"] components = self.path_components if filecache is None: # TODO: Pass proper environment to properly extract artifacts filecache = FileCache() - if location.startswith(':root:'): - return filecache.get_artifact_size(ds_type, location[6:], self._attempt, *components) + if location.startswith(":root:"): + return filecache.get_artifact_size( + ds_type, location[6:], self._attempt, *components + ) else: - return filecache.get_artifact_size_by_location(ds_type, location, self._attempt, *components) + return filecache.get_artifact_size_by_location( + ds_type, location, self._attempt, *components + ) # TODO add # @property @@ -802,7 +836,7 @@ def sha(self): string Hash of this artifact """ - return self._object['sha'] + return self._object["sha"] @property def finished_at(self): @@ -865,16 +899,16 @@ class Task(MetaflowObject): Information about the execution environment (for example Conda) """ - _NAME = 'task' - _PARENT_CLASS = 'step' - _CHILD_CLASS = 'artifact' + _NAME = "task" + _PARENT_CLASS = "step" + _CHILD_CLASS = "artifact" def __init__(self, *args, **kwargs): super(Task, self).__init__(*args, **kwargs) def _iter_filter(self, x): # exclude private data artifacts - return x.id[0] != '_' + return x.id[0] != "_" @property def metadata(self): @@ -890,13 +924,19 @@ def metadata(self): Metadata produced by this task """ all_metadata = self._metaflow.metadata.get_object( - self._NAME, 'metadata', None, self._attempt, *self.path_components) + self._NAME, "metadata", None, self._attempt, *self.path_components + ) all_metadata = all_metadata if all_metadata else [] - return [Metadata(name=obj.get('field_name'), - value=obj.get('value'), - created_at=obj.get('ts_epoch'), - type=obj.get('type'), - task=self) for obj in all_metadata] + return [ + Metadata( + name=obj.get("field_name"), + value=obj.get("value"), + created_at=obj.get("ts_epoch"), + type=obj.get("type"), + task=self, + ) + for obj in all_metadata + ] @property def metadata_dict(self): @@ -915,8 +955,9 @@ def metadata_dict(self): Dictionary mapping metadata name with value """ # use the newest version of each key, hence sorting - return {m.name: m.value - for m in sorted(self.metadata, key=lambda m: m.created_at)} + return { + m.name: m.value for m in sorted(self.metadata, key=lambda m: m.created_at) + } @property def index(self): @@ -933,7 +974,7 @@ def index(self): Index in the innermost loop for this task """ try: - return self['_foreach_stack'].data[-1].index + return self["_foreach_stack"].data[-1].index except (KeyError, IndexError): return None @@ -972,7 +1013,7 @@ def artifacts(self): Container of all DataArtifacts produced by this task """ arts = list(self) - obj = namedtuple('MetaflowArtifacts', [art.id for art in arts]) + obj = namedtuple("MetaflowArtifacts", [art.id for art in arts]) return obj._make(arts) @property @@ -989,7 +1030,7 @@ def successful(self): True if the task completed successfully and False otherwise """ try: - return self['_success'].data + return self["_success"].data except KeyError: return False @@ -1007,7 +1048,7 @@ def finished(self): True if the task completed and False otherwise """ try: - return self['_task_ok'].data + return self["_task_ok"].data except KeyError: return False @@ -1026,7 +1067,7 @@ def exception(self): Exception raised by the task or None if not applicable """ try: - return self['_exception'].data + return self["_exception"].data except KeyError: return None @@ -1044,7 +1085,7 @@ def finished_at(self): Datetime of when the task finished """ try: - return self['_task_ok'].created_at + return self["_task_ok"].created_at except KeyError: return None @@ -1060,8 +1101,8 @@ def runtime_name(self): Name of the runtime this task executed on """ for t in self._tags: - if t.startswith('runtime:'): - return t.split(':')[1] + if t.startswith("runtime:"): + return t.split(":")[1] return None @property @@ -1081,7 +1122,7 @@ def stdout(self): string Standard output of this task """ - return self._load_log('stdout') + return self._load_log("stdout") @property def stdout_size(self): @@ -1097,7 +1138,7 @@ def stdout_size(self): int Size of the stdout log content (in bytes) """ - return self._get_logsize('stdout') + return self._get_logsize("stdout") @property def stderr(self): @@ -1116,7 +1157,7 @@ def stderr(self): string Standard error of this task """ - return self._load_log('stderr') + return self._load_log("stderr") @property def stderr_size(self): @@ -1132,7 +1173,7 @@ def stderr_size(self): int Size of the stderr log content (in bytes) """ - return self._get_logsize('stderr') + return self._get_logsize("stderr") @property def current_attempt(self): @@ -1158,7 +1199,7 @@ def current_attempt(self): # here. It is possible that logs exists for a newer attempt that # just failed to record metadata. We could make this logic more robust # and guarantee that we always return the latest available log. - attempt = int(self.metadata_dict.get('attempt', 0)) + attempt = int(self.metadata_dict.get("attempt", 0)) return attempt @cached_property @@ -1173,7 +1214,7 @@ def code(self): MetaflowCode Code package for this task """ - code_package = self.metadata_dict.get('code-package') + code_package = self.metadata_dict.get("code-package") if code_package: return MetaflowCode(self.path_components[0], code_package) return None @@ -1195,22 +1236,21 @@ def environment_info(self): my_code = self.code if not my_code: return None - env_type = my_code.info['environment_type'] + env_type = my_code.info["environment_type"] if not env_type: return None - env = [m for m in ENVIRONMENTS + - [MetaflowEnvironment] if m.TYPE == env_type][0] + env = [m for m in ENVIRONMENTS + [MetaflowEnvironment] if m.TYPE == env_type][0] return env.get_client_info(self.path_components[0], self.metadata_dict) def _load_log(self, stream): - log_location = self.metadata_dict.get('log_location_%s' % stream) + log_location = self.metadata_dict.get("log_location_%s" % stream) if log_location: return self._load_log_legacy(log_location, stream) else: - return ''.join(line + '\n' for _, line in self.loglines(stream)) + return "".join(line + "\n" for _, line in self.loglines(stream)) def _get_logsize(self, stream): - log_location = self.metadata_dict.get('log_location_%s' % stream) + log_location = self.metadata_dict.get("log_location_%s" % stream) if log_location: return self._legacy_log_size(log_location, stream) else: @@ -1224,19 +1264,21 @@ def loglines(self, stream, as_unicode=True): it is returned as a (unicode) string. """ from metaflow.mflog.mflog import merge_logs + global filecache - ds_type = self.metadata_dict.get('ds-type') - ds_root = self.metadata_dict.get('ds-root') + ds_type = self.metadata_dict.get("ds-type") + ds_root = self.metadata_dict.get("ds-root") if ds_type is None or ds_root is None: - yield None, '' + yield None, "" return if filecache is None: filecache = FileCache() attempt = self.current_attempt logs = filecache.get_logs_stream( - ds_type, ds_root, stream, attempt, *self.path_components) + ds_type, ds_root, stream, attempt, *self.path_components + ) for line in merge_logs([blob for _, blob in logs]): msg = to_unicode(line.msg) if as_unicode else line.msg yield line.utc_tstamp, msg @@ -1246,15 +1288,16 @@ def _load_log_legacy(self, log_location, logtype, as_unicode=True): global filecache log_info = json.loads(log_location) - location = log_info['location'] - ds_type = log_info['ds_type'] - attempt = log_info['attempt'] + location = log_info["location"] + ds_type = log_info["ds_type"] + attempt = log_info["attempt"] if filecache is None: filecache = FileCache() ret_val = filecache.get_log_legacy( - ds_type, location, logtype, int(attempt), *self.path_components) + ds_type, location, logtype, int(attempt), *self.path_components + ) if as_unicode and (ret_val is not None): - return ret_val.decode(encoding='utf8') + return ret_val.decode(encoding="utf8") else: return ret_val @@ -1262,20 +1305,21 @@ def _legacy_log_size(self, log_location, logtype): global filecache log_info = json.loads(log_location) - location = log_info['location'] - ds_type = log_info['ds_type'] - attempt = log_info['attempt'] + location = log_info["location"] + ds_type = log_info["ds_type"] + attempt = log_info["attempt"] if filecache is None: filecache = FileCache() return filecache.get_legacy_log_size( - ds_type, location, logtype, int(attempt), *self.path_components) + ds_type, location, logtype, int(attempt), *self.path_components + ) def _log_size(self, stream): global filecache - ds_type = self.metadata_dict.get('ds-type') - ds_root = self.metadata_dict.get('ds-root') + ds_type = self.metadata_dict.get("ds-type") + ds_root = self.metadata_dict.get("ds-root") if ds_type is None or ds_root is None: return 0 if filecache is None: @@ -1283,8 +1327,8 @@ def _log_size(self, stream): attempt = self.current_attempt return filecache.get_log_size( - ds_type, ds_root, stream, attempt, *self.path_components) - + ds_type, ds_root, stream, attempt, *self.path_components + ) class Step(MetaflowObject): @@ -1305,9 +1349,9 @@ class Step(MetaflowObject): Information about the execution environment (for example Conda) """ - _NAME = 'step' - _PARENT_CLASS = 'run' - _CHILD_CLASS = 'task' + _NAME = "step" + _PARENT_CLASS = "run" + _CHILD_CLASS = "task" @property def task(self): @@ -1380,7 +1424,7 @@ def control_tasks(self, *tags): filter_tags.extend(tags) for child in children: if all(tag in child.tags for tag in filter_tags): - yield child + yield child def __iter__(self): children = super(Step, self).__iter__() @@ -1449,13 +1493,13 @@ class Run(MetaflowObject): Task for the end step (if it is present already) """ - _NAME = 'run' - _PARENT_CLASS = 'flow' - _CHILD_CLASS = 'step' + _NAME = "run" + _PARENT_CLASS = "flow" + _CHILD_CLASS = "step" def _iter_filter(self, x): # exclude _parameters step - return x.id[0] != '_' + return x.id[0] != "_" def steps(self, *tags): """ @@ -1489,8 +1533,8 @@ def code(self): MetaflowCode Code package for this run """ - if 'start' in self: - return self['start'].task.code + if "start" in self: + return self["start"].task.code @property def data(self): @@ -1579,7 +1623,7 @@ def end_task(self): The 'end' task """ try: - end_step = self['end'] + end_step = self["end"] except KeyError: return None @@ -1601,9 +1645,9 @@ class Flow(MetaflowObject): Latest successfully completed Run of this Flow """ - _NAME = 'flow' + _NAME = "flow" _PARENT_CLASS = None - _CHILD_CLASS = 'run' + _CHILD_CLASS = "run" def __init__(self, *args, **kwargs): super(Flow, self).__init__(*args, **kwargs) @@ -1659,8 +1703,8 @@ def runs(self, *tags): return self._filtered_children(*tags) -_CLASSES['flow'] = Flow -_CLASSES['run'] = Run -_CLASSES['step'] = Step -_CLASSES['task'] = Task -_CLASSES['artifact'] = DataArtifact +_CLASSES["flow"] = Flow +_CLASSES["run"] = Run +_CLASSES["step"] = Step +_CLASSES["task"] = Task +_CLASSES["artifact"] = DataArtifact diff --git a/metaflow/client/filecache.py b/metaflow/client/filecache.py index 63f60b30bd9..5adf1a06c60 100644 --- a/metaflow/client/filecache.py +++ b/metaflow/client/filecache.py @@ -9,14 +9,21 @@ from metaflow.datastore import DATASTORES, FlowDataStore from metaflow.datastore.content_addressed_store import BlobCache from metaflow.exception import MetaflowException -from metaflow.metaflow_config import CLIENT_CACHE_PATH, CLIENT_CACHE_MAX_SIZE, \ - CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT, CLIENT_CACHE_MAX_TASKDATASTORE_COUNT +from metaflow.metaflow_config import ( + CLIENT_CACHE_PATH, + CLIENT_CACHE_MAX_SIZE, + CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT, + CLIENT_CACHE_MAX_TASKDATASTORE_COUNT, +) NEW_FILE_QUARANTINE = 10 if sys.version_info[0] >= 3 and sys.version_info[1] >= 2: + def od_move_to_end(od, key): od.move_to_end(key) + + else: # Not very efficient but works and most people are on 3.2+ def od_move_to_end(od, key): @@ -24,8 +31,9 @@ def od_move_to_end(od, key): del od[key] od[key] = v + class FileCacheException(MetaflowException): - headline = 'File cache error' + headline = "File cache error" class FileCache(object): @@ -43,7 +51,7 @@ def __init__(self, cache_dir=None, max_size=None): self._blob_caches = {} # We also keep a cache for FlowDataStore objects because some of them - # may have long-lived persistent connections; this is purely a + # may have long-lived persistent connections; this is purely a # performance optimization. Uses OrderedDict to implement a kind of LRU # cache and keep only a certain number of these caches around. self._store_caches = OrderedDict() @@ -60,28 +68,33 @@ def cache_dir(self): return self._cache_dir def get_logs_stream( - self, ds_type, ds_root, stream, attempt, flow_name, run_id, - step_name, task_id): + self, ds_type, ds_root, stream, attempt, flow_name, run_id, step_name, task_id + ): from metaflow.mflog import LOG_SOURCES ds = self._get_flow_datastore(ds_type, ds_root, flow_name) task_ds = ds.get_task_datastore( - run_id, step_name, task_id, - data_metadata={'objects': {}, 'info': {}}) + run_id, step_name, task_id, data_metadata={"objects": {}, "info": {}} + ) return task_ds.load_logs(LOG_SOURCES, stream, attempt_override=attempt) def get_log_legacy( - self, ds_type, location, logtype, attempt, flow_name, run_id, - step_name, task_id): + self, ds_type, location, logtype, attempt, flow_name, run_id, step_name, task_id + ): ds_cls = self._get_datastore_storage_impl(ds_type) ds_root = ds_cls.path_join(*ds_cls.path_split(location)[:-5]) cache_id = self._flow_ds_id(ds_type, ds_root, flow_name) - token = '%s.cached' % sha1(os.path.join( - run_id, step_name, task_id, '%s_log' % logtype).\ - encode('utf-8')).hexdigest() + token = ( + "%s.cached" + % sha1( + os.path.join(run_id, step_name, task_id, "%s_log" % logtype).encode( + "utf-8" + ) + ).hexdigest() + ) path = os.path.join(self._cache_dir, cache_id, token[:2], token) cached_log = self.read_file(path) @@ -91,32 +104,45 @@ def get_log_legacy( ds = self._get_flow_datastore(ds_type, ds_root, flow_name) task_ds = ds.get_task_datastore( - run_id, step_name, task_id, - data_metadata={'objects': {}, 'info': {}}) + run_id, step_name, task_id, data_metadata={"objects": {}, "info": {}} + ) log = task_ds.load_log_legacy(logtype, attempt_override=attempt) # Store this in the file cache as well self.create_file(path, log) return log - def get_legacy_log_size(self, ds_type, location, logtype, attempt, flow_name, run_id, step_name, task_id): + def get_legacy_log_size( + self, ds_type, location, logtype, attempt, flow_name, run_id, step_name, task_id + ): ds_cls = self._get_datastore_storage_impl(ds_type) ds_root = ds_cls.path_join(*ds_cls.path_split(location)[:-5]) ds = self._get_flow_datastore(ds_type, ds_root, flow_name) task_ds = ds.get_task_datastore( - run_id, step_name, task_id, attempt=attempt, - data_metadata={'objects': {}, 'info': {}}) + run_id, + step_name, + task_id, + attempt=attempt, + data_metadata={"objects": {}, "info": {}}, + ) return task_ds.get_legacy_log_size(logtype) - def get_log_size(self, ds_type, ds_root, logtype, attempt, flow_name, run_id, step_name, task_id): + def get_log_size( + self, ds_type, ds_root, logtype, attempt, flow_name, run_id, step_name, task_id + ): from metaflow.mflog import LOG_SOURCES + ds = self._get_flow_datastore(ds_type, ds_root, flow_name) task_ds = ds.get_task_datastore( - run_id, step_name, task_id, attempt=attempt, - data_metadata={'objects': {}, 'info': {}}) + run_id, + step_name, + task_id, + attempt=attempt, + data_metadata={"objects": {}, "info": {}}, + ) return task_ds.get_log_size(LOG_SOURCES, logtype) @@ -127,61 +153,100 @@ def get_data(self, ds_type, flow_name, location, key): return next(ds.load_data([key], force_raw=True)) - def get_artifact_size_by_location(self, ds_type, location, attempt, flow_name, run_id, - step_name, task_id, name): + def get_artifact_size_by_location( + self, ds_type, location, attempt, flow_name, run_id, step_name, task_id, name + ): """Gets the size of the artifact content (in bytes) for the name at the location""" ds_cls = self._get_datastore_storage_impl(ds_type) ds_root = ds_cls.get_datastore_root_from_location(location, flow_name) return self.get_artifact_size( - ds_type, ds_root, attempt, flow_name, run_id, step_name, task_id, name) + ds_type, ds_root, attempt, flow_name, run_id, step_name, task_id, name + ) - def get_artifact_size(self, ds_type, ds_root, attempt, flow_name, run_id, - step_name, task_id, name): + def get_artifact_size( + self, ds_type, ds_root, attempt, flow_name, run_id, step_name, task_id, name + ): """Gets the size of the artifact content (in bytes) for the name""" task_ds = self._get_task_datastore( - ds_type, ds_root, flow_name, run_id, step_name, task_id, attempt) + ds_type, ds_root, flow_name, run_id, step_name, task_id, attempt + ) _, size = next(task_ds.get_artifact_sizes([name])) return size def get_artifact_by_location( - self, ds_type, location, data_metadata, flow_name, run_id, - step_name, task_id, name): + self, + ds_type, + location, + data_metadata, + flow_name, + run_id, + step_name, + task_id, + name, + ): ds_cls = self._get_datastore_storage_impl(ds_type) ds_root = ds_cls.get_datastore_root_from_location(location, flow_name) return self.get_artifact( - ds_type, ds_root, data_metadata, flow_name, run_id, step_name, - task_id, name) + ds_type, ds_root, data_metadata, flow_name, run_id, step_name, task_id, name + ) def get_artifact( - self, ds_type, ds_root, data_metadata, flow_name, run_id, step_name, - task_id, name): - _, obj = next(self.get_artifacts( - ds_type, ds_root, data_metadata, flow_name, run_id, step_name, - task_id, [name])) + self, + ds_type, + ds_root, + data_metadata, + flow_name, + run_id, + step_name, + task_id, + name, + ): + _, obj = next( + self.get_artifacts( + ds_type, + ds_root, + data_metadata, + flow_name, + run_id, + step_name, + task_id, + [name], + ) + ) return obj def get_all_artifacts( - self, ds_type, ds_root, data_metadata, flow_name, run_id, step_name, - task_id): + self, ds_type, ds_root, data_metadata, flow_name, run_id, step_name, task_id + ): ds = self._get_flow_datastore(ds_type, ds_root, flow_name) # We get the task datastore for this task task_ds = ds.get_task_datastore( - run_id, step_name, task_id, data_metadata=data_metadata) + run_id, step_name, task_id, data_metadata=data_metadata + ) # This will reuse the blob cache if needed. We do not have an # artifact cache so the unpickling happens every time here. return task_ds.load_artifacts([n for n, _ in task_ds.items()]) def get_artifacts( - self, ds_type, ds_root, data_metadata, flow_name, run_id, step_name, - task_id, names): + self, + ds_type, + ds_root, + data_metadata, + flow_name, + run_id, + step_name, + task_id, + names, + ): ds = self._get_flow_datastore(ds_type, ds_root, flow_name) # We get the task datastore for this task task_ds = ds.get_task_datastore( - run_id, step_name, task_id, data_metadata=data_metadata) + run_id, step_name, task_id, data_metadata=data_metadata + ) # note that load_artifacts uses flow_datastore.castore which goes # through one of the self._blob_cache return task_ds.load_artifacts(names) @@ -195,10 +260,8 @@ def create_file(self, path, value): try: FileCache._makedirs(dirname) except: # noqa E722 - raise FileCacheException( - 'Could not create directory: %s' % dirname) - tmpfile = NamedTemporaryFile( - dir=dirname, prefix='dlobj', delete=False) + raise FileCacheException("Could not create directory: %s" % dirname) + tmpfile = NamedTemporaryFile(dir=dirname, prefix="dlobj", delete=False) # Now write out the file try: tmpfile.write(value) @@ -215,7 +278,7 @@ def create_file(self, path, value): def read_file(self, path): if os.path.exists(path): try: - with open(path, 'rb') as f: + with open(path, "rb") as f: return f.read() except IOError: # It may have been concurrently garbage collected by another @@ -236,27 +299,28 @@ def _index_objects(self): continue for obj in os.listdir(root): sha, ext = os.path.splitext(obj) - if ext in ['cached', 'blob']: + if ext in ["cached", "blob"]: path = os.path.join(root, obj) - objects.insert(0, (os.path.getctime(path), - os.path.getsize(path), - path)) + objects.insert( + 0, (os.path.getctime(path), os.path.getsize(path), path) + ) self._total = sum(size for _, size, _ in objects) self._objects = sorted(objects, reverse=False) @staticmethod def _flow_ds_id(ds_type, ds_root, flow_name): - return '.'.join([ds_type, ds_root, flow_name]) + return ".".join([ds_type, ds_root, flow_name]) @staticmethod def _task_ds_id(ds_type, ds_root, flow_name, run_id, step_name, task_id, attempt): - return '.'.join( - [ds_type, ds_root, flow_name, run_id, step_name, task_id, str(attempt)]) + return ".".join( + [ds_type, ds_root, flow_name, run_id, step_name, task_id, str(attempt)] + ) def _garbage_collect(self): now = time.time() - while self._objects and self._total > self._max_size * 1024**2: + while self._objects and self._total > self._max_size * 1024 ** 2: if now - self._objects[0][0] < NEW_FILE_QUARANTINE: break ctime, size, path = self._objects.pop(0) @@ -283,7 +347,7 @@ def _makedirs(path): def _get_datastore_storage_impl(ds_type): storage_impl = DATASTORES.get(ds_type, None) if storage_impl is None: - raise FileCacheException('Datastore %s was not found' % ds_type) + raise FileCacheException("Datastore %s was not found" % ds_type) return storage_impl def _get_flow_datastore(self, ds_type, ds_root, flow_name): @@ -297,11 +361,13 @@ def _get_flow_datastore(self, ds_type, ds_root, flow_name): storage_impl = self._get_datastore_storage_impl(ds_type) cached_flow_datastore = FlowDataStore( flow_name=flow_name, - environment=None, # TODO: Add environment here + environment=None, # TODO: Add environment here storage_impl=storage_impl, - ds_root=ds_root) + ds_root=ds_root, + ) blob_cache = self._blob_caches.setdefault( - cache_id, FileBlobCache(self, cache_id)) + cache_id, FileBlobCache(self, cache_id) + ) cached_flow_datastore.ca_store.set_blob_cache(blob_cache) self._store_caches[cache_id] = cached_flow_datastore if len(self._store_caches) > CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT: @@ -310,21 +376,28 @@ def _get_flow_datastore(self, ds_type, ds_root, flow_name): return cached_flow_datastore def _get_task_datastore( - self, ds_type, ds_root, flow_name, run_id, step_name, task_id, attempt): + self, ds_type, ds_root, flow_name, run_id, step_name, task_id, attempt + ): flow_ds = self._get_flow_datastore(ds_type, ds_root, flow_name) cached_metadata = None if attempt is not None: cache_id = self._task_ds_id( - ds_type, ds_root, flow_name, run_id, step_name, task_id, attempt) + ds_type, ds_root, flow_name, run_id, step_name, task_id, attempt + ) cached_metadata = self._task_metadata_caches.get(cache_id) if cached_metadata: od_move_to_end(self._task_metadata_caches, cache_id) return flow_ds.get_task_datastore( - run_id, step_name, task_id, attempt=attempt, - data_metadata=cached_metadata) + run_id, + step_name, + task_id, + attempt=attempt, + data_metadata=cached_metadata, + ) # If we are here, we either have attempt=None or nothing in the cache task_ds = flow_ds.get_task_datastore( - run_id, step_name, task_id, attempt=attempt) + run_id, step_name, task_id, attempt=attempt + ) cache_id = self._task_ds_id( ds_type, ds_root, flow_name, run_id, step_name, task_id, task_ds.attempt ) @@ -333,8 +406,8 @@ def _get_task_datastore( self._task_metadata_caches.popitem(last=False) return task_ds -class FileBlobCache(BlobCache): +class FileBlobCache(BlobCache): def __init__(self, filecache, cache_id): self._filecache = filecache self._cache_id = cache_id @@ -342,12 +415,11 @@ def __init__(self, filecache, cache_id): def _path(self, key): key_dir = key[:2] return os.path.join( - self._filecache.cache_dir, self._cache_id, key_dir, '%s.blob' % key) + self._filecache.cache_dir, self._cache_id, key_dir, "%s.blob" % key + ) def load_key(self, key): return self._filecache.read_file(self._path(key)) def store_key(self, key, blob): self._filecache.create_file(self._path(key), blob) - - diff --git a/metaflow/cmd_with_io.py b/metaflow/cmd_with_io.py index 2651c33b577..3a1e8656178 100644 --- a/metaflow/cmd_with_io.py +++ b/metaflow/cmd_with_io.py @@ -3,18 +3,20 @@ from metaflow.util import to_bytes + def cmd(cmdline, input, output): for path, data in input.items(): - with open(path, 'wb') as f: + with open(path, "wb") as f: f.write(to_bytes(data)) if subprocess.call(cmdline, shell=True): - raise ExternalCommandFailed("Command '%s' returned a non-zero " - "exit code." % cmdline) + raise ExternalCommandFailed( + "Command '%s' returned a non-zero " "exit code." % cmdline + ) out = [] for path in output: - with open(path, 'rb') as f: + with open(path, "rb") as f: out.append(f.read()) if len(out) == 1: diff --git a/metaflow/current.py b/metaflow/current.py index e3a9ff16882..e5092fbd4e1 100644 --- a/metaflow/current.py +++ b/metaflow/current.py @@ -1,6 +1,4 @@ - class Current(object): - def __init__(self): self._flow_name = None self._run_id = None @@ -12,16 +10,18 @@ def __init__(self): self._username = None self._is_running = False - def _set_env(self, - flow_name=None, - run_id=None, - step_name=None, - task_id=None, - retry_count=None, - origin_run_id=None, - namespace=None, - username=None, - is_running=True): + def _set_env( + self, + flow_name=None, + run_id=None, + step_name=None, + task_id=None, + retry_count=None, + origin_run_id=None, + namespace=None, + username=None, + is_running=True, + ): self._flow_name = flow_name self._run_id = run_id @@ -73,10 +73,7 @@ def origin_run_id(self): @property def pathspec(self): - return '/'.join((self._flow_name, - self._run_id, - self._step_name, - self._task_id)) + return "/".join((self._flow_name, self._run_id, self._step_name, self._task_id)) @property def namespace(self): @@ -86,6 +83,7 @@ def namespace(self): def username(self): return self._username + # instantiate the Current singleton. This will be populated # by task.MetaflowTask before a task is executed. current = Current() diff --git a/metaflow/datastore/__init__.py b/metaflow/datastore/__init__.py index f93c0b5aa86..d5dd68f30c8 100644 --- a/metaflow/datastore/__init__.py +++ b/metaflow/datastore/__init__.py @@ -5,5 +5,4 @@ from .local_storage import LocalStorage from .s3_storage import S3Storage -DATASTORES = {'local': LocalStorage, - 's3': S3Storage} +DATASTORES = {"local": LocalStorage, "s3": S3Storage} diff --git a/metaflow/datastore/content_addressed_store.py b/metaflow/datastore/content_addressed_store.py index 45a985092cd..4b261523ad9 100644 --- a/metaflow/datastore/content_addressed_store.py +++ b/metaflow/datastore/content_addressed_store.py @@ -97,9 +97,7 @@ def packing_iter(): # We don't actually want to overwrite but by saying =True, we avoid # checking again saving some operations. We are already sure we are not # sending duplicate files since we already checked. - self._storage_impl.save_bytes( - packing_iter(), overwrite=True, len_hint=len_hint - ) + self._storage_impl.save_bytes(packing_iter(), overwrite=True, len_hint=len_hint) return results def load_blobs(self, keys, force_raw=False): @@ -133,9 +131,7 @@ def load_blobs(self, keys, force_raw=False): path = self._storage_impl.path_join(self._prefix, key[:2], key) load_paths.append((key, path)) - with self._storage_impl.load_bytes( - [p for _, p in load_paths] - ) as loaded: + with self._storage_impl.load_bytes([p for _, p in load_paths]) as loaded: for (key, _), (_, file_path, meta) in zip(load_paths, loaded): # At this point, we either return the object as is (if raw) or # decode it according to the encoding version @@ -151,12 +147,9 @@ def load_blobs(self, keys, force_raw=False): version = meta.get("cas_version", -1) if version == -1: raise DataException( - "Could not extract encoding version for '%s'" - % path + "Could not extract encoding version for '%s'" % path ) - unpack_code = getattr( - self, "_unpack_v%d" % version, None - ) + unpack_code = getattr(self, "_unpack_v%d" % version, None) if unpack_code is None: raise DataException( "Unknown encoding version %d for '%s' -- " @@ -168,7 +161,8 @@ def load_blobs(self, keys, force_raw=False): blob = unpack_code(f) except Exception as e: raise DataException( - "Could not unpack artifact '%s': %s" % (path, e)) + "Could not unpack artifact '%s': %s" % (path, e) + ) if self._blob_cache: self._blob_cache.store_key(key, blob) diff --git a/metaflow/datastore/datastore_set.py b/metaflow/datastore/datastore_set.py index c4685e4b8a9..1b977079ac1 100644 --- a/metaflow/datastore/datastore_set.py +++ b/metaflow/datastore/datastore_set.py @@ -10,17 +10,22 @@ cache and lets you access them. As a performance optimization it also lets you prefetch select data artifacts leveraging a shared cache. """ + + class TaskDataStoreSet(object): - def __init__(self, - flow_datastore, - run_id, - steps=None, - pathspecs=None, - prefetch_data_artifacts=None, - allow_not_done=False): + def __init__( + self, + flow_datastore, + run_id, + steps=None, + pathspecs=None, + prefetch_data_artifacts=None, + allow_not_done=False, + ): task_datastores = flow_datastore.get_latest_task_datastores( - run_id, steps=steps, pathspecs=pathspecs, allow_not_done=allow_not_done) + run_id, steps=steps, pathspecs=pathspecs, allow_not_done=allow_not_done + ) if prefetch_data_artifacts: # produce a set of SHA keys to prefetch based on artifact names @@ -29,7 +34,7 @@ def __init__(self, prefetch.update(ds.keys_for_artifacts(prefetch_data_artifacts)) # ignore missing keys prefetch.discard(None) - + # prefetch artifacts and share them with all datastores # in this DatastoreSet preloaded = dict(flow_datastore.ca_store.load_blobs(prefetch)) @@ -52,12 +57,14 @@ def __iter__(self): for v in self.pathspec_cache.values(): yield v + """ This class ensures that blobs that correspond to artifacts that are common to all datastores in this set are only loaded once """ -class ImmutableBlobCache(BlobCache): + +class ImmutableBlobCache(BlobCache): def __init__(self, preloaded): self._preloaded = preloaded diff --git a/metaflow/datastore/datastore_storage.py b/metaflow/datastore/datastore_storage.py index 01879e4c5d4..264d2b15833 100644 --- a/metaflow/datastore/datastore_storage.py +++ b/metaflow/datastore/datastore_storage.py @@ -10,6 +10,7 @@ class CloseAfterUse(object): This class should be used in a with statement and, when the with scope exits, `close` will be called on the closer object """ + def __init__(self, data, closer=None): self.data = data self._closer = closer @@ -24,7 +25,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): class DataStoreStorage(object): """ - A DataStoreStorage defines the interface of communication between the + A DataStoreStorage defines the interface of communication between the higher-level datastores and the actual storage system. Both the ContentAddressedStore and the TaskDataStore use these methods to @@ -32,11 +33,12 @@ class DataStoreStorage(object): be low-level; they are in a class to provide better abstraction but this class itself is not meant to be initialized. """ + TYPE = None datastore_root = None path_rexp = None - list_content_result = namedtuple('list_content_result', 'path is_file') + list_content_result = namedtuple("list_content_result", "path is_file") def __init__(self, root=None): self.datastore_root = root if root else self.datastore_root @@ -82,38 +84,42 @@ def get_datastore_root_from_location(cls, path, flow_name): Raised if the path is not a valid path from this datastore. """ if cls.path_rexp is None: - cls.path_rexp = re.compile(cls.path_join( - '(?P.*)', - '(?P[_a-zA-Z][_a-zA-Z0-9]+)', - 'data', - '(?P[0-9a-f]{2})', - '(?:r_)?(?P=init)[0-9a-f]{38}')) + cls.path_rexp = re.compile( + cls.path_join( + "(?P.*)", + "(?P[_a-zA-Z][_a-zA-Z0-9]+)", + "data", + "(?P[0-9a-f]{2})", + "(?:r_)?(?P=init)[0-9a-f]{38}", + ) + ) m = cls.path_rexp.match(path) - if not m or m.group('flow_name') != flow_name: + if not m or m.group("flow_name") != flow_name: raise DataException( "Location '%s' does not correspond to a valid location for " - "flow '%s'." % (path, flow_name)) - return m.group('root') + "flow '%s'." % (path, flow_name) + ) + return m.group("root") @classmethod def path_join(cls, *components): if len(components) == 0: - return '' - component = components[0].rstrip('/') - components = [component] + [c.strip('/') for c in components[1:]] - return '/'.join(components) + return "" + component = components[0].rstrip("/") + components = [component] + [c.strip("/") for c in components[1:]] + return "/".join(components) @classmethod def path_split(cls, path): - return path.split('/') + return path.split("/") @classmethod def basename(cls, path): - return path.split('/')[-1] + return path.split("/")[-1] @classmethod def dirname(cls, path): - return path.rsplit('/', 1)[0] + return path.rsplit("/", 1)[0] def full_uri(self, path): return self.path_join(self.datastore_root, path) diff --git a/metaflow/datastore/exceptions.py b/metaflow/datastore/exceptions.py index 1ed108f93f7..9225ccdfe5b 100644 --- a/metaflow/datastore/exceptions.py +++ b/metaflow/datastore/exceptions.py @@ -1,5 +1,6 @@ from ..exception import MetaflowException + class DataException(MetaflowException): headline = "Data store error" @@ -7,6 +8,6 @@ class DataException(MetaflowException): class UnpicklableArtifactException(MetaflowException): headline = "Cannot pickle artifact" - def __init__(self,artifact_name): + def __init__(self, artifact_name): msg = 'Cannot pickle dump artifact named "%s"' % artifact_name - super().__init__(msg=msg, lineno=None) \ No newline at end of file + super().__init__(msg=msg, lineno=None) diff --git a/metaflow/datastore/flow_datastore.py b/metaflow/datastore/flow_datastore.py index 71e1f9b2e38..9a830debf6c 100644 --- a/metaflow/datastore/flow_datastore.py +++ b/metaflow/datastore/flow_datastore.py @@ -6,17 +6,20 @@ from .content_addressed_store import ContentAddressedStore from .task_datastore import TaskDataStore + class FlowDataStore(object): default_storage_impl = None - def __init__(self, - flow_name, - environment, - metadata=None, - event_logger=None, - monitor=None, - storage_impl=None, - ds_root=None): + def __init__( + self, + flow_name, + environment, + metadata=None, + event_logger=None, + monitor=None, + storage_impl=None, + ds_root=None, + ): """ Initialize a Flow level datastore. @@ -43,8 +46,7 @@ def __init__(self, The optional root for this datastore; if not provided, use the default for the DataStoreStorage, optional """ - storage_impl = storage_impl if storage_impl else \ - self.default_storage_impl + storage_impl = storage_impl if storage_impl else self.default_storage_impl if storage_impl is None: raise RuntimeError("No datastore storage implementation specified") self._storage_impl = storage_impl(ds_root) @@ -58,15 +60,16 @@ def __init__(self, self.monitor = monitor self.ca_store = ContentAddressedStore( - self._storage_impl.path_join(self.flow_name, 'data'), - self._storage_impl) + self._storage_impl.path_join(self.flow_name, "data"), self._storage_impl + ) @property def datastore_root(self): return self._storage_impl.datastore_root def get_latest_task_datastores( - self, run_id=None, steps=None, pathspecs=None, allow_not_done=False): + self, run_id=None, steps=None, pathspecs=None, allow_not_done=False + ): """ Return a list of TaskDataStore for a subset of the tasks. @@ -101,28 +104,41 @@ def get_latest_task_datastores( # eventually consistent `list_content` operation, and directly construct # the task_urls list. if pathspecs: - task_urls = [self._storage_impl.path_join(self.flow_name, pathspec) - for pathspec in pathspecs] + task_urls = [ + self._storage_impl.path_join(self.flow_name, pathspec) + for pathspec in pathspecs + ] else: run_prefix = self._storage_impl.path_join(self.flow_name, run_id) if steps: - step_urls = [self._storage_impl.path_join(run_prefix, step) - for step in steps] + step_urls = [ + self._storage_impl.path_join(run_prefix, step) for step in steps + ] else: - step_urls = [step.path for step in self._storage_impl.list_content( - [run_prefix]) if step.is_file is False] - task_urls = [task.path for task in self._storage_impl.list_content( - step_urls) if task.is_file is False] + step_urls = [ + step.path + for step in self._storage_impl.list_content([run_prefix]) + if step.is_file is False + ] + task_urls = [ + task.path + for task in self._storage_impl.list_content(step_urls) + if task.is_file is False + ] urls = [] for task_url in task_urls: for attempt in range(metaflow_config.MAX_ATTEMPTS): - for suffix in [TaskDataStore.METADATA_DATA_SUFFIX, - TaskDataStore.METADATA_ATTEMPT_SUFFIX, - TaskDataStore.METADATA_DONE_SUFFIX]: - urls.append(self._storage_impl.path_join( - task_url, - TaskDataStore.metadata_name_for_attempt(suffix, attempt) - )) + for suffix in [ + TaskDataStore.METADATA_DATA_SUFFIX, + TaskDataStore.METADATA_ATTEMPT_SUFFIX, + TaskDataStore.METADATA_DONE_SUFFIX, + ]: + urls.append( + self._storage_impl.path_join( + task_url, + TaskDataStore.metadata_name_for_attempt(suffix, attempt), + ) + ) latest_started_attempts = {} done_attempts = set() @@ -136,31 +152,41 @@ def get_latest_task_datastores( if fname == TaskDataStore.METADATA_DONE_SUFFIX: done_attempts.add((run, step, task, attempt)) elif fname == TaskDataStore.METADATA_ATTEMPT_SUFFIX: - latest_started_attempts[(run, step, task)] = \ - max(latest_started_attempts.get((run, step, task), 0), - attempt) + latest_started_attempts[(run, step, task)] = max( + latest_started_attempts.get((run, step, task), 0), attempt + ) elif fname == TaskDataStore.METADATA_DATA_SUFFIX: # This somewhat breaks the abstraction since we are using # load_bytes directly instead of load_metadata - with open(path, 'rb') as f: + with open(path, "rb") as f: data_objs[(run, step, task, attempt)] = json.load(f) # We now figure out the latest attempt that started *and* finished. # Note that if an attempt started but didn't finish, we do *NOT* return # the previous attempt latest_started_attempts = set( (run, step, task, attempt) - for (run, step, task), attempt in latest_started_attempts.items()) + for (run, step, task), attempt in latest_started_attempts.items() + ) if allow_not_done: latest_to_fetch = latest_started_attempts else: latest_to_fetch = latest_started_attempts & done_attempts - latest_to_fetch = [(v[0], v[1], v[2], v[3], data_objs[v], 'r', allow_not_done) - for v in latest_to_fetch] + latest_to_fetch = [ + (v[0], v[1], v[2], v[3], data_objs[v], "r", allow_not_done) + for v in latest_to_fetch + ] return list(itertools.starmap(self.get_task_datastore, latest_to_fetch)) def get_task_datastore( - self, run_id, step_name, task_id, attempt=None, - data_metadata=None, mode='r', allow_not_done=False): + self, + run_id, + step_name, + task_id, + attempt=None, + data_metadata=None, + mode="r", + allow_not_done=False, + ): return TaskDataStore( self, @@ -170,7 +196,8 @@ def get_task_datastore( attempt=attempt, data_metadata=data_metadata, mode=mode, - allow_not_done=allow_not_done) + allow_not_done=allow_not_done, + ) def save_data(self, data_iter, len_hint=0): """Saves data to the underlying content-addressed store diff --git a/metaflow/datastore/inputs.py b/metaflow/datastore/inputs.py index cea6ea020b8..0fb00d08454 100644 --- a/metaflow/datastore/inputs.py +++ b/metaflow/datastore/inputs.py @@ -4,6 +4,7 @@ class Inputs(object): foreach: inputs[0].x both: (inp.x for inp in inputs) """ + def __init__(self, flows): # TODO sort by foreach index self.flows = list(flows) diff --git a/metaflow/datastore/local_storage.py b/metaflow/datastore/local_storage.py index 694c8b622b7..dc0343ffb97 100644 --- a/metaflow/datastore/local_storage.py +++ b/metaflow/datastore/local_storage.py @@ -5,9 +5,10 @@ from .datastore_storage import CloseAfterUse, DataStoreStorage from .exceptions import DataException + class LocalStorage(DataStoreStorage): - TYPE = 'local' - METADATA_DIR = '_meta' + TYPE = "local" + METADATA_DIR = "_meta" @classmethod def get_datastore_root_from_config(cls, echo, create_on_absent=True): @@ -16,7 +17,7 @@ def get_datastore_root_from_config(cls, echo, create_on_absent=True): try: # Python2 current_path = os.getcwdu() - except: # noqa E722 + except: # noqa E722 current_path = os.getcwd() check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR) check_dir = os.path.realpath(check_dir) @@ -32,8 +33,9 @@ def get_datastore_root_from_config(cls, echo, create_on_absent=True): if top_level_reached: if create_on_absent: # Could not find any directory to use so create a new one - echo('Creating local datastore in current directory (%s)' - % orig_path) + echo( + "Creating local datastore in current directory (%s)" % orig_path + ) os.mkdir(orig_path) result = orig_path else: @@ -66,7 +68,7 @@ def info_file(self, path): if file_exists: full_meta_path = "%s_meta" % self.full_uri(path) try: - with open(full_meta_path, 'r') as f: + with open(full_meta_path, "r") as f: return True, json.load(f) except OSError: return True, None @@ -95,9 +97,9 @@ def list_content(self, paths): results.append( self.list_content_result( path=self.path_join(path, f), - is_file=self.is_file( - [self.path_join(path, f)])[0]) - ) + is_file=self.is_file([self.path_join(path, f)])[0], + ) + ) except FileNotFoundError as e: pass return results @@ -112,10 +114,10 @@ def save_bytes(self, path_and_bytes_iter, overwrite=False, len_hint=0): if not overwrite and os.path.exists(full_path): continue LocalStorage._makedirs(os.path.dirname(full_path)) - with open(full_path, mode='wb') as f: + with open(full_path, mode="wb") as f: f.write(byte_obj.read()) if metadata: - with open("%s_meta" % full_path, mode='w') as f: + with open("%s_meta" % full_path, mode="w") as f: json.dump(metadata, f) def load_bytes(self, paths): @@ -125,9 +127,10 @@ def iter_results(): metadata = None if os.path.exists(full_path): if os.path.exists("%s_meta" % full_path): - with open("%s_meta" % full_path, mode='r') as f: + with open("%s_meta" % full_path, mode="r") as f: metadata = json.load(f) yield path, full_path, metadata else: yield path, None, None + return CloseAfterUse(iter_results()) diff --git a/metaflow/datastore/s3_storage.py b/metaflow/datastore/s3_storage.py index d61b923296d..130d099ea4e 100644 --- a/metaflow/datastore/s3_storage.py +++ b/metaflow/datastore/s3_storage.py @@ -1,4 +1,3 @@ - import os from itertools import starmap @@ -17,7 +16,7 @@ class S3Storage(DataStoreStorage): - TYPE = 's3' + TYPE = "s3" def __init__(self, root=None): super(S3Storage, self).__init__(root) @@ -28,8 +27,11 @@ def get_datastore_root_from_config(cls, echo, create_on_absent=True): return DATASTORE_SYSROOT_S3 def is_file(self, paths): - with S3(s3root=self.datastore_root, - tmproot=os.getcwd(), external_client=self.s3_client) as s3: + with S3( + s3root=self.datastore_root, + tmproot=os.getcwd(), + external_client=self.s3_client, + ) as s3: if len(paths) > 10: s3objs = s3.info_many(paths, return_missing=True) return [s3obj.exists for s3obj in s3objs] @@ -40,24 +42,37 @@ def is_file(self, paths): return result def info_file(self, path): - with S3(s3root=self.datastore_root, - tmproot=os.getcwd(), external_client=self.s3_client) as s3: + with S3( + s3root=self.datastore_root, + tmproot=os.getcwd(), + external_client=self.s3_client, + ) as s3: s3obj = s3.info(path, return_missing=True) return s3obj.exists, s3obj.metadata def size_file(self, path): - with S3(s3root=self.datastore_root, - tmproot=os.getcwd(), external_client=self.s3_client) as s3: + with S3( + s3root=self.datastore_root, + tmproot=os.getcwd(), + external_client=self.s3_client, + ) as s3: s3obj = s3.info(path, return_missing=True) return s3obj.size def list_content(self, paths): - strip_prefix_len = len(self.datastore_root.rstrip('/')) + 1 - with S3(s3root=self.datastore_root, - tmproot=os.getcwd(), external_client=self.s3_client) as s3: + strip_prefix_len = len(self.datastore_root.rstrip("/")) + 1 + with S3( + s3root=self.datastore_root, + tmproot=os.getcwd(), + external_client=self.s3_client, + ) as s3: results = s3.list_paths(paths) - return [self.list_content_result( - path=o.url[strip_prefix_len:], is_file=o.exists) for o in results] + return [ + self.list_content_result( + path=o.url[strip_prefix_len:], is_file=o.exists + ) + for o in results + ] def save_bytes(self, path_and_bytes_iter, overwrite=False, len_hint=0): def _convert(): @@ -69,8 +84,11 @@ def _convert(): else: yield path, obj, None, None, None - with S3(s3root=self.datastore_root, - tmproot=os.getcwd(), external_client=self.s3_client) as s3: + with S3( + s3root=self.datastore_root, + tmproot=os.getcwd(), + external_client=self.s3_client, + ) as s3: # HACK: The S3 datatools we rely on does not currently do a good job # determining if uploading things in parallel is more efficient than # serially. We use a heuristic for now where if we have a lot of @@ -100,8 +118,11 @@ def load_bytes(self, paths): if len(paths) == 0: return CloseAfterUse(iter([])) - s3 = S3(s3root=self.datastore_root, - tmproot=os.getcwd(), external_client=self.s3_client) + s3 = S3( + s3root=self.datastore_root, + tmproot=os.getcwd(), + external_client=self.s3_client, + ) def iter_results(): # We similarly do things in parallel for many files. This is again @@ -120,4 +141,5 @@ def iter_results(): yield r.key, r.path, r.metadata else: yield r.key, None, None + return CloseAfterUse(iter_results(), closer=s3) diff --git a/metaflow/datastore/task_datastore.py b/metaflow/datastore/task_datastore.py index a5da8cb93fa..f04b129ebf2 100644 --- a/metaflow/datastore/task_datastore.py +++ b/metaflow/datastore/task_datastore.py @@ -15,16 +15,21 @@ from .exceptions import DataException, UnpicklableArtifactException + def only_if_not_done(f): @wraps(f) def method(self, *args, **kwargs): if self._is_done_set: - raise MetaflowInternalError("Tried to write to datastore "\ - "(method %s) after it was marked "\ - ".done()" % f.__name__) + raise MetaflowInternalError( + "Tried to write to datastore " + "(method %s) after it was marked " + ".done()" % f.__name__ + ) return f(self, *args, **kwargs) + return method + def require_mode(mode): def wrapper(f): @wraps(f) @@ -32,14 +37,19 @@ def method(self, *args, **kwargs): if mode is not None and self._mode != mode: raise MetaflowInternalError( "Attempting a datastore operation '%s' requiring mode '%s' " - "but have mode '%s'" % (f.__name__, mode, self._mode)) + "but have mode '%s'" % (f.__name__, mode, self._mode) + ) return f(self, *args, **kwargs) + return method + return wrapper + class ArtifactTooLarge(object): def __str__(self): - return '< artifact too large >' + return "< artifact too large >" + class TaskDataStore(object): """ @@ -61,29 +71,31 @@ class TaskDataStore(object): differently (they are not JSON-encodable dictionaries). """ - METADATA_ATTEMPT_SUFFIX = 'attempt.json' - METADATA_DONE_SUFFIX = 'DONE.lock' - METADATA_DATA_SUFFIX = 'data.json' + METADATA_ATTEMPT_SUFFIX = "attempt.json" + METADATA_DONE_SUFFIX = "DONE.lock" + METADATA_DATA_SUFFIX = "data.json" @staticmethod def metadata_name_for_attempt(name, attempt): if attempt is None: return name - return '%d.%s' % (attempt, name) + return "%d.%s" % (attempt, name) @staticmethod def parse_attempt_metadata(name): - return name.split('.', 1) - - def __init__(self, - flow_datastore, - run_id, - step_name, - task_id, - attempt=None, - data_metadata=None, - mode='r', - allow_not_done=False): + return name.split(".", 1) + + def __init__( + self, + flow_datastore, + run_id, + step_name, + task_id, + attempt=None, + data_metadata=None, + mode="r", + allow_not_done=False, + ): self._storage_impl = flow_datastore._storage_impl self.TYPE = self._storage_impl.TYPE @@ -93,30 +105,31 @@ def __init__(self, self._step_name = step_name self._task_id = task_id self._path = self._storage_impl.path_join( - flow_datastore.flow_name, run_id, step_name, task_id) + flow_datastore.flow_name, run_id, step_name, task_id + ) self._mode = mode self._attempt = attempt self._metadata = flow_datastore.metadata self._parent = flow_datastore # The GZIP encodings are for backward compatibility - self._encodings = {'pickle-v2', 'gzip+pickle-v2'} + self._encodings = {"pickle-v2", "gzip+pickle-v2"} ver = sys.version_info[0] * 10 + sys.version_info[1] if ver >= 34: - self._encodings.add('pickle-v4') - self._encodings.add('gzip+pickle-v4') + self._encodings.add("pickle-v4") + self._encodings.add("gzip+pickle-v4") self._is_done_set = False # If the mode is 'write', we initialize things to empty - if self._mode == 'w': + if self._mode == "w": self._objects = {} self._info = {} - elif self._mode == 'r': + elif self._mode == "r": if data_metadata is not None: # We already loaded the data metadata so just use that - self._objects = data_metadata.get('objects', {}) - self._info = data_metadata.get('info', {}) + self._objects = data_metadata.get("objects", {}) + self._info = data_metadata.get("info", {}) else: # What is the latest attempt ID for this task store. # NOTE: We *only* access to the data if the attempt that @@ -129,7 +142,8 @@ def __init__(self, max_attempt = None for i in range(metaflow_config.MAX_ATTEMPTS): check_meta = self._metadata_name_for_attempt( - self.METADATA_ATTEMPT_SUFFIX, i) + self.METADATA_ATTEMPT_SUFFIX, i + ) if self.has_metadata(check_meta, add_attempt=False): max_attempt = i if self._attempt is None: @@ -150,17 +164,18 @@ def __init__(self, elif self._attempt is None or not allow_not_done: raise DataException( "No completed attempts of the task was found for task '%s'" - % self._path) + % self._path + ) if data_obj is not None: - self._objects = data_obj.get('objects', {}) - self._info = data_obj.get('info', {}) + self._objects = data_obj.get("objects", {}) + self._info = data_obj.get("info", {}) else: raise DataException("Unknown datastore mode: '%s'" % self._mode) @property def pathspec(self): - return '/'.join([self.run_id, self.step_name, self.task_id]) + return "/".join([self.run_id, self.step_name, self.task_id]) @property def run_id(self): @@ -180,15 +195,12 @@ def attempt(self): @property def ds_metadata(self): - return { - 'objects': self._objects.copy(), - 'info': self._info.copy() - } + return {"objects": self._objects.copy(), "info": self._info.copy()} @property def pathspec_index(self): - idxstr = ','.join(map(str, (f.index for f in self['_foreach_stack']))) - return '%s/%s[%s]' % (self._run_id, self._step_name, idxstr) + idxstr = ",".join(map(str, (f.index for f in self["_foreach_stack"]))) + return "%s/%s[%s]" % (self._run_id, self._step_name, idxstr) @property def parent_datastore(self): @@ -198,26 +210,26 @@ def parent_datastore(self): def get_log_location(self, logprefix, stream): log_name = self._get_log_location(logprefix, stream) path = self._storage_impl.path_join( - self._path, self._metadata_name_for_attempt(log_name)) + self._path, self._metadata_name_for_attempt(log_name) + ) return self._storage_impl.full_uri(path) - @require_mode('r') + @require_mode("r") def keys_for_artifacts(self, names): return [self._objects.get(name) for name in names] @only_if_not_done - @require_mode('w') + @require_mode("w") def init_task(self): """ Call this to initialize the datastore with a new attempt. This method requires mode 'w'. """ - self.save_metadata( - {self.METADATA_ATTEMPT_SUFFIX: {'time': time.time()}}) + self.save_metadata({self.METADATA_ATTEMPT_SUFFIX: {"time": time.time()}}) @only_if_not_done - @require_mode('w') + @require_mode("w") def save_artifacts(self, artifacts_iter, force_v4=False, len_hint=0): """ Saves Metaflow Artifacts (Python objects) to the datastore and stores @@ -246,47 +258,51 @@ def save_artifacts(self, artifacts_iter, force_v4=False, len_hint=0): def pickle_iter(): for name, obj in artifacts_iter: - do_v4 = force_v4 and \ - force_v4 if isinstance(force_v4, bool) else \ - force_v4.get(name, False) + do_v4 = ( + force_v4 and force_v4 + if isinstance(force_v4, bool) + else force_v4.get(name, False) + ) if do_v4: - encode_type = 'gzip+pickle-v4' + encode_type = "gzip+pickle-v4" if encode_type not in self._encodings: raise DataException( "Artifact *%s* requires a serialization encoding that " - "requires Python 3.4 or newer." % name) + "requires Python 3.4 or newer." % name + ) try: blob = pickle.dumps(obj, protocol=4) - except TypeError as e: + except TypeError as e: raise UnpicklableArtifactException(name) else: try: blob = pickle.dumps(obj, protocol=2) - encode_type = 'gzip+pickle-v2' + encode_type = "gzip+pickle-v2" except (SystemError, OverflowError): - encode_type = 'gzip+pickle-v4' + encode_type = "gzip+pickle-v4" if encode_type not in self._encodings: raise DataException( "Artifact *%s* is very large (over 2GB). " "You need to use Python 3.4 or newer if you want to " - "serialize large objects." % name) + "serialize large objects." % name + ) try: blob = pickle.dumps(obj, protocol=4) - except TypeError as e: + except TypeError as e: raise UnpicklableArtifactException(name) - except TypeError as e: + except TypeError as e: raise UnpicklableArtifactException(name) self._info[name] = { - 'size': len(blob), - 'type': str(type(obj)), - 'encoding': encode_type + "size": len(blob), + "type": str(type(obj)), + "encoding": encode_type, } artifact_names.append(name) yield blob + # Use the content-addressed store to store all artifacts - save_result = self._ca_store.save_blobs(pickle_iter(), - len_hint=len_hint) + save_result = self._ca_store.save_blobs(pickle_iter(), len_hint=len_hint) for name, result in zip(artifact_names, save_result): self._objects[name] = result.key @@ -319,7 +335,8 @@ def load_artifacts(self, names): if not self._info: raise DataException( "Datastore for task '%s' does not have the required metadata to " - "load artifacts" % self._path) + "load artifacts" % self._path + ) to_load = [] sha_to_names = {} for name in names: @@ -328,12 +345,13 @@ def load_artifacts(self, names): # This datastore will always include the proper encoding version so # this is just to be able to read very old artifacts if info: - encode_type = info.get('encoding', 'gzip+pickle-v2') + encode_type = info.get("encoding", "gzip+pickle-v2") else: - encode_type = 'gzip+pickle-v2' + encode_type = "gzip+pickle-v2" if encode_type not in self._encodings: raise DataException( - "Python 3.4 or later is required to load artifact '%s'" % name) + "Python 3.4 or later is required to load artifact '%s'" % name + ) else: sha = self._objects[name] sha_to_names[sha] = name @@ -345,12 +363,12 @@ def load_artifacts(self, names): for sha, blob in self._ca_store.load_blobs(to_load): yield sha_to_names[sha], pickle.loads(blob) - @require_mode('r') + @require_mode("r") def get_artifact_sizes(self, names): """ Retrieves file sizes of artifacts defined in 'names' from their respective stored file metadata. - + Usage restricted to only 'r' mode due to depending on the metadata being written Parameters @@ -365,21 +383,20 @@ def get_artifact_sizes(self, names): """ for name in names: info = self._info.get(name) - yield name, info.get('size', 0) + yield name, info.get("size", 0) - @require_mode('r') + @require_mode("r") def get_legacy_log_size(self, stream): - name = self._metadata_name_for_attempt('%s.log' % stream) + name = self._metadata_name_for_attempt("%s.log" % stream) path = self._storage_impl.path_join(self._path, name) return self._storage_impl.size_file(path) - @require_mode('r') + @require_mode("r") def get_log_size(self, logsources, stream): def _path(s): # construct path for fetching of a single log source - _p = self._metadata_name_for_attempt( - self._get_log_location(s, stream)) + _p = self._metadata_name_for_attempt(self._get_log_location(s, stream)) return self._storage_impl.path_join(self._path, _p) paths = list(map(_path, logsources)) @@ -388,7 +405,7 @@ def _path(s): return sum(size for size in sizes if size is not None) @only_if_not_done - @require_mode('w') + @require_mode("w") def save_metadata(self, contents, allow_overwrite=True, add_attempt=True): """ Save task metadata. This is very similar to save_artifacts; this @@ -410,10 +427,12 @@ def save_metadata(self, contents, allow_overwrite=True, add_attempt=True): True """ return self._save_file( - {k: json.dumps(v).encode('utf-8') for k, v in contents.items()}, - allow_overwrite, add_attempt) + {k: json.dumps(v).encode("utf-8") for k, v in contents.items()}, + allow_overwrite, + add_attempt, + ) - @require_mode('r') + @require_mode("r") def load_metadata(self, names, add_attempt=True): """ Loads metadata saved with `save_metadata` @@ -431,8 +450,9 @@ def load_metadata(self, names, add_attempt=True): Dict: string -> JSON decoded object Results indexed by the name of the metadata loaded """ - return {k: json.loads(v) for k, v \ - in self._load_file(names, add_attempt).items()} + return { + k: json.loads(v) for k, v in self._load_file(names, add_attempt).items() + } @require_mode(None) def has_metadata(self, name, add_attempt=True): @@ -458,7 +478,8 @@ def has_metadata(self, name, add_attempt=True): """ if add_attempt: path = self._storage_impl.path_join( - self._path, self._metadata_name_for_attempt(name)) + self._path, self._metadata_name_for_attempt(name) + ) else: path = self._storage_impl.path_join(self._path, name) return self._storage_impl.is_file([path])[0] @@ -485,7 +506,7 @@ def get(self, name, default=None): return default return default - @require_mode('r') + @require_mode("r") def is_none(self, name): """ Convenience method to test if an artifact is None @@ -501,7 +522,7 @@ def is_none(self, name): return True info = self._info.get(name) if info: - obj_type = info.get('type') + obj_type = info.get("type") # Conservatively check if the actual object is None, # in case the artifact is stored using a different python version. # Note that if an object is None and stored in Py2 and accessed in @@ -513,47 +534,61 @@ def is_none(self, name): return self.get(name) is None @only_if_not_done - @require_mode('w') + @require_mode("w") def done(self): """ Mark this task-datastore as 'done' for the current attempt Will throw an exception if mode != 'w' """ - self.save_metadata({ - self.METADATA_DATA_SUFFIX: - {'datastore': self.TYPE, - 'version': '1.0', - 'attempt': self._attempt, - 'python_version': sys.version, - 'objects': self._objects, - 'info': self._info}, - self.METADATA_DONE_SUFFIX: "" - }) + self.save_metadata( + { + self.METADATA_DATA_SUFFIX: { + "datastore": self.TYPE, + "version": "1.0", + "attempt": self._attempt, + "python_version": sys.version, + "objects": self._objects, + "info": self._info, + }, + self.METADATA_DONE_SUFFIX: "", + } + ) if self._metadata: self._metadata.register_metadata( - self._run_id, self._step_name, self._task_id, - [MetaDatum(field='attempt-done', value=str(self._attempt), - type='attempt-done', - tags=['attempt_id:{0}'.format(self._attempt)])]) - artifacts = [DataArtifact( - name=var, - ds_type=self.TYPE, - ds_root=self._storage_impl.datastore_root, - url=None, - sha=sha, - type=self._info[var]['encoding']) - for var, sha in self._objects.items()] + self._run_id, + self._step_name, + self._task_id, + [ + MetaDatum( + field="attempt-done", + value=str(self._attempt), + type="attempt-done", + tags=["attempt_id:{0}".format(self._attempt)], + ) + ], + ) + artifacts = [ + DataArtifact( + name=var, + ds_type=self.TYPE, + ds_root=self._storage_impl.datastore_root, + url=None, + sha=sha, + type=self._info[var]["encoding"], + ) + for var, sha in self._objects.items() + ] self._metadata.register_data_artifacts( - self.run_id, self.step_name, self.task_id, self._attempt, - artifacts) + self.run_id, self.step_name, self.task_id, self._attempt, artifacts + ) self._is_done_set = True @only_if_not_done - @require_mode('w') + @require_mode("w") def clone(self, origin): """ Clone the information located in the TaskDataStore origin into this @@ -568,7 +603,7 @@ def clone(self, origin): self._info = origin._info @only_if_not_done - @require_mode('w') + @require_mode("w") def passdown_partial(self, origin, variables): # Pass-down from datastore origin all information related to vars to # this datastore. In other words, this adds to the current datastore all @@ -583,7 +618,7 @@ def passdown_partial(self, origin, variables): self._info[var] = origin._info[var] @only_if_not_done - @require_mode('w') + @require_mode("w") def persist(self, flow): """ Persist any new artifacts that were produced when running flow @@ -606,17 +641,20 @@ def persist(self, flow): # artifacts_iter so we can provide a len_hint below valid_artifacts = [] for var in dir(flow): - if var.startswith('__') or var in flow._EPHEMERAL: + if var.startswith("__") or var in flow._EPHEMERAL: continue # Skip over properties of the class (Parameters) - if hasattr(flow.__class__, var) and \ - isinstance(getattr(flow.__class__, var), property): + if hasattr(flow.__class__, var) and isinstance( + getattr(flow.__class__, var), property + ): continue val = getattr(flow, var) - if not (isinstance(val, MethodType) or - isinstance(val, FunctionType) or - isinstance(val, Parameter)): + if not ( + isinstance(val, MethodType) + or isinstance(val, FunctionType) + or isinstance(val, Parameter) + ): valid_artifacts.append((var, val)) def artifacts_iter(): @@ -626,7 +664,7 @@ def artifacts_iter(): # artifacts in memory simultaneously while valid_artifacts: var, val = valid_artifacts.pop() - if not var.startswith('_') and var != 'name': + if not var.startswith("_") and var != "name": # NOTE: Destructive mutation of the flow object. We keep # around artifacts called 'name' and anything starting with # '_' as they are used by the Metaflow runtime. @@ -636,7 +674,7 @@ def artifacts_iter(): self.save_artifacts(artifacts_iter(), len_hint=len(valid_artifacts)) @only_if_not_done - @require_mode('w') + @require_mode("w") def save_logs(self, logsource, stream_data): """ Save log files for multiple streams, represented as @@ -657,29 +695,36 @@ def save_logs(self, logsource, stream_data): for stream, data in stream_data.items(): n = self._get_log_location(logsource, stream) if isinstance(data, Path): - to_store_dict[n] = FileIO(str(data), mode='r') + to_store_dict[n] = FileIO(str(data), mode="r") else: to_store_dict[n] = data self._save_file(to_store_dict) - @require_mode('r') + @require_mode("r") def load_log_legacy(self, stream, attempt_override=None): """ Load old-style, pre-mflog, log file represented as a bytes object. """ - name = self._metadata_name_for_attempt( - '%s.log' % stream, attempt_override) + name = self._metadata_name_for_attempt("%s.log" % stream, attempt_override) r = self._load_file([name], add_attempt=False)[name] - return r if r is not None else b'' + return r if r is not None else b"" - @require_mode('r') + @require_mode("r") def load_logs(self, logsources, stream, attempt_override=None): - paths = dict(map( - lambda s: (self._metadata_name_for_attempt( - self._get_log_location(s, stream), - attempt_override=attempt_override), s), logsources)) + paths = dict( + map( + lambda s: ( + self._metadata_name_for_attempt( + self._get_log_location(s, stream), + attempt_override=attempt_override, + ), + s, + ), + logsources, + ) + ) r = self._load_file(paths.keys(), add_attempt=False) - return [(paths[k], v if v is not None else b'') for k, v in r.items()] + return [(paths[k], v if v is not None else b"") for k, v in r.items()] @require_mode(None) def items(self): @@ -693,22 +738,23 @@ def to_dict(self, show_private=False, max_value_size=None, include=None): for k, _ in self.items(): if include and k not in include: continue - if k[0] == '_' and not show_private: + if k[0] == "_" and not show_private: continue - if max_value_size is not None and\ - self._info[k]['size'] > max_value_size: + if max_value_size is not None and self._info[k]["size"] > max_value_size: d[k] = ArtifactTooLarge() else: d[k] = self[k] return d - @require_mode('r') + @require_mode("r") def format(self, **kwargs): def lines(): for k, v in self.to_dict(**kwargs).items(): - yield k, '*{key}* [size: {size} type: {type}] = {value}'\ - .format(key=k, value=v, **self._info[k]) - return '\n'.join(line for k, line in sorted(lines())) + yield k, "*{key}* [size: {size} type: {type}] = {value}".format( + key=k, value=v, **self._info[k] + ) + + return "\n".join(line for k, line in sorted(lines())) @require_mode(None) def __contains__(self, name): @@ -721,24 +767,24 @@ def __getitem__(self, name): _, obj = next(self.load_artifacts([name])) return obj - @require_mode('r') + @require_mode("r") def __iter__(self): if self._objects: return iter(self._objects) return iter([]) - @require_mode('r') + @require_mode("r") def __str__(self): return self.format(show_private=True, max_value_size=1000) def _metadata_name_for_attempt(self, name, attempt_override=None): return self.metadata_name_for_attempt( - name, self._attempt if attempt_override is None else - attempt_override) + name, self._attempt if attempt_override is None else attempt_override + ) @staticmethod def _get_log_location(logprefix, stream): - return '%s_%s.log' % (logprefix, stream) + return "%s_%s.log" % (logprefix, stream) def _save_file(self, contents, allow_overwrite=True, add_attempt=True): """ @@ -756,21 +802,25 @@ def _save_file(self, contents, allow_overwrite=True, add_attempt=True): If True, adds the attempt identifier to the metadata, defaults to True """ + def blob_iter(): for name, value in contents.items(): if add_attempt: path = self._storage_impl.path_join( - self._path, self._metadata_name_for_attempt(name)) + self._path, self._metadata_name_for_attempt(name) + ) else: path = self._storage_impl.path_join(self._path, name) - if isinstance(value, (RawIOBase, BufferedIOBase)) and \ - value.readable(): + if isinstance(value, (RawIOBase, BufferedIOBase)) and value.readable(): yield path, value elif is_stringish(value): yield path, to_fileobj(value) else: - raise DataException("Metadata '%s' for task '%s' has an invalid type: %s" % - (name, self._path, type(value))) + raise DataException( + "Metadata '%s' for task '%s' has an invalid type: %s" + % (name, self._path, type(value)) + ) + self._storage_impl.save_bytes(blob_iter(), overwrite=allow_overwrite) def _load_file(self, names, add_attempt=True): @@ -795,7 +845,8 @@ def _load_file(self, names, add_attempt=True): for name in names: if add_attempt: path = self._storage_impl.path_join( - self._path, self._metadata_name_for_attempt(name)) + self._path, self._metadata_name_for_attempt(name) + ) else: path = self._storage_impl.path_join(self._path, name) to_load.append(path) @@ -804,12 +855,13 @@ def _load_file(self, names, add_attempt=True): for key, path, meta in load_results: if add_attempt: _, name = self.parse_attempt_metadata( - self._storage_impl.basename(key)) + self._storage_impl.basename(key) + ) else: name = self._storage_impl.basename(key) if path is None: results[name] = None else: - with open(path, 'rb') as f: + with open(path, "rb") as f: results[name] = f.read() return results diff --git a/metaflow/datatools/__init__.py b/metaflow/datatools/__init__.py index d65626be034..3fc48796f46 100644 --- a/metaflow/datatools/__init__.py +++ b/metaflow/datatools/__init__.py @@ -30,11 +30,14 @@ def read_in_chunks(dst, src, src_sz, max_chunk_size): # e.name is set to the name of the package that fails to load # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) - if not (isinstance(e, ModuleNotFoundError) and \ - e.name in ['metaflow_extensions', 'metaflow_extensions.datatools']): + if not ( + isinstance(e, ModuleNotFoundError) + and e.name in ["metaflow_extensions", "metaflow_extensions.datatools"] + ): print( "Cannot load metaflow_extensions exceptions -- " - "if you want to ignore, uninstall metaflow_extensions package") + "if you want to ignore, uninstall metaflow_extensions package" + ) raise else: # We load into globals whatever we have in extension_module @@ -42,28 +45,41 @@ def read_in_chunks(dst, src, src_sz, max_chunk_size): # *except* for ones that are part of metaflow_extensions (basically providing # an aliasing mechanism) lazy_load_custom_modules = {} - addl_modules = extension_module.__dict__.get('__mf_promote_submodules__') + addl_modules = extension_module.__dict__.get("__mf_promote_submodules__") if addl_modules: # We make an alias for these modules which the metaflow_extensions author # wants to expose but that may not be loaded yet lazy_load_custom_modules = { - 'metaflow.datatools.%s' % k: 'metaflow_extensions.datatools.%s' % k - for k in addl_modules} + "metaflow.datatools.%s" % k: "metaflow_extensions.datatools.%s" % k + for k in addl_modules + } for n, o in extension_module.__dict__.items(): - if not n.startswith('__') and not isinstance(o, types.ModuleType): + if not n.startswith("__") and not isinstance(o, types.ModuleType): globals()[n] = o - elif isinstance(o, types.ModuleType) and o.__package__ and \ - o.__package__.startswith('metaflow_extensions'): - lazy_load_custom_modules['metaflow.datatools.%s' % n] = o + elif ( + isinstance(o, types.ModuleType) + and o.__package__ + and o.__package__.startswith("metaflow_extensions") + ): + lazy_load_custom_modules["metaflow.datatools.%s" % n] = o if lazy_load_custom_modules: from metaflow import _LazyLoader + sys.meta_path = [_LazyLoader(lazy_load_custom_modules)] + sys.meta_path finally: # Erase all temporary names to avoid leaking things - for _n in ['ver', 'n', 'o', 'e', 'lazy_load_custom_modules', - 'extension_module', '_LazyLoader', 'addl_modules']: + for _n in [ + "ver", + "n", + "o", + "e", + "lazy_load_custom_modules", + "extension_module", + "_LazyLoader", + "addl_modules", + ]: try: del globals()[_n] except KeyError: pass - del globals()['_n'] + del globals()["_n"] diff --git a/metaflow/datatools/s3.py b/metaflow/datatools/s3.py index 85e6dfb3ccf..e08c9dc679d 100644 --- a/metaflow/datatools/s3.py +++ b/metaflow/datatools/s3.py @@ -12,13 +12,15 @@ from .. import FlowSpec from ..current import current from ..metaflow_config import DATATOOLS_S3ROOT, S3_RETRY_COUNT -from ..util import namedtuple_with_defaults,\ - is_stringish,\ - to_bytes,\ - to_unicode,\ - to_fileobj,\ - url_quote,\ - url_unquote +from ..util import ( + namedtuple_with_defaults, + is_stringish, + to_bytes, + to_unicode, + to_fileobj, + url_quote, + url_unquote, +) from ..exception import MetaflowException from ..debug import debug @@ -34,40 +36,49 @@ try: import boto3 from boto3.s3.transfer import TransferConfig + DOWNLOAD_FILE_THRESHOLD = 2 * TransferConfig().multipart_threshold DOWNLOAD_MAX_CHUNK = 2 * 1024 * 1024 * 1024 - 1 boto_found = True except: boto_found = False + def ensure_unicode(x): return None if x is None else to_unicode(x) -S3GetObject = namedtuple_with_defaults( - 'S3GetObject', 'key offset length') + +S3GetObject = namedtuple_with_defaults("S3GetObject", "key offset length") S3PutObject = namedtuple_with_defaults( - 'S3PutObject', 'key value path content_type metadata', - defaults=(None, None, None, None)) + "S3PutObject", + "key value path content_type metadata", + defaults=(None, None, None, None), +) RangeInfo = namedtuple_with_defaults( - 'RangeInfo', 'total_size request_offset request_length', - defaults=(0, -1)) + "RangeInfo", "total_size request_offset request_length", defaults=(0, -1) +) + class MetaflowS3InvalidObject(MetaflowException): - headline = 'Not a string-like object' + headline = "Not a string-like object" + class MetaflowS3URLException(MetaflowException): - headline = 'Invalid address' + headline = "Invalid address" + class MetaflowS3Exception(MetaflowException): - headline = 'S3 access failed' + headline = "S3 access failed" + class MetaflowS3NotFound(MetaflowException): - headline = 'S3 object not found' + headline = "S3 object not found" + class MetaflowS3AccessDenied(MetaflowException): - headline = 'S3 access denied' + headline = "S3 access denied" class S3Object(object): @@ -78,8 +89,15 @@ class S3Object(object): """ def __init__( - self, prefix, url, path, - size=None, content_type=None, metadata=None, range_info=None): + self, + prefix, + url, + path, + size=None, + content_type=None, + metadata=None, + range_info=None, + ): # all fields of S3Object should return a unicode object prefix, url, path = map(ensure_unicode, (prefix, url, path)) @@ -91,14 +109,15 @@ def __init__( self._content_type = content_type self._metadata = None - if metadata is not None and 'metaflow-user-attributes' in metadata: - self._metadata = json.loads(metadata['metaflow-user-attributes']) + if metadata is not None and "metaflow-user-attributes" in metadata: + self._metadata = json.loads(metadata["metaflow-user-attributes"]) - if range_info and (range_info.request_length is None or\ - range_info.request_length < 0): + if range_info and ( + range_info.request_length is None or range_info.request_length < 0 + ): self._range_info = RangeInfo( - range_info.total_size, - range_info.request_offset, range_info.total_size) + range_info.total_size, range_info.request_offset, range_info.total_size + ) else: self._range_info = range_info @@ -109,7 +128,7 @@ def __init__( self._key = url self._prefix = None else: - self._key = url[len(prefix.rstrip('/')) + 1:].rstrip('/') + self._key = url[len(prefix.rstrip("/")) + 1 :].rstrip("/") self._prefix = prefix @property @@ -165,7 +184,7 @@ def blob(self): Returns None if this S3Object has not been downloaded. """ if self._path: - with open(self._path, 'rb') as f: + with open(self._path, "rb") as f: return f.read() @property @@ -175,7 +194,7 @@ def text(self): Returns None if this S3Object has not been downloaded. """ if self._path: - return self.blob.decode('utf-8', errors='replace') + return self.blob.decode("utf-8", errors="replace") @property def size(self): @@ -220,11 +239,11 @@ def range_info(self): def __str__(self): if self._path: - return '' % (self._url, self._size) + return "" % (self._url, self._size) elif self._size: - return '' % (self._url, self._size) + return "" % (self._url, self._size) else: - return '' % self._url + return "" % self._url def __repr__(self): return str(self) @@ -252,18 +271,13 @@ def reset_client(self): class S3(object): - @classmethod def get_root_from_config(cls, echo, create_on_absent=True): return DATATOOLS_S3ROOT - def __init__(self, - tmproot='.', - bucket=None, - prefix=None, - run=None, - s3root=None, - **kwargs): + def __init__( + self, tmproot=".", bucket=None, prefix=None, run=None, s3root=None, **kwargs + ): """ Initialize a new context for S3 operations. This object is used as a context manager for a with statement. @@ -294,30 +308,30 @@ def __init__(self, prefix = parsed.path if isinstance(run, FlowSpec): if current.is_running_flow: - prefix = os.path.join(prefix, - current.flow_name, - current.run_id) + prefix = os.path.join(prefix, current.flow_name, current.run_id) else: - raise MetaflowS3URLException(\ + raise MetaflowS3URLException( "Initializing S3 with a FlowSpec outside of a running " - "flow is not supported.") + "flow is not supported." + ) else: prefix = os.path.join(prefix, run.parent.id, run.id) - self._s3root = u's3://%s' % os.path.join(bucket, prefix.strip('/')) + self._s3root = u"s3://%s" % os.path.join(bucket, prefix.strip("/")) elif s3root: # 2. use an explicit S3 prefix parsed = urlparse(to_unicode(s3root)) - if parsed.scheme != 's3': - raise MetaflowS3URLException(\ - "s3root needs to be an S3 URL prefxied with s3://.") - self._s3root = s3root.rstrip('/') + if parsed.scheme != "s3": + raise MetaflowS3URLException( + "s3root needs to be an S3 URL prefxied with s3://." + ) + self._s3root = s3root.rstrip("/") else: # 3. use the client only with full URLs self._s3root = None - self._s3_client = kwargs.get('external_client', S3Client()) - self._tmpdir = mkdtemp(dir=tmproot, prefix='metaflow.s3.') + self._s3_client = kwargs.get("external_client", S3Client()) + self._tmpdir = mkdtemp(dir=tmproot, prefix="metaflow.s3.") def __enter__(self): return self @@ -342,34 +356,37 @@ def _url(self, key_value): # string in py3) internally. We expect that all URLs passed to this # class as either Unicode or UTF-8 encoded byte strings. All URLs # returned are Unicode. - key = getattr(key_value, 'key', key_value) + key = getattr(key_value, "key", key_value) if self._s3root is None: parsed = urlparse(to_unicode(key)) - if parsed.scheme == 's3' and parsed.path: + if parsed.scheme == "s3" and parsed.path: return key else: if current.is_running_flow: - raise MetaflowS3URLException(\ + raise MetaflowS3URLException( "Specify S3(run=self) when you use S3 inside a running " "flow. Otherwise you have to use S3 with full " - "s3:// urls.") + "s3:// urls." + ) else: - raise MetaflowS3URLException(\ + raise MetaflowS3URLException( "Initialize S3 with an 's3root' or 'run' if you don't " - "want to specify full s3:// urls.") + "want to specify full s3:// urls." + ) elif key: - if key.startswith('s3://'): - raise MetaflowS3URLException(\ + if key.startswith("s3://"): + raise MetaflowS3URLException( "Don't use absolute S3 URLs when the S3 client is " - "initialized with a prefix. URL: %s" % key) + "initialized with a prefix. URL: %s" % key + ) return os.path.join(self._s3root, key) else: return self._s3root def _url_and_range(self, key_value): url = self._url(key_value) - start = getattr(key_value, 'offset', None) - length = getattr(key_value, 'length', None) + start = getattr(key_value, "offset", None) + length = getattr(key_value, "length", None) range_str = None # Range specification are inclusive so getting from offset 500 for 100 # bytes will read as bytes=500-599 @@ -387,7 +404,6 @@ def _url_and_range(self, key_value): range_str = "bytes=%d-%d" % (start, start + length - 1) return url, range_str - def list_paths(self, keys=None): """ List the next level of paths in S3. If multiple keys are @@ -406,11 +422,12 @@ def list_paths(self, keys=None): first S3Object has .exists == False, since it does not refer to an object in S3. It is just a prefix. """ + def _list(keys): if keys is None: keys = [None] - urls = ((self._url(key).rstrip('/') + '/', None) for key in keys) - res = self._read_many_files('list', urls) + urls = ((self._url(key).rstrip("/") + "/", None) for key in keys) + res = self._read_many_files("list", urls) for s3prefix, s3url, size in res: if size: yield s3prefix, s3url, None, int(size) @@ -435,14 +452,16 @@ def list_recursive(self, keys=None): D/E In this case, list_recursive(['A', 'D']), returns ['A/B/C', 'D/E']. """ + def _list(keys): if keys is None: keys = [None] - res = self._read_many_files('list', - map(self._url_and_range, keys), - recursive=True) + res = self._read_many_files( + "list", map(self._url_and_range, keys), recursive=True + ) for s3prefix, s3url, size in res: yield s3prefix, s3url, None, int(size) + return list(starmap(S3Object, _list(keys))) def info(self, key=None, return_missing=False): @@ -464,9 +483,10 @@ def info(self, key=None, return_missing=False): def _info(s3, tmp): resp = s3.head_object(Bucket=src.netloc, Key=src.path.lstrip('/"')) return { - 'content_type': resp['ContentType'], - 'metadata': resp['Metadata'], - 'size': resp['ContentLength']} + "content_type": resp["ContentType"], + "metadata": resp["Metadata"], + "size": resp["ContentLength"], + } info_results = None try: @@ -478,11 +498,13 @@ def _info(s3, tmp): raise if info_results: return S3Object( - self._s3root, url, + self._s3root, + url, path=None, - size=info_results['size'], - content_type=info_results['content_type'], - metadata=info_results['metadata']) + size=info_results["size"], + content_type=info_results["content_type"], + metadata=info_results["metadata"], + ) return S3Object(self._s3root, url, None) def info_many(self, keys, return_missing=False): @@ -498,36 +520,39 @@ def info_many(self, keys, return_missing=False): downloaded property will be false and exists will indicate whether or not the file exists. """ + def _head(): from . import s3op - res = self._read_many_files('info', - map(self._url_and_range, keys), - verbose=False, - listing=True) + + res = self._read_many_files( + "info", map(self._url_and_range, keys), verbose=False, listing=True + ) for s3prefix, s3url, fname in res: if fname: # We have a metadata file to read from - with open(os.path.join(self._tmpdir, fname), 'r') as f: + with open(os.path.join(self._tmpdir, fname), "r") as f: info = json.load(f) - if info['error'] is not None: + if info["error"] is not None: # We have an error, we check if it is a missing file - if info['error'] == s3op.ERROR_URL_NOT_FOUND: + if info["error"] == s3op.ERROR_URL_NOT_FOUND: if return_missing: yield self._s3root, s3url, None else: raise MetaflowS3NotFound() - elif info['error'] == s3op.ERROR_URL_ACCESS_DENIED: + elif info["error"] == s3op.ERROR_URL_ACCESS_DENIED: raise MetaflowS3AccessDenied() else: - raise MetaflowS3Exception("Got error: %d" % info['error']) + raise MetaflowS3Exception("Got error: %d" % info["error"]) else: - yield self._s3root, s3url, None, \ - info['size'], info['content_type'], info['metadata'] + yield self._s3root, s3url, None, info["size"], info[ + "content_type" + ], info["metadata"] else: # This should not happen; we should always get a response # even if it contains an error inside it raise MetaflowS3Exception("Did not get a response to HEAD") + return list(starmap(S3Object, _head())) def get(self, key=None, return_missing=False, return_info=True): @@ -551,26 +576,23 @@ def get(self, key=None, return_missing=False, return_info=True): def _download(s3, tmp): if r: resp = s3.get_object( - Bucket=src.netloc, - Key=src.path.lstrip('/'), - Range=r) + Bucket=src.netloc, Key=src.path.lstrip("/"), Range=r + ) else: - resp = s3.get_object( - Bucket=src.netloc, - Key=src.path.lstrip('/')) - sz = resp['ContentLength'] + resp = s3.get_object(Bucket=src.netloc, Key=src.path.lstrip("/")) + sz = resp["ContentLength"] if not r and sz > DOWNLOAD_FILE_THRESHOLD: # In this case, it is more efficient to use download_file as it # will download multiple parts in parallel (it does it after # multipart_threshold) - s3.download_file(src.netloc, src.path.lstrip('/'), tmp) + s3.download_file(src.netloc, src.path.lstrip("/"), tmp) else: - with open(tmp, mode='wb') as t: - read_in_chunks(t, resp['Body'], sz, DOWNLOAD_MAX_CHUNK) + with open(tmp, mode="wb") as t: + read_in_chunks(t, resp["Body"], sz, DOWNLOAD_MAX_CHUNK) if return_info: return { - 'content_type': resp['ContentType'], - 'metadata': resp['Metadata'] + "content_type": resp["ContentType"], + "metadata": resp["Metadata"], } return None @@ -584,9 +606,12 @@ def _download(s3, tmp): raise if addl_info: return S3Object( - self._s3root, url, path, - content_type=addl_info['content_type'], - metadata=addl_info['metadata']) + self._s3root, + url, + path, + content_type=addl_info["content_type"], + metadata=addl_info["metadata"], + ) return S3Object(self._s3root, url, path) def get_many(self, keys, return_missing=False, return_info=True): @@ -605,24 +630,29 @@ def get_many(self, keys, return_missing=False, return_info=True): Returns: a list of S3Objects corresponding to the objects requested. """ + def _get(): - res = self._read_many_files('get', - map(self._url_and_range, keys), - allow_missing=return_missing, - verify=True, - verbose=False, - info=return_info, - listing=True) + res = self._read_many_files( + "get", + map(self._url_and_range, keys), + allow_missing=return_missing, + verify=True, + verbose=False, + info=return_info, + listing=True, + ) for s3prefix, s3url, fname in res: if return_info: if fname: # We have a metadata file to read from - with open(os.path.join(self._tmpdir, '%s_meta' % fname), - 'r') as f: + with open( + os.path.join(self._tmpdir, "%s_meta" % fname), "r" + ) as f: info = json.load(f) - yield self._s3root, s3url, os.path.join(self._tmpdir, fname), \ - None, info['content_type'], info['metadata'] + yield self._s3root, s3url, os.path.join( + self._tmpdir, fname + ), None, info["content_type"], info["metadata"] else: yield self._s3root, s3prefix, None else: @@ -631,6 +661,7 @@ def _get(): else: # missing entries per return_missing=True yield self._s3root, s3prefix, None + return list(starmap(S3Object, _get())) def get_recursive(self, keys, return_info=False): @@ -644,25 +675,29 @@ def get_recursive(self, keys, return_info=False): Returns: a list of S3Objects corresponding to the objects requested. """ + def _get(): - res = self._read_many_files('get', - map(self._url_and_range, keys), - recursive=True, - verify=True, - verbose=False, - info=return_info, - listing=True) + res = self._read_many_files( + "get", + map(self._url_and_range, keys), + recursive=True, + verify=True, + verbose=False, + info=return_info, + listing=True, + ) for s3prefix, s3url, fname in res: if return_info: # We have a metadata file to read from - with open(os.path.join(self._tmpdir, '%s_meta' % fname), - 'r') as f: + with open(os.path.join(self._tmpdir, "%s_meta" % fname), "r") as f: info = json.load(f) - yield self._s3root, s3url, os.path.join(self._tmpdir, fname), \ - None, info['content_type'], info['metadata'] + yield self._s3root, s3url, os.path.join( + self._tmpdir, fname + ), None, info["content_type"], info["metadata"] else: yield s3prefix, s3url, os.path.join(self._tmpdir, fname) + return list(starmap(S3Object, _get())) def get_all(self, return_info=False): @@ -676,8 +711,9 @@ def get_all(self, return_info=False): a list of S3Objects corresponding to the objects requested. """ if self._s3root is None: - raise MetaflowS3URLException(\ - "Can't get_all() when S3 is initialized without a prefix") + raise MetaflowS3URLException( + "Can't get_all() when S3 is initialized without a prefix" + ) else: return self.get_recursive([None], return_info) @@ -698,14 +734,16 @@ def put(self, key, obj, overwrite=True, content_type=None, metadata=None): if isinstance(obj, (RawIOBase, BufferedIOBase)): if not obj.readable() or not obj.seekable(): raise MetaflowS3InvalidObject( - "Object corresponding to the key '%s' is not readable or seekable" % - key) + "Object corresponding to the key '%s' is not readable or seekable" + % key + ) blob = obj else: if not is_stringish(obj): - raise MetaflowS3InvalidObject(\ + raise MetaflowS3InvalidObject( "Object corresponding to the key '%s' is not a string " - "or a bytes object." % key) + "or a bytes object." % key + ) blob = to_fileobj(obj) # We override the close functionality to prevent closing of the # file if it is used multiple times when uploading (since upload_fileobj @@ -719,24 +757,27 @@ def put(self, key, obj, overwrite=True, content_type=None, metadata=None): if content_type or metadata: extra_args = {} if content_type: - extra_args['ContentType'] = content_type + extra_args["ContentType"] = content_type if metadata: - extra_args['Metadata'] = { - 'metaflow-user-attributes': json.dumps(metadata)} - + extra_args["Metadata"] = { + "metaflow-user-attributes": json.dumps(metadata) + } + def _upload(s3, _): # We make sure we are at the beginning in case we are retrying blob.seek(0) s3.upload_fileobj( - blob, src.netloc, src.path.lstrip('/'), ExtraArgs=extra_args) + blob, src.netloc, src.path.lstrip("/"), ExtraArgs=extra_args + ) if overwrite: self._one_boto_op(_upload, url, create_tmp_file=False) real_close() return url else: + def _head(s3, _): - s3.head_object(Bucket=src.netloc, Key=src.path.lstrip('/')) + s3.head_object(Bucket=src.netloc, Key=src.path.lstrip("/")) try: self._one_boto_op(_head, url, create_tmp_file=False) @@ -760,6 +801,7 @@ def put_many(self, key_objs, overwrite=True): Returns: a list of (key, S3 URL) tuples corresponding to the files sent. """ + def _store(): for key_obj in key_objs: if isinstance(key_obj, tuple): @@ -769,35 +811,39 @@ def _store(): key = key_obj.key obj = key_obj.value store_info = { - 'key': key, - 'content_type': getattr(key_obj, 'content_type', None) + "key": key, + "content_type": getattr(key_obj, "content_type", None), } - metadata = getattr(key_obj, 'metadata', None) + metadata = getattr(key_obj, "metadata", None) if metadata: - store_info['metadata'] = { - 'metaflow-user-attributes': json.dumps(metadata)} + store_info["metadata"] = { + "metaflow-user-attributes": json.dumps(metadata) + } if isinstance(obj, (RawIOBase, BufferedIOBase)): if not obj.readable() or not obj.seekable(): raise MetaflowS3InvalidObject( - "Object corresponding to the key '%s' is not readable or seekable" % - key) + "Object corresponding to the key '%s' is not readable or seekable" + % key + ) else: if not is_stringish(obj): - raise MetaflowS3InvalidObject(\ + raise MetaflowS3InvalidObject( "Object corresponding to the key '%s' is not a string " - "or a bytes object." % key) + "or a bytes object." % key + ) obj = to_fileobj(obj) - with NamedTemporaryFile(dir=self._tmpdir, - delete=False, - mode='wb', - prefix='metaflow.s3.put_many.') as tmp: + with NamedTemporaryFile( + dir=self._tmpdir, + delete=False, + mode="wb", + prefix="metaflow.s3.put_many.", + ) as tmp: tmp.write(obj.read()) tmp.close() yield tmp.name, self._url(key), store_info return self._put_many_files(_store(), overwrite) - def put_files(self, key_paths, overwrite=True): """ Put files to S3 in parallel. @@ -811,6 +857,7 @@ def put_files(self, key_paths, overwrite=True): Returns: a list of (key, S3 URL) tuples corresponding to the files sent. """ + def _check(): for key_path in key_paths: if isinstance(key_path, tuple): @@ -820,13 +867,14 @@ def _check(): key = key_path.key path = key_path.path store_info = { - 'key': key, - 'content_type': getattr(key_path, 'content_type', None), + "key": key, + "content_type": getattr(key_path, "content_type", None), } - metadata = getattr(key_path, 'metadata', None) + metadata = getattr(key_path, "metadata", None) if metadata: - store_info['metadata'] = { - 'metaflow-user-attributes': json.dumps(metadata)} + store_info["metadata"] = { + "metaflow-user-attributes": json.dumps(metadata) + } if not os.path.exists(path): raise MetaflowS3NotFound("Local file not found: %s" % path) yield path, self._url(key), store_info @@ -834,25 +882,25 @@ def _check(): return self._put_many_files(_check(), overwrite) def _one_boto_op(self, op, url, create_tmp_file=True): - error = '' + error = "" for i in range(S3_RETRY_COUNT + 1): tmp = None if create_tmp_file: - tmp = NamedTemporaryFile(dir=self._tmpdir, - prefix='metaflow.s3.one_file.', - delete=False) + tmp = NamedTemporaryFile( + dir=self._tmpdir, prefix="metaflow.s3.one_file.", delete=False + ) try: - side_results = op( - self._s3_client.client, tmp.name if tmp else None) + side_results = op(self._s3_client.client, tmp.name if tmp else None) return tmp.name if tmp else None, side_results except self._s3_client.error as err: from . import s3op + error_code = s3op.normalize_client_error(err) if error_code == 404: raise MetaflowS3NotFound(url) elif error_code == 403: raise MetaflowS3AccessDenied(url) - elif error_code == 'NoSuchBucket': + elif error_code == "NoSuchBucket": raise MetaflowS3URLException("Specified S3 bucket doesn't exist.") error = str(err) except Exception as ex: @@ -862,100 +910,119 @@ def _one_boto_op(self, op, url, create_tmp_file=True): os.unlink(tmp.name) self._s3_client.reset_client() # add some jitter to make sure retries are not synchronized - time.sleep(2**i + random.randint(0, 10)) - raise MetaflowS3Exception("S3 operation failed.\n"\ - "Key requested: %s\n"\ - "Error: %s" % (url, error)) + time.sleep(2 ** i + random.randint(0, 10)) + raise MetaflowS3Exception( + "S3 operation failed.\n" "Key requested: %s\n" "Error: %s" % (url, error) + ) # NOTE: re: _read_many_files and _put_many_files # All file IO is through binary files - we write bytes, we read # bytes. All inputs and outputs from these functions are Unicode. - # Conversion between bytes and unicode is done through + # Conversion between bytes and unicode is done through # and url_unquote. def _read_many_files(self, op, prefixes_and_ranges, **options): prefixes_and_ranges = list(prefixes_and_ranges) - with NamedTemporaryFile(dir=self._tmpdir, - mode='wb', - delete=not debug.s3client, - prefix='metaflow.s3.inputs.') as inputfile: - inputfile.write(b'\n'.join( - [b' '.join([url_quote(prefix)] + ([url_quote(r)] if r else [])) - for prefix, r in prefixes_and_ranges])) + with NamedTemporaryFile( + dir=self._tmpdir, + mode="wb", + delete=not debug.s3client, + prefix="metaflow.s3.inputs.", + ) as inputfile: + inputfile.write( + b"\n".join( + [ + b" ".join([url_quote(prefix)] + ([url_quote(r)] if r else [])) + for prefix, r in prefixes_and_ranges + ] + ) + ) inputfile.flush() - stdout, stderr = self._s3op_with_retries(op, - inputs=inputfile.name, - **options) + stdout, stderr = self._s3op_with_retries( + op, inputs=inputfile.name, **options + ) if stderr: - raise MetaflowS3Exception("Getting S3 files failed.\n"\ - "First prefix requested: %s\n"\ - "Error: %s" % (prefixes_and_ranges[0], stderr)) + raise MetaflowS3Exception( + "Getting S3 files failed.\n" + "First prefix requested: %s\n" + "Error: %s" % (prefixes_and_ranges[0], stderr) + ) else: for line in stdout.splitlines(): - yield tuple(map(url_unquote, line.strip(b'\n').split(b' '))) + yield tuple(map(url_unquote, line.strip(b"\n").split(b" "))) def _put_many_files(self, url_info, overwrite): url_info = list(url_info) - url_dicts = [dict( - chain([ - ('local', os.path.realpath(local)), - ('url', url)], info.items())) for local, url, info in url_info] - - with NamedTemporaryFile(dir=self._tmpdir, - mode='wb', - delete=not debug.s3client, - prefix='metaflow.s3.put_inputs.') as inputfile: + url_dicts = [ + dict( + chain([("local", os.path.realpath(local)), ("url", url)], info.items()) + ) + for local, url, info in url_info + ] + + with NamedTemporaryFile( + dir=self._tmpdir, + mode="wb", + delete=not debug.s3client, + prefix="metaflow.s3.put_inputs.", + ) as inputfile: lines = [to_bytes(json.dumps(x)) for x in url_dicts] - inputfile.write(b'\n'.join(lines)) + inputfile.write(b"\n".join(lines)) inputfile.flush() - stdout, stderr = self._s3op_with_retries('put', - filelist=inputfile.name, - verbose=False, - overwrite=overwrite, - listing=True) + stdout, stderr = self._s3op_with_retries( + "put", + filelist=inputfile.name, + verbose=False, + overwrite=overwrite, + listing=True, + ) if stderr: - raise MetaflowS3Exception("Uploading S3 files failed.\n"\ - "First key: %s\n"\ - "Error: %s" % (url_info[0][2]['key'], - stderr)) + raise MetaflowS3Exception( + "Uploading S3 files failed.\n" + "First key: %s\n" + "Error: %s" % (url_info[0][2]["key"], stderr) + ) else: urls = set() for line in stdout.splitlines(): - url, _, _ = map(url_unquote, line.strip(b'\n').split(b' ')) + url, _, _ = map(url_unquote, line.strip(b"\n").split(b" ")) urls.add(url) - return [(info['key'], url) for _, url, info in url_info if url in urls] + return [(info["key"], url) for _, url, info in url_info if url in urls] def _s3op_with_retries(self, mode, **options): from . import s3op + cmdline = [sys.executable, os.path.abspath(s3op.__file__), mode] for key, value in options.items(): - key = key.replace('_', '-') + key = key.replace("_", "-") if isinstance(value, bool): if value: - cmdline.append('--%s' % key) + cmdline.append("--%s" % key) else: - cmdline.append('--no-%s' % key) + cmdline.append("--no-%s" % key) else: - cmdline.extend(('--%s' % key, value)) + cmdline.extend(("--%s" % key, value)) for i in range(S3_RETRY_COUNT + 1): - with NamedTemporaryFile(dir=self._tmpdir, - mode='wb+', - delete=not debug.s3client, - prefix='metaflow.s3op.stderr') as stderr: + with NamedTemporaryFile( + dir=self._tmpdir, + mode="wb+", + delete=not debug.s3client, + prefix="metaflow.s3op.stderr", + ) as stderr: try: debug.s3client_exec(cmdline) - stdout = subprocess.check_output(cmdline, - cwd=self._tmpdir, - stderr=stderr.file) + stdout = subprocess.check_output( + cmdline, cwd=self._tmpdir, stderr=stderr.file + ) return stdout, None except subprocess.CalledProcessError as ex: stderr.seek(0) - err_out = stderr.read().decode('utf-8', errors='replace') + err_out = stderr.read().decode("utf-8", errors="replace") stderr.seek(0) if ex.returncode == s3op.ERROR_URL_NOT_FOUND: raise MetaflowS3NotFound(err_out) elif ex.returncode == s3op.ERROR_URL_ACCESS_DENIED: raise MetaflowS3AccessDenied(err_out) - time.sleep(2**i + random.randint(0, 10)) + time.sleep(2 ** i + random.randint(0, 10)) - return None, err_out \ No newline at end of file + return None, err_out diff --git a/metaflow/datatools/s3op.py b/metaflow/datatools/s3op.py index d69ddfc15eb..ec7cda21d6d 100644 --- a/metaflow/datatools/s3op.py +++ b/metaflow/datatools/s3op.py @@ -26,8 +26,7 @@ # s3op can be launched as a stand-alone script. We must set # PYTHONPATH for the parent Metaflow explicitly. -sys.path.insert(0,\ - os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))) +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))) # we use Metaflow's parallel_imap_unordered instead of # multiprocessing.Pool because https://bugs.python.org/issue31886 @@ -39,10 +38,20 @@ DOWNLOAD_FILE_THRESHOLD = 2 * TransferConfig().multipart_threshold DOWNLOAD_MAX_CHUNK = 2 * 1024 * 1024 * 1024 - 1 + + class S3Url(object): - def __init__(self, - bucket, path, url, local, prefix, - content_type=None, metadata=None, range=None): + def __init__( + self, + bucket, + path, + url, + local, + prefix, + content_type=None, + metadata=None, + range=None, + ): self.bucket = bucket self.path = path @@ -56,6 +65,7 @@ def __init__(self, def __str__(self): return self.url + # We use error codes instead of Exceptions, which are trickier to # handle reliably in a multi-process world ERROR_INVALID_URL = 4 @@ -66,31 +76,35 @@ def __str__(self): ERROR_VERIFY_FAILED = 9 ERROR_LOCAL_FILE_NOT_FOUND = 10 -def format_triplet(prefix, url='', local=''): - return u' '.join(url_quote(x).decode('utf-8') for x in (prefix, url, local)) + +def format_triplet(prefix, url="", local=""): + return u" ".join(url_quote(x).decode("utf-8") for x in (prefix, url, local)) + # I can't understand what's the right way to deal # with boto errors. This function can be replaced # with better error handling code. def normalize_client_error(err): - error_code = err.response['Error']['Code'] + error_code = err.response["Error"]["Code"] try: return int(error_code) except ValueError: - if error_code == 'AccessDenied': + if error_code == "AccessDenied": return 403 - if error_code == 'NoSuchKey': + if error_code == "NoSuchKey": return 404 return error_code + # S3 worker pool + def worker(result_file_name, queue, mode): # Interpret mode, it can either be a single op or something like # info_download or info_upload which implies: # - for download: we need to return the information as well # - for upload: we need to not overwrite the file if it exists - modes = mode.split('_') + modes = mode.split("_") pre_op_info = False if len(modes) > 1: pre_op_info = True @@ -102,55 +116,54 @@ def op_info(url): try: head = s3.head_object(Bucket=url.bucket, Key=url.path) to_return = { - 'error': None, - 'size': head['ContentLength'], - 'content_type': head['ContentType'], - 'metadata': head['Metadata']} + "error": None, + "size": head["ContentLength"], + "content_type": head["ContentType"], + "metadata": head["Metadata"], + } except client_error as err: error_code = normalize_client_error(err) if error_code == 404: - to_return = {'error': ERROR_URL_NOT_FOUND, 'raise_error': err} + to_return = {"error": ERROR_URL_NOT_FOUND, "raise_error": err} elif error_code == 403: - to_return = {'error': ERROR_URL_ACCESS_DENIED, 'raise_error': err} + to_return = {"error": ERROR_URL_ACCESS_DENIED, "raise_error": err} else: - to_return = {'error': error_code, 'raise_error': err} + to_return = {"error": error_code, "raise_error": err} return to_return - with open(result_file_name, 'w') as result_file: + with open(result_file_name, "w") as result_file: try: from metaflow.datatools.s3util import get_s3_client + s3, client_error = get_s3_client() while True: url, idx = queue.get() if url is None: break - if mode == 'info': + if mode == "info": result = op_info(url) - orig_error = result.get('raise_error', None) + orig_error = result.get("raise_error", None) if orig_error: - del result['raise_error'] - with open(url.local, 'w') as f: + del result["raise_error"] + with open(url.local, "w") as f: json.dump(result, f) - elif mode == 'download': - tmp = NamedTemporaryFile(dir='.', mode='wb', delete=False) + elif mode == "download": + tmp = NamedTemporaryFile(dir=".", mode="wb", delete=False) try: if url.range: resp = s3.get_object( - Bucket=url.bucket, - Key=url.path, - Range=url.range) + Bucket=url.bucket, Key=url.path, Range=url.range + ) else: - resp = s3.get_object( - Bucket=url.bucket, - Key=url.path) - sz = resp['ContentLength'] + resp = s3.get_object(Bucket=url.bucket, Key=url.path) + sz = resp["ContentLength"] if not url.range and sz > DOWNLOAD_FILE_THRESHOLD: # In this case, it is more efficient to use download_file as it # will download multiple parts in parallel (it does it after # multipart_threshold) s3.download_file(url.bucket, url.path, tmp.name) else: - read_in_chunks(tmp, resp['Body'], sz, DOWNLOAD_MAX_CHUNK) + read_in_chunks(tmp, resp["Body"], sz, DOWNLOAD_MAX_CHUNK) tmp.close() os.rename(tmp.name, url.local) except client_error as err: @@ -161,44 +174,46 @@ def op_info(url): result_file.write("%d %d\n" % (idx, -ERROR_URL_NOT_FOUND)) continue elif error_code == 403: - result_file.write("%d %d\n" % (idx, -ERROR_URL_ACCESS_DENIED)) + result_file.write( + "%d %d\n" % (idx, -ERROR_URL_ACCESS_DENIED) + ) continue else: raise # TODO specific error message for out of disk space # If we need the metadata, get it and write it out if pre_op_info: - with open('%s_meta' % url.local, mode='w') as f: - args = {'size': resp['ContentLength']} - if resp['ContentType']: - args['content_type'] = resp['ContentType'] - if resp['Metadata'] is not None: - args['metadata'] = resp['Metadata'] + with open("%s_meta" % url.local, mode="w") as f: + args = {"size": resp["ContentLength"]} + if resp["ContentType"]: + args["content_type"] = resp["ContentType"] + if resp["Metadata"] is not None: + args["metadata"] = resp["Metadata"] json.dump(args, f) # Finally, we push out the size to the result_pipe since # the size is used for verification and other purposes and # we want to avoid file operations for this simple process - result_file.write("%d %d\n" % (idx, resp['ContentLength'])) + result_file.write("%d %d\n" % (idx, resp["ContentLength"])) else: # This is upload, if we have a pre_op, it means we do not # want to overwrite do_upload = False if pre_op_info: result_info = op_info(url) - if result_info['error'] == ERROR_URL_NOT_FOUND: + if result_info["error"] == ERROR_URL_NOT_FOUND: # We only upload if the file is not found do_upload = True else: # No pre-op so we upload do_upload = True if do_upload: - extra=None + extra = None if url.content_type or url.metadata: extra = {} if url.content_type: - extra['ContentType'] = url.content_type + extra["ContentType"] = url.content_type if url.metadata is not None: - extra['Metadata'] = url.metadata + extra["Metadata"] = url.metadata s3.upload_file(url.local, url.bucket, url.path, ExtraArgs=extra) # We indicate that the file was uploaded result_file.write("%d %d\n" % (idx, 0)) @@ -206,6 +221,7 @@ def op_info(url): traceback.print_exc() sys.exit(ERROR_WORKER_EXCEPTION) + def start_workers(mode, urls, num_workers): # We start the minimum of len(urls) or num_workers to avoid starting # workers that will definitely do nothing @@ -222,7 +238,7 @@ def start_workers(mode, urls, num_workers): queue.put((None, None)) # 3. Prepare the result structure - sz_results = [None]*len(urls) + sz_results = [None] * len(urls) # 4. start processes with TempDir() as output_dir: @@ -240,13 +256,12 @@ def start_workers(mode, urls, num_workers): proc.join(timeout=1) if proc.exitcode is not None: if proc.exitcode != 0: - msg = 'Worker process failed (exit code %d)'\ - % proc.exitcode + msg = "Worker process failed (exit code %d)" % proc.exitcode exit(msg, proc.exitcode) # Read the output file if all went well - with open(out_path, 'r') as out_file: + with open(out_path, "r") as out_file: for line in out_file: - line_split = line.split(' ') + line_split = line.split(" ") sz_results[int(line_split[0])] = int(line_split[1]) else: # Put this process back in the processes to check @@ -254,11 +269,11 @@ def start_workers(mode, urls, num_workers): procs = new_procs return sz_results + def process_urls(mode, urls, verbose, num_workers): if verbose: - print('%sing %d files..' % (mode.capitalize(), len(urls)), - file=sys.stderr) + print("%sing %d files.." % (mode.capitalize(), len(urls)), file=sys.stderr) start = time.time() sz_results = start_workers(mode, urls, num_workers) @@ -267,38 +282,45 @@ def process_urls(mode, urls, verbose, num_workers): if verbose: total_size = sum(sz for sz in sz_results if sz is not None and sz > 0) bw = total_size / (end - start) - print('%sed %d files, %s in total, in %d seconds (%s/s).'\ - % (mode.capitalize(), - len(urls), - with_unit(total_size), - end - start, - with_unit(bw)), - file=sys.stderr) + print( + "%sed %d files, %s in total, in %d seconds (%s/s)." + % ( + mode.capitalize(), + len(urls), + with_unit(total_size), + end - start, + with_unit(bw), + ), + file=sys.stderr, + ) return sz_results + # Utility functions + def with_unit(x): - if x > 1024**3: - return '%.1fGB' % (x / 1024.**3) - elif x > 1024**2: - return '%.1fMB' % (x / 1024.**2) + if x > 1024 ** 3: + return "%.1fGB" % (x / 1024.0 ** 3) + elif x > 1024 ** 2: + return "%.1fMB" % (x / 1024.0 ** 2) elif x > 1024: - return '%.1fKB' % (x / 1024.) + return "%.1fKB" % (x / 1024.0) else: - return '%d bytes' % x + return "%d bytes" % x + # S3Ops class is just a wrapper for get_size and list_prefix # required by @aws_retry decorator, which needs the reset_client # method. Otherwise they would be just stand-alone functions. class S3Ops(object): - def __init__(self): self.s3 = None self.client_error = None def reset_client(self, hard_reset=False): from metaflow.datatools.s3util import get_s3_client + if hard_reset or self.s3 is None: self.s3, self.client_error = get_s3_client() @@ -307,15 +329,25 @@ def get_info(self, url): self.reset_client() try: head = self.s3.head_object(Bucket=url.bucket, Key=url.path) - return True, url, [(S3Url( - bucket=url.bucket, - path=url.path, - url=url.url, - local=url.local, - prefix=url.prefix, - content_type=head['ContentType'], - metadata=head['Metadata'], - range=url.range), head['ContentLength'])] + return ( + True, + url, + [ + ( + S3Url( + bucket=url.bucket, + path=url.path, + url=url.url, + local=url.local, + prefix=url.prefix, + content_type=head["ContentType"], + metadata=head["Metadata"], + range=url.range, + ), + head["ContentLength"], + ) + ], + ) except self.client_error as err: error_code = normalize_client_error(err) if error_code == 404: @@ -326,86 +358,94 @@ def get_info(self, url): raise @aws_retry - def list_prefix(self, prefix_url, delimiter=''): + def list_prefix(self, prefix_url, delimiter=""): self.reset_client() - url_base = 's3://%s/' % prefix_url.bucket + url_base = "s3://%s/" % prefix_url.bucket try: - paginator = self.s3.get_paginator('list_objects_v2') + paginator = self.s3.get_paginator("list_objects_v2") urls = [] - for page in paginator.paginate(Bucket=prefix_url.bucket, - Prefix=prefix_url.path, - Delimiter=delimiter): + for page in paginator.paginate( + Bucket=prefix_url.bucket, Prefix=prefix_url.path, Delimiter=delimiter + ): # note that an url may be both a prefix and an object # - the trailing slash is significant in S3 - if 'Contents' in page: - for key in page.get('Contents', []): - url = url_base + key['Key'] - urlobj = S3Url(url=url, - bucket=prefix_url.bucket, - path=key['Key'], - local=generate_local_path(url), - prefix=prefix_url.url) - urls.append((urlobj, key['Size'])) - if 'CommonPrefixes' in page: + if "Contents" in page: + for key in page.get("Contents", []): + url = url_base + key["Key"] + urlobj = S3Url( + url=url, + bucket=prefix_url.bucket, + path=key["Key"], + local=generate_local_path(url), + prefix=prefix_url.url, + ) + urls.append((urlobj, key["Size"])) + if "CommonPrefixes" in page: # we get CommonPrefixes if Delimiter is a non-empty string - for key in page.get('CommonPrefixes', []): - url = url_base + key['Prefix'] - urlobj = S3Url(url=url, - bucket=prefix_url.bucket, - path=key['Prefix'], - local=None, - prefix=prefix_url.url) + for key in page.get("CommonPrefixes", []): + url = url_base + key["Prefix"] + urlobj = S3Url( + url=url, + bucket=prefix_url.bucket, + path=key["Prefix"], + local=None, + prefix=prefix_url.url, + ) urls.append((urlobj, None)) return True, prefix_url, urls except self.s3.exceptions.NoSuchBucket: return False, prefix_url, ERROR_URL_NOT_FOUND except self.client_error as err: - if err.response['Error']['Code'] == 'AccessDenied': + if err.response["Error"]["Code"] == "AccessDenied": return False, prefix_url, ERROR_URL_ACCESS_DENIED else: raise + # We want to reuse an s3 client instance over multiple operations. # This is accomplished by op_ functions below. + def op_get_info(urls): s3 = S3Ops() return [s3.get_info(url) for url in urls] + def op_list_prefix(prefix_urls): s3 = S3Ops() return [s3.list_prefix(prefix) for prefix in prefix_urls] + def op_list_prefix_nonrecursive(prefix_urls): s3 = S3Ops() - return [s3.list_prefix(prefix, delimiter='/') for prefix in prefix_urls] + return [s3.list_prefix(prefix, delimiter="/") for prefix in prefix_urls] + def exit(exit_code, url): if exit_code == ERROR_INVALID_URL: - msg = 'Invalid url: %s' % url.url + msg = "Invalid url: %s" % url.url elif exit_code == ERROR_NOT_FULL_PATH: - msg = 'URL not a full path: %s' % url.url + msg = "URL not a full path: %s" % url.url elif exit_code == ERROR_URL_NOT_FOUND: - msg = 'URL not found: %s' % url.url + msg = "URL not found: %s" % url.url elif exit_code == ERROR_URL_ACCESS_DENIED: - msg = 'Access denied to URL: %s' % url.url + msg = "Access denied to URL: %s" % url.url elif exit_code == ERROR_WORKER_EXCEPTION: - msg = 'Download failed' + msg = "Download failed" elif exit_code == ERROR_VERIFY_FAILED: - msg = 'Verification failed for URL %s, local file %s'\ - % (url.url, url.local) + msg = "Verification failed for URL %s, local file %s" % (url.url, url.local) elif exit_code == ERROR_LOCAL_FILE_NOT_FOUND: - msg = 'Local file not found: %s' % url + msg = "Local file not found: %s" % url else: - msg = 'Unknown error' - print('s3op failed:\n%s' % msg, file=sys.stderr) + msg = "Unknown error" + print("s3op failed:\n%s" % msg, file=sys.stderr) sys.exit(exit_code) + def verify_results(urls, verbose=False): for url, expected in urls: if verbose: - print('verifying %s, expected %s' % (url, expected), - file=sys.stderr) + print("verifying %s, expected %s" % (url, expected), file=sys.stderr) try: got = os.stat(url.local).st_size except OSError: @@ -416,21 +456,23 @@ def verify_results(urls, verbose=False): if url.content_type or url.metadata: # Verify that we also have a metadata file present try: - os.stat('%s_meta' % url.local) + os.stat("%s_meta" % url.local) except OSError: exit(ERROR_VERIFY_FAILED, url) + def generate_local_path(url, suffix=None): # this function generates a safe local file name corresponding to # an S3 URL. URLs may be longer than maximum file length limit on Linux, # so we mostly hash the URL but retain the leaf part as a convenience # feature to ease eyeballing quoted = url_quote(url) - fname = quoted.split(b'/')[-1].replace(b'.', b'_').replace(b'-', b'_') + fname = quoted.split(b"/")[-1].replace(b".", b"_").replace(b"-", b"_") sha = sha1(quoted).hexdigest() if suffix: - return u'-'.join((sha, fname.decode('utf-8'), suffix)) - return u'-'.join((sha, fname.decode('utf-8'))) + return u"-".join((sha, fname.decode("utf-8"), suffix)) + return u"-".join((sha, fname.decode("utf-8"))) + def parallel_op(op, lst, num_workers): # parallel op divides work equally amongst num_workers @@ -460,39 +502,47 @@ def parallel_op(op, lst, num_workers): for x in chain.from_iterable(it): yield x + # CLI + @click.group() def cli(): pass -@cli.command('list', help='List S3 objects') -@click.option('--inputs', - type=click.Path(exists=True), - help='Read input prefixes from the given file.') -@click.option('--num-workers', - default=NUM_WORKERS_DEFAULT, - show_default=True, - help='Number of concurrent connections.') -@click.option('--recursive/--no-recursive', - default=False, - show_default=True, - help='Download prefixes recursively.') -@click.argument('prefixes', nargs=-1) -def lst(prefixes, - inputs=None, - num_workers=None, - recursive=None): + +@cli.command("list", help="List S3 objects") +@click.option( + "--inputs", + type=click.Path(exists=True), + help="Read input prefixes from the given file.", +) +@click.option( + "--num-workers", + default=NUM_WORKERS_DEFAULT, + show_default=True, + help="Number of concurrent connections.", +) +@click.option( + "--recursive/--no-recursive", + default=False, + show_default=True, + help="Download prefixes recursively.", +) +@click.argument("prefixes", nargs=-1) +def lst(prefixes, inputs=None, num_workers=None, recursive=None): urllist = [] for prefix, _ in _populate_prefixes(prefixes, inputs): src = urlparse(prefix) - url = S3Url(url=prefix, - bucket=src.netloc, - path=src.path.lstrip('/'), - local=None, - prefix=prefix) - if src.scheme != 's3': + url = S3Url( + url=prefix, + bucket=src.netloc, + path=src.path.lstrip("/"), + local=None, + prefix=prefix, + ) + if src.scheme != "s3": exit(ERROR_INVALID_URL, url) urllist.append(url) @@ -510,78 +560,94 @@ def lst(prefixes, else: print(format_triplet(url.prefix, url.url, str(size))) -@cli.command(help='Upload files to S3') -@click.option('--file', - 'files', - type=(click.Path(exists=True), str), - multiple=True, - help='Local file->S3Url pair to upload. ' - 'Can be specified multiple times.') -@click.option('--filelist', - type=click.Path(exists=True), - help='Read local file -> S3 URL mappings from the given file.') -@click.option('--num-workers', - default=NUM_WORKERS_DEFAULT, - show_default=True, - help='Number of concurrent connections.') -@click.option('--verbose/--no-verbose', - default=True, - show_default=True, - help='Print status information on stderr.') -@click.option('--overwrite/--no-overwrite', - default=True, - show_default=True, - help='Overwrite key if it already exists in S3.') -@click.option('--listing/--no-listing', - default=False, - show_default=True, - help='Print S3 URLs upload to on stdout.') -def put(files=None, - filelist=None, - num_workers=None, - verbose=None, - overwrite=True, - listing=None): +@cli.command(help="Upload files to S3") +@click.option( + "--file", + "files", + type=(click.Path(exists=True), str), + multiple=True, + help="Local file->S3Url pair to upload. " "Can be specified multiple times.", +) +@click.option( + "--filelist", + type=click.Path(exists=True), + help="Read local file -> S3 URL mappings from the given file.", +) +@click.option( + "--num-workers", + default=NUM_WORKERS_DEFAULT, + show_default=True, + help="Number of concurrent connections.", +) +@click.option( + "--verbose/--no-verbose", + default=True, + show_default=True, + help="Print status information on stderr.", +) +@click.option( + "--overwrite/--no-overwrite", + default=True, + show_default=True, + help="Overwrite key if it already exists in S3.", +) +@click.option( + "--listing/--no-listing", + default=False, + show_default=True, + help="Print S3 URLs upload to on stdout.", +) +def put( + files=None, + filelist=None, + num_workers=None, + verbose=None, + overwrite=True, + listing=None, +): def _files(): for local, url in files: yield url_unquote(local), url_unquote(url), None, None if filelist: - for line in open(filelist, mode='rb'): + for line in open(filelist, mode="rb"): r = json.loads(line) - local = r['local'] - url = r['url'] - content_type = r.get('content_type', None) - metadata = r.get('metadata', None) + local = r["local"] + url = r["url"] + content_type = r.get("content_type", None) + metadata = r.get("metadata", None) if not os.path.exists(local): exit(ERROR_LOCAL_FILE_NOT_FOUND, local) yield local, url, content_type, metadata def _make_url(local, user_url, content_type, metadata): src = urlparse(user_url) - url = S3Url(url=user_url, - bucket=src.netloc, - path=src.path.lstrip('/'), - local=local, - prefix=None, - content_type=content_type, - metadata=metadata) - if src.scheme != 's3': + url = S3Url( + url=user_url, + bucket=src.netloc, + path=src.path.lstrip("/"), + local=local, + prefix=None, + content_type=content_type, + metadata=metadata, + ) + if src.scheme != "s3": exit(ERROR_INVALID_URL, url) if not src.path: exit(ERROR_NOT_FULL_PATH, url) return url urls = list(starmap(_make_url, _files())) - ul_op = 'upload' + ul_op = "upload" if not overwrite: - ul_op = 'info_upload' + ul_op = "info_upload" sz_results = process_urls(ul_op, urls, verbose, num_workers) urls = [url for url, sz in zip(urls, sz_results) if sz is not None] if listing: for url in urls: print(format_triplet(url.url)) + def _populate_prefixes(prefixes, inputs): # Returns a tuple: first element is the prefix and second element # is the optional range (or None if the entire prefix is requested) @@ -590,82 +656,103 @@ def _populate_prefixes(prefixes, inputs): else: prefixes = [] if inputs: - with open(inputs, mode='rb') as f: + with open(inputs, mode="rb") as f: for l in f: - s = l.split(b' ') + s = l.split(b" ") if len(s) > 1: prefixes.append( - (url_unquote(s[0].strip()), url_unquote(s[1].strip()))) + (url_unquote(s[0].strip()), url_unquote(s[1].strip())) + ) else: prefixes.append((url_unquote(s[0].strip()), None)) return prefixes -@cli.command(help='Download files from S3') -@click.option('--recursive/--no-recursive', - default=False, - show_default=True, - help='Download prefixes recursively.') -@click.option('--num-workers', - default=NUM_WORKERS_DEFAULT, - show_default=True, - help='Number of concurrent connections.') -@click.option('--inputs', - type=click.Path(exists=True), - help='Read input prefixes from the given file.') -@click.option('--verify/--no-verify', - default=True, - show_default=True, - help='Verify that files were loaded correctly.') -@click.option('--info/--no-info', - default=True, - show_default=True, - help='Return user tags and content-type') -@click.option('--allow-missing/--no-allow-missing', - default=False, - show_default=True, - help='Do not exit if missing files are detected. '\ - 'Implies --verify.') -@click.option('--verbose/--no-verbose', - default=True, - show_default=True, - help='Print status information on stderr.') -@click.option('--listing/--no-listing', - default=False, - show_default=True, - help='Print S3 URL -> local file mapping on stdout.') -@click.argument('prefixes', nargs=-1) -def get(prefixes, - recursive=None, - num_workers=None, - inputs=None, - verify=None, - info=None, - allow_missing=None, - verbose=None, - listing=None): + +@cli.command(help="Download files from S3") +@click.option( + "--recursive/--no-recursive", + default=False, + show_default=True, + help="Download prefixes recursively.", +) +@click.option( + "--num-workers", + default=NUM_WORKERS_DEFAULT, + show_default=True, + help="Number of concurrent connections.", +) +@click.option( + "--inputs", + type=click.Path(exists=True), + help="Read input prefixes from the given file.", +) +@click.option( + "--verify/--no-verify", + default=True, + show_default=True, + help="Verify that files were loaded correctly.", +) +@click.option( + "--info/--no-info", + default=True, + show_default=True, + help="Return user tags and content-type", +) +@click.option( + "--allow-missing/--no-allow-missing", + default=False, + show_default=True, + help="Do not exit if missing files are detected. " "Implies --verify.", +) +@click.option( + "--verbose/--no-verbose", + default=True, + show_default=True, + help="Print status information on stderr.", +) +@click.option( + "--listing/--no-listing", + default=False, + show_default=True, + help="Print S3 URL -> local file mapping on stdout.", +) +@click.argument("prefixes", nargs=-1) +def get( + prefixes, + recursive=None, + num_workers=None, + inputs=None, + verify=None, + info=None, + allow_missing=None, + verbose=None, + listing=None, +): # Construct a list of URL (prefix) objects urllist = [] for prefix, r in _populate_prefixes(prefixes, inputs): src = urlparse(prefix) - url = S3Url(url=prefix, - bucket=src.netloc, - path=src.path.lstrip('/'), - local=generate_local_path(prefix), - prefix=prefix, - range=r) - if src.scheme != 's3': + url = S3Url( + url=prefix, + bucket=src.netloc, + path=src.path.lstrip("/"), + local=generate_local_path(prefix), + prefix=prefix, + range=r, + ) + if src.scheme != "s3": exit(ERROR_INVALID_URL, url) if not recursive and not src.path: exit(ERROR_NOT_FULL_PATH, url) urllist.append(url) # Construct a url->size mapping and get content-type and metadata if needed op = None - dl_op = 'download' + dl_op = "download" if recursive: op = op_list_prefix if verify or verbose or info: - dl_op = 'info_download' + dl_op = "info_download" if op: urls = [] # NOTE - we must retain the order of prefixes requested @@ -703,8 +790,14 @@ def get(prefixes, # Postprocess if verify: # Verify only results with an actual size (so actual files) - verify_results([(url, sz) for url, sz in zip(to_load, sz_results) - if sz != -ERROR_URL_NOT_FOUND], verbose=verbose) + verify_results( + [ + (url, sz) + for url, sz in zip(to_load, sz_results) + if sz != -ERROR_URL_NOT_FOUND + ], + verbose=verbose, + ) idx_in_sz = 0 if listing: @@ -720,48 +813,56 @@ def get(prefixes, else: print(format_triplet(url.prefix, url.url, url.local)) -@cli.command(help='Get info about files from S3') -@click.option('--num-workers', - default=NUM_WORKERS_DEFAULT, - show_default=True, - help='Number of concurrent connections.') -@click.option('--inputs', - type=click.Path(exists=True), - help='Read input prefixes from the given file.') -@click.option('--verbose/--no-verbose', - default=True, - show_default=True, - help='Print status information on stderr.') -@click.option('--listing/--no-listing', - default=False, - show_default=True, - help='Print S3 URL -> local file mapping on stdout.') -@click.argument('prefixes', nargs=-1) -def info(prefixes, - num_workers=None, - inputs=None, - verbose=None, - listing=None): + +@cli.command(help="Get info about files from S3") +@click.option( + "--num-workers", + default=NUM_WORKERS_DEFAULT, + show_default=True, + help="Number of concurrent connections.", +) +@click.option( + "--inputs", + type=click.Path(exists=True), + help="Read input prefixes from the given file.", +) +@click.option( + "--verbose/--no-verbose", + default=True, + show_default=True, + help="Print status information on stderr.", +) +@click.option( + "--listing/--no-listing", + default=False, + show_default=True, + help="Print S3 URL -> local file mapping on stdout.", +) +@click.argument("prefixes", nargs=-1) +def info(prefixes, num_workers=None, inputs=None, verbose=None, listing=None): # Construct a list of URL (prefix) objects urllist = [] for prefix, _ in _populate_prefixes(prefixes, inputs): src = urlparse(prefix) - url = S3Url(url=prefix, - bucket=src.netloc, - path=src.path.lstrip('/'), - local=generate_local_path(prefix, suffix='info'), - prefix=prefix, - range=None) - if src.scheme != 's3': + url = S3Url( + url=prefix, + bucket=src.netloc, + path=src.path.lstrip("/"), + local=generate_local_path(prefix, suffix="info"), + prefix=prefix, + range=None, + ) + if src.scheme != "s3": exit(ERROR_INVALID_URL, url) urllist.append(url) - process_urls('info', urllist, verbose, num_workers) + process_urls("info", urllist, verbose, num_workers) if listing: for url in urllist: print(format_triplet(url.prefix, url.url, url.local)) -if __name__ == '__main__': - cli(auto_envvar_prefix='S3OP') \ No newline at end of file + +if __name__ == "__main__": + cli(auto_envvar_prefix="S3OP") diff --git a/metaflow/datatools/s3tail.py b/metaflow/datatools/s3tail.py index f336e8ba08f..aec7cd6db6d 100644 --- a/metaflow/datatools/s3tail.py +++ b/metaflow/datatools/s3tail.py @@ -1,4 +1,3 @@ - from io import BytesIO from .s3util import aws_retry, get_s3_client @@ -9,14 +8,15 @@ # python3 from urllib.parse import urlparse + class S3Tail(object): def __init__(self, s3url): url = urlparse(s3url) self.s3, self.ClientError = get_s3_client() self._bucket = url.netloc - self._key = url.path.lstrip('/') + self._key = url.path.lstrip("/") self._pos = 0 - self._tail = b'' + self._tail = b"" def clone(self, s3url): tail = S3Tail(s3url) @@ -36,7 +36,7 @@ def __iter__(self): buf = self._fill_buf() if buf is not None: for line in buf: - if line.endswith(b'\n'): + if line.endswith(b"\n"): yield line else: self._tail = line @@ -45,14 +45,14 @@ def __iter__(self): @aws_retry def _make_range_request(self): try: - return self.s3.get_object(Bucket=self._bucket, - Key=self._key, - Range='bytes=%d-' % self._pos) + return self.s3.get_object( + Bucket=self._bucket, Key=self._key, Range="bytes=%d-" % self._pos + ) except self.ClientError as err: - code = err.response['Error']['Code'] + code = err.response["Error"]["Code"] # NOTE we deliberately regard NoSuchKey as an ignorable error. # We assume that the file just hasn't appeared in S3 yet. - if code in ('InvalidRange', 'NoSuchKey'): + if code in ("InvalidRange", "NoSuchKey"): return None else: raise @@ -61,18 +61,19 @@ def _fill_buf(self): resp = self._make_range_request() if resp is None: return None - code = str(resp['ResponseMetadata']['HTTPStatusCode']) - if code[0] == '2': - data = resp['Body'].read() + code = str(resp["ResponseMetadata"]["HTTPStatusCode"]) + if code[0] == "2": + data = resp["Body"].read() if data: buf = BytesIO(self._tail + data) self._pos += len(data) - self._tail = b'' + self._tail = b"" return buf else: return None - elif code[0] == '5': + elif code[0] == "5": return None else: - raise Exception('Retrieving %s/%s failed: %s' % (self._bucket, self._key, code)) - + raise Exception( + "Retrieving %s/%s failed: %s" % (self._bucket, self._key, code) + ) diff --git a/metaflow/datatools/s3util.py b/metaflow/datatools/s3util.py index 0574486ee13..f10f076b189 100644 --- a/metaflow/datatools/s3util.py +++ b/metaflow/datatools/s3util.py @@ -5,17 +5,25 @@ import os from metaflow.exception import MetaflowException -from metaflow.metaflow_config import S3_ENDPOINT_URL, S3_VERIFY_CERTIFICATE, S3_RETRY_COUNT +from metaflow.metaflow_config import ( + S3_ENDPOINT_URL, + S3_VERIFY_CERTIFICATE, + S3_RETRY_COUNT, +) -TEST_S3_RETRY = 'TEST_S3_RETRY' in os.environ +TEST_S3_RETRY = "TEST_S3_RETRY" in os.environ + def get_s3_client(): from metaflow.plugins.aws.aws_client import get_aws_client + return get_aws_client( - 's3', + "s3", with_error=True, - params={'endpoint_url': S3_ENDPOINT_URL, 'verify': S3_VERIFY_CERTIFICATE }) + params={"endpoint_url": S3_ENDPOINT_URL, "verify": S3_VERIFY_CERTIFICATE}, + ) + # decorator to retry functions that access S3 def aws_retry(f): @@ -25,9 +33,11 @@ def retry_wrapper(self, *args, **kwargs): try: ret = f(self, *args, **kwargs) if TEST_S3_RETRY and i == 0: - raise Exception("TEST_S3_RETRY env var set. " - "Pretending that an S3 op failed. " - "This is not a real failure.") + raise Exception( + "TEST_S3_RETRY env var set. " + "Pretending that an S3 op failed. " + "This is not a real failure." + ) else: return ret except MetaflowException as ex: @@ -38,17 +48,21 @@ def retry_wrapper(self, *args, **kwargs): function_name = f.func_name except AttributeError: function_name = f.__name__ - sys.stderr.write("S3 datastore operation %s failed (%s). " - "Retrying %d more times..\n" - % (function_name, ex, S3_RETRY_COUNT - i)) + sys.stderr.write( + "S3 datastore operation %s failed (%s). " + "Retrying %d more times..\n" + % (function_name, ex, S3_RETRY_COUNT - i) + ) self.reset_client(hard_reset=True) last_exc = ex # exponential backoff for real failures if not (TEST_S3_RETRY and i == 0): - time.sleep(2**i + random.randint(0, 5)) + time.sleep(2 ** i + random.randint(0, 5)) raise last_exc + return retry_wrapper + # Read an AWS source in a chunked manner. # We read in chunks (at most 2GB -- here this is passed via max_chunk_size) # because of https://bugs.python.org/issue42853 (Py3 bug); this also helps @@ -63,4 +77,4 @@ def read_in_chunks(dst, src, src_sz, max_chunk_size): # Py2 doesn't return the number of bytes written so calculate size # separately dst.write(buf) - remaining -= len(buf) \ No newline at end of file + remaining -= len(buf) diff --git a/metaflow/debug.py b/metaflow/debug.py index 1a213fa6cc9..a3745f5d079 100644 --- a/metaflow/debug.py +++ b/metaflow/debug.py @@ -17,17 +17,19 @@ # variable also disables automatic cleaning of subdirectories, which can # fill up disk space quickly + class Debug(object): def __init__(self): import metaflow.metaflow_config as config + for typ in config.DEBUG_OPTIONS: - if getattr(config, 'METAFLOW_DEBUG_%s' % typ.upper()): + if getattr(config, "METAFLOW_DEBUG_%s" % typ.upper()): op = partial(self.log, typ) else: op = self.noop # use debug.$type_exec(args) to log command line for subprocesses # of type $type - setattr(self, '%s_exec' % typ, op) + setattr(self, "%s_exec" % typ, op) # use the debug.$type flag to check if logging is enabled for $type setattr(self, typ, op != self.noop) @@ -35,8 +37,8 @@ def log(self, typ, args): if is_stringish(args): s = args else: - s = ' '.join(args) - print('debug[%s]: %s' % (typ, s), file=sys.stderr) + s = " ".join(args) + print("debug[%s]: %s" % (typ, s), file=sys.stderr) def noop(self, args): pass diff --git a/metaflow/decorators.py b/metaflow/decorators.py index 19ccaf2e97c..241124db3b8 100644 --- a/metaflow/decorators.py +++ b/metaflow/decorators.py @@ -3,21 +3,27 @@ import re from .flowspec import FlowSpec -from .exception import MetaflowInternalError, \ - MetaflowException, InvalidDecoratorAttribute +from .exception import ( + MetaflowInternalError, + MetaflowException, + InvalidDecoratorAttribute, +) import click + class BadStepDecoratorException(MetaflowException): headline = "Syntax error" def __init__(self, deco, func): - msg =\ - "You tried to apply decorator '{deco}' on '{func}' which is "\ - "not declared as a @step. Make sure you apply this decorator "\ - "on a function which has @step on the line just before the "\ - "function name and @{deco} is above @step.".format(deco=deco, - func=func.__name__) + msg = ( + "You tried to apply decorator '{deco}' on '{func}' which is " + "not declared as a @step. Make sure you apply this decorator " + "on a function which has @step on the line just before the " + "function name and @{deco} is above @step.".format( + deco=deco, func=func.__name__ + ) + ) super(BadStepDecoratorException, self).__init__(msg) @@ -25,9 +31,10 @@ class BadFlowDecoratorException(MetaflowException): headline = "Syntax error" def __init__(self, deconame): - msg =\ - "Decorator '%s' can be applied only to FlowSpecs. Make sure "\ + msg = ( + "Decorator '%s' can be applied only to FlowSpecs. Make sure " "the decorator is above a class definition." % deconame + ) super(BadFlowDecoratorException, self).__init__(msg) @@ -36,10 +43,14 @@ class UnknownStepDecoratorException(MetaflowException): def __init__(self, deconame): from .plugins import STEP_DECORATORS - decos = ', '.join(t.name for t in STEP_DECORATORS - if not t.name.endswith('_internal')) - msg = "Unknown step decorator *{deconame}*. The following decorators are "\ - "supported: *{decos}*".format(deconame=deconame, decos=decos) + + decos = ", ".join( + t.name for t in STEP_DECORATORS if not t.name.endswith("_internal") + ) + msg = ( + "Unknown step decorator *{deconame}*. The following decorators are " + "supported: *{decos}*".format(deconame=deconame, decos=decos) + ) super(UnknownStepDecoratorException, self).__init__(msg) @@ -47,9 +58,12 @@ class DuplicateStepDecoratorException(MetaflowException): headline = "Duplicate decorators" def __init__(self, deco, func): - msg = "Step '{step}' already has a decorator '{deco}'. "\ - "You can specify each decorator only once."\ - .format(step=func.__name__, deco=deco) + msg = ( + "Step '{step}' already has a decorator '{deco}'. " + "You can specify each decorator only once.".format( + step=func.__name__, deco=deco + ) + ) super(DuplicateStepDecoratorException, self).__init__(msg) @@ -58,9 +72,12 @@ class UnknownFlowDecoratorException(MetaflowException): def __init__(self, deconame): from .plugins import FLOW_DECORATORS - decos = ', '.join(t.name for t in FLOW_DECORATORS) - msg = "Unknown flow decorator *{deconame}*. The following decorators are "\ - "supported: *{decos}*".format(deconame=deconame, decos=decos) + + decos = ", ".join(t.name for t in FLOW_DECORATORS) + msg = ( + "Unknown flow decorator *{deconame}*. The following decorators are " + "supported: *{decos}*".format(deconame=deconame, decos=decos) + ) super(UnknownFlowDecoratorException, self).__init__(msg) @@ -68,9 +85,10 @@ class DuplicateFlowDecoratorException(MetaflowException): headline = "Duplicate decorators" def __init__(self, deco): - msg = "Flow already has a decorator '{deco}'. "\ - "You can specify each decorator only once."\ - .format(deco=deco) + msg = ( + "Flow already has a decorator '{deco}'. " + "You can specify each decorator only once.".format(deco=deco) + ) super(DuplicateFlowDecoratorException, self).__init__(msg) @@ -79,12 +97,10 @@ class Decorator(object): Base class for all decorators. """ - name = 'NONAME' + name = "NONAME" defaults = {} - def __init__(self, - attributes=None, - statically_defined=False): + def __init__(self, attributes=None, statically_defined=False): self.attributes = self.defaults.copy() self.statically_defined = statically_defined @@ -93,34 +109,35 @@ def __init__(self, if k in self.defaults: self.attributes[k] = v else: - raise InvalidDecoratorAttribute( - self.name, k, self.defaults) + raise InvalidDecoratorAttribute(self.name, k, self.defaults) @classmethod def _parse_decorator_spec(cls, deco_spec): - top = deco_spec.split(':', 1) + top = deco_spec.split(":", 1) if len(top) == 1: return cls() else: name, attrspec = top - attrs = dict(map(lambda x: x.strip(), a.split('=')) - for a in re.split(''',(?=[\s\w]+=)''', attrspec.strip('"\''))) + attrs = dict( + map(lambda x: x.strip(), a.split("=")) + for a in re.split(""",(?=[\s\w]+=)""", attrspec.strip("\"'")) + ) return cls(attributes=attrs) def make_decorator_spec(self): attrs = {k: v for k, v in self.attributes.items() if v is not None} if attrs: - attrstr = ','.join('%s=%s' % x for x in attrs.items()) - return '%s:%s' % (self.name, attrstr) + attrstr = ",".join("%s=%s" % x for x in attrs.items()) + return "%s:%s" % (self.name, attrstr) else: return self.name def __str__(self): - mode = 'decorated' if self.statically_defined else 'cli' - attrs = ' '.join('%s=%s' % x for x in self.attributes.items()) + mode = "decorated" if self.statically_defined else "cli" + attrs = " ".join("%s=%s" % x for x in self.attributes.items()) if attrs: - attrs = ' ' + attrs - fmt = '%s<%s%s>' % (self.name, mode, attrs) + attrs = " " + attrs + fmt = "%s<%s%s>" % (self.name, mode, attrs) return fmt @@ -135,15 +152,9 @@ def __init__(self, *args, **kwargs): self._flow_decorators.append(self) super(FlowDecorator, self).__init__(*args, **kwargs) - def flow_init(self, - flow, - graph, - environment, - flow_datastore, - metadata, - logger, - echo, - options): + def flow_init( + self, flow, graph, environment, flow_datastore, metadata, logger, echo, options + ): """ Called when all decorators have been created for this flow. """ @@ -167,20 +178,23 @@ def add_decorator_options(cmd): for deco in flow_decorators(): for option, kwargs in deco.options.items(): if option in seen: - msg = "Flow decorator '%s' uses an option '%s' which is also "\ - "used by the decorator '%s'. This is a bug in Metaflow. "\ - "Please file a ticket on GitHub."\ - % (deco.name, option, seen[option]) + msg = ( + "Flow decorator '%s' uses an option '%s' which is also " + "used by the decorator '%s'. This is a bug in Metaflow. " + "Please file a ticket on GitHub." + % (deco.name, option, seen[option]) + ) raise MetaflowInternalError(msg) else: seen[option] = deco.name - cmd.params.insert(0, click.Option(('--' + option,), **kwargs)) + cmd.params.insert(0, click.Option(("--" + option,), **kwargs)) return cmd def flow_decorators(): return FlowDecorator._flow_decorators + class StepDecorator(Decorator): """ Base class for all step decorators. @@ -207,11 +221,13 @@ class MyDecorator(StepDecorator): state easily. TODO (savin): Initialize the decorators with flow, graph, - step.__name__ etc., so that we don't have to - pass them around with every lifecycle call. + step.__name__ etc., so that we don't have to + pass them around with every lifecycle call. """ - def step_init(self, flow, graph, step_name, decorators, environment, flow_datastore, logger): + def step_init( + self, flow, graph, step_name, decorators, environment, flow_datastore, logger + ): """ Called when all decorators have been created for this step """ @@ -230,8 +246,8 @@ def step_task_retry_count(self): are attempts to run the process after the user code has failed all its retries. - Typically, the runtime takes the maximum of retry counts across - decorators and user specification to determine the task retry count. + Typically, the runtime takes the maximum of retry counts across + decorators and user specification to determine the task retry count. If you want to force no retries, return the special values (None, None). """ return 0, 0 @@ -243,13 +259,9 @@ def runtime_init(self, flow, graph, package, run_id): """ pass - def runtime_task_created(self, - task_datastore, - task_id, - split_index, - input_paths, - is_cloned, - ubf_context): + def runtime_task_created( + self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context + ): """ Called when the runtime has created a task related to this step. """ @@ -261,61 +273,50 @@ def runtime_finished(self, exception): """ pass - def runtime_step_cli(self, - cli_args, - retry_count, - max_user_code_retries, - ubf_context): + def runtime_step_cli( + self, cli_args, retry_count, max_user_code_retries, ubf_context + ): """ Access the command line for a step execution in the runtime context. """ pass - def task_pre_step(self, - step_name, - task_datastore, - metadata, - run_id, - task_id, - flow, - graph, - retry_count, - max_user_code_retries, - ubf_context, - inputs): + def task_pre_step( + self, + step_name, + task_datastore, + metadata, + run_id, + task_id, + flow, + graph, + retry_count, + max_user_code_retries, + ubf_context, + inputs, + ): """ Run before the step function in the task context. """ pass - def task_decorate(self, - step_func, - flow, - graph, - retry_count, - max_user_code_retries, - ubf_context): + def task_decorate( + self, step_func, flow, graph, retry_count, max_user_code_retries, ubf_context + ): return step_func - def task_post_step(self, - step_name, - flow, - graph, - retry_count, - max_user_code_retries): + def task_post_step( + self, step_name, flow, graph, retry_count, max_user_code_retries + ): """ Run after the step function has finished successfully in the task context. """ pass - def task_exception(self, - exception, - step_name, - flow, - graph, - retry_count, - max_user_code_retries): + def task_exception( + self, exception, step_name, flow, graph, retry_count, max_user_code_retries + ): """ Run if the step function raised an exception in the task context. @@ -324,13 +325,9 @@ def task_exception(self, """ pass - def task_finished(self, - step_name, - flow, - graph, - is_task_ok, - retry_count, - max_user_code_retries): + def task_finished( + self, step_name, flow, graph, is_task_ok, retry_count, max_user_code_retries + ): """ Run after the task context has been finalized. @@ -362,8 +359,9 @@ def _base_flow_decorator(decofunc, *args, **kwargs): if decofunc.name in cls._flow_decorators: raise DuplicateFlowDecoratorException(decofunc.name) else: - cls._flow_decorators[decofunc.name] = decofunc(attributes=kwargs, - statically_defined=True) + cls._flow_decorators[decofunc.name] = decofunc( + attributes=kwargs, statically_defined=True + ) else: raise BadFlowDecoratorException(decofunc.name) return cls @@ -373,6 +371,7 @@ def _base_flow_decorator(decofunc, *args, **kwargs): # function to be decorated as the first argument. def wrap(f): return _base_flow_decorator(decofunc, f, **kwargs) + return wrap @@ -385,15 +384,14 @@ def _base_step_decorator(decotype, *args, **kwargs): # No keyword arguments specified for the decorator, e.g. @foobar. # The first argument is the function to be decorated. func = args[0] - if not hasattr(func, 'is_step'): + if not hasattr(func, "is_step"): raise BadStepDecoratorException(decotype.name, func) # Only the first decorator applies if decotype.name in [deco.name for deco in func.decorators]: raise DuplicateStepDecoratorException(decotype.name, func) else: - func.decorators.append(decotype(attributes=kwargs, - statically_defined=True)) + func.decorators.append(decotype(attributes=kwargs, statically_defined=True)) return func else: @@ -402,6 +400,7 @@ def _base_step_decorator(decotype, *args, **kwargs): # function to be decorated as the first argument. def wrap(f): return _base_step_decorator(decotype, f, **kwargs) + return wrap @@ -420,6 +419,7 @@ def _attach_decorators(flow, decospecs): for step in flow: _attach_decorators_to_step(step, decospecs) + def _attach_decorators_to_step(step, decospecs): """ Attach decorators to a step during runtime. This has the same @@ -427,9 +427,10 @@ def _attach_decorators_to_step(step, decospecs): the step. """ from .plugins import STEP_DECORATORS + decos = {decotype.name: decotype for decotype in STEP_DECORATORS} for decospec in decospecs: - deconame = decospec.strip("'").split(':')[0] + deconame = decospec.strip("'").split(":")[0] if deconame not in decos: raise UnknownStepDecoratorException(deconame) # Attach the decorator to step if it doesn't have the decorator @@ -439,25 +440,29 @@ def _attach_decorators_to_step(step, decospecs): deco = decos[deconame]._parse_decorator_spec(decospec) step.decorators.append(deco) -def _init_flow_decorators(flow, - graph, - environment, - flow_datastore, - metadata, - logger, - echo, - deco_options): + +def _init_flow_decorators( + flow, graph, environment, flow_datastore, metadata, logger, echo, deco_options +): for deco in flow._flow_decorators.values(): opts = {option: deco_options[option] for option in deco.options} - deco.flow_init(flow, graph, environment, - flow_datastore, metadata, logger, echo, opts) + deco.flow_init( + flow, graph, environment, flow_datastore, metadata, logger, echo, opts + ) def _init_step_decorators(flow, graph, environment, flow_datastore, logger): for step in flow: for deco in step.decorators: - deco.step_init(flow, graph, step.__name__, - step.decorators, environment, flow_datastore, logger) + deco.step_init( + flow, + graph, + step.__name__, + step.decorators, + environment, + flow_datastore, + logger, + ) def step(f): @@ -477,10 +482,11 @@ def step(f): def _import_plugin_decorators(globals_dict): """ - Auto-generate a decorator function for every decorator + Auto-generate a decorator function for every decorator defined in plugins.STEP_DECORATORS and plugins.FLOW_DECORATORS. """ from .plugins import STEP_DECORATORS, FLOW_DECORATORS + # Q: Why not use StepDecorators directly as decorators? # A: Getting an object behave as a decorator that can work # both with and without arguments is surprisingly hard. diff --git a/metaflow/event_logger.py b/metaflow/event_logger.py index 2e9d631cdf4..7e559c443fd 100644 --- a/metaflow/event_logger.py +++ b/metaflow/event_logger.py @@ -1,8 +1,8 @@ from .sidecar import SidecarSubProcess from .sidecar_messages import Message, MessageTypes -class NullEventLogger(object): +class NullEventLogger(object): def __init__(self, *args, **kwargs): pass @@ -15,8 +15,8 @@ def log(self, payload): def terminate(self): pass -class EventLogger(NullEventLogger): +class EventLogger(NullEventLogger): def __init__(self, logger_type): # type: (str) -> None self.sidecar_process = None diff --git a/metaflow/exception.py b/metaflow/exception.py index 24cfcd6378d..b283f4abd68 100644 --- a/metaflow/exception.py +++ b/metaflow/exception.py @@ -8,12 +8,12 @@ # worker processes that exit with this code should be retried (if retry counts left) METAFLOW_EXIT_ALLOW_RETRY = 203 + class MetaflowExceptionWrapper(Exception): def __init__(self, exc=None): if exc is not None: self.exception = str(exc) - self.type = '%s.%s' % (exc.__class__.__module__, - exc.__class__.__name__) + self.type = "%s.%s" % (exc.__class__.__module__, exc.__class__.__name__) if sys.exc_info()[0] is None: self.stacktrace = None else: @@ -30,7 +30,7 @@ def __getstate__(self): def __setstate__(self, state): self.__dict__ = state - + def __repr__(self): return str(self) @@ -38,77 +38,96 @@ def __str__(self): if self.stacktrace: return self.stacktrace else: - return '[no stacktrace]\n%s: %s' % (self.type, self.exception) + return "[no stacktrace]\n%s: %s" % (self.type, self.exception) + class MetaflowException(Exception): - headline = 'Flow failed' - def __init__(self, msg='', lineno=None): + headline = "Flow failed" + + def __init__(self, msg="", lineno=None): self.message = msg self.line_no = lineno super(MetaflowException, self).__init__() def __str__(self): - prefix = 'line %d: ' % self.line_no if self.line_no else '' - return '%s%s' % (prefix, self.message) + prefix = "line %d: " % self.line_no if self.line_no else "" + return "%s%s" % (prefix, self.message) + class ParameterFieldFailed(MetaflowException): headline = "Parameter field failed" def __init__(self, name, field): exc = traceback.format_exc() - msg = "When evaluating the field *%s* for the Parameter *%s*, "\ - "the following exception occurred:\n\n%s" % (field, name, exc) + msg = ( + "When evaluating the field *%s* for the Parameter *%s*, " + "the following exception occurred:\n\n%s" % (field, name, exc) + ) super(ParameterFieldFailed, self).__init__(msg) + class ParameterFieldTypeMismatch(MetaflowException): headline = "Parameter field with a mismatching type" def __init__(self, msg): super(ParameterFieldTypeMismatch, self).__init__(msg) + class ExternalCommandFailed(MetaflowException): headline = "External command failed" def __init__(self, msg): super(ExternalCommandFailed, self).__init__(msg) + class MetaflowNotFound(MetaflowException): - headline = 'Object not found' + headline = "Object not found" + class MetaflowNamespaceMismatch(MetaflowException): - headline = 'Object not in the current namespace' + headline = "Object not in the current namespace" def __init__(self, namespace): msg = "Object not in namespace '%s'" % namespace super(MetaflowNamespaceMismatch, self).__init__(msg) + class MetaflowInternalError(MetaflowException): - headline = 'Internal error' + headline = "Internal error" + class MetaflowUnknownUser(MetaflowException): - headline = 'Unknown user' + headline = "Unknown user" def __init__(self): - msg = "Metaflow could not determine your user name based on "\ - "environment variables ($USERNAME etc.)" + msg = ( + "Metaflow could not determine your user name based on " + "environment variables ($USERNAME etc.)" + ) super(MetaflowUnknownUser, self).__init__(msg) + class InvalidDecoratorAttribute(MetaflowException): headline = "Unknown decorator attribute" + def __init__(self, deconame, attr, defaults): - msg = "Decorator '{deco}' does not support the attribute '{attr}'. "\ - "These attributes are supported: {defaults}."\ - .format(deco=deconame, - attr=attr, - defaults=', '.join(defaults)) + msg = ( + "Decorator '{deco}' does not support the attribute '{attr}'. " + "These attributes are supported: {defaults}.".format( + deco=deconame, attr=attr, defaults=", ".join(defaults) + ) + ) super(InvalidDecoratorAttribute, self).__init__(msg) + class CommandException(MetaflowException): headline = "Invalid command" + class MetaflowDataMissing(MetaflowException): headline = "Data missing" + class UnhandledInMergeArtifactsException(MetaflowException): headline = "Unhandled artifacts in merge" @@ -116,6 +135,7 @@ def __init__(self, msg, unhandled): super(UnhandledInMergeArtifactsException, self).__init__(msg) self.artifact_names = unhandled + class MissingInMergeArtifactsException(MetaflowException): headline = "Missing artifacts in merge" @@ -123,6 +143,7 @@ def __init__(self, msg, unhandled): super(MissingInMergeArtifactsException, self).__init__(msg) self.artifact_names = unhandled + # Import any exceptions defined by a Metaflow extensions package try: import metaflow_extensions.exceptions as extension_module @@ -132,23 +153,26 @@ def __init__(self, msg, unhandled): # e.name is set to the name of the package that fails to load # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) - if not (isinstance(e, ModuleNotFoundError) and \ - e.name in ['metaflow_extensions', 'metaflow_extensions.exceptions']): + if not ( + isinstance(e, ModuleNotFoundError) + and e.name in ["metaflow_extensions", "metaflow_extensions.exceptions"] + ): print( "Cannot load metaflow_extensions exceptions -- " - "if you want to ignore, uninstall metaflow_extensions package") + "if you want to ignore, uninstall metaflow_extensions package" + ) raise else: # We load into globals whatever we have in extension_module # We specifically exclude any modules that may be included (like sys, os, etc) for n, o in extension_module.__dict__.items(): - if not n.startswith('__') and not isinstance(o, types.ModuleType): + if not n.startswith("__") and not isinstance(o, types.ModuleType): globals()[n] = o finally: # Erase all temporary names to avoid leaking things - for _n in ['ver', 'n', 'o', 'e', 'extension_module']: + for _n in ["ver", "n", "o", "e", "extension_module"]: try: del globals()[_n] except KeyError: pass - del globals()['_n'] + del globals()["_n"] diff --git a/metaflow/flowspec.py b/metaflow/flowspec.py index fb21a1cb0aa..775cfee424a 100644 --- a/metaflow/flowspec.py +++ b/metaflow/flowspec.py @@ -6,8 +6,12 @@ from . import cmd_with_io from .parameters import Parameter -from .exception import MetaflowException, MetaflowInternalError, \ - MissingInMergeArtifactsException, UnhandledInMergeArtifactsException +from .exception import ( + MetaflowException, + MetaflowInternalError, + MissingInMergeArtifactsException, + UnhandledInMergeArtifactsException, +) from .graph import FlowGraph from .unbounded_foreach import UnboundedForeachInput @@ -42,20 +46,22 @@ class FlowSpec(object): # Attributes that are not saved in the datastore when checkpointing. # Name starting with '__', methods, functions and Parameters do not need # to be listed. - _EPHEMERAL = {'_EPHEMERAL', - '_NON_PARAMETERS', - '_datastore', - '_cached_input', - '_graph', - '_flow_decorators', - '_steps', - 'index', - 'input'} + _EPHEMERAL = { + "_EPHEMERAL", + "_NON_PARAMETERS", + "_datastore", + "_cached_input", + "_graph", + "_flow_decorators", + "_steps", + "index", + "input", + } # When checking for parameters, we look at dir(self) but we want to exclude # attributes that are definitely not parameters and may be expensive to # compute (like anything related to the `foreach_stack`). We don't need to exclude # names starting with `_` as those are already excluded from `_get_parameters`. - _NON_PARAMETERS = {'cmd', 'foreach_stack', 'index', 'input', 'script_name', 'name'} + _NON_PARAMETERS = {"cmd", "foreach_stack", "index", "input", "script_name", "name"} _flow_decorators = {} @@ -82,6 +88,7 @@ def __init__(self, use_cli=True): # we import cli here to make sure custom parameters in # args.py get fully evaluated before cli.py is imported. from . import cli + cli.main(self) @property @@ -95,13 +102,13 @@ def script_name(self): A string containing the name of the script """ fname = inspect.getfile(self.__class__) - if fname.endswith('.pyc'): + if fname.endswith(".pyc"): fname = fname[:-1] return os.path.basename(fname) def _get_parameters(self): for var in dir(self): - if var[0] == '_' or var in self._NON_PARAMETERS: + if var[0] == "_" or var in self._NON_PARAMETERS: continue try: val = getattr(self, var) @@ -132,13 +139,10 @@ def __getattr__(self, name): setattr(self, name, x) return x else: - raise AttributeError("Flow %s has no attribute '%s'" % - (self.name, name)) + raise AttributeError("Flow %s has no attribute '%s'" % (self.name, name)) def cmd(self, cmdline, input={}, output=[]): - return cmd_with_io.cmd(cmdline, - input=input, - output=output) + return cmd_with_io.cmd(cmdline, input=input, output=output) @property def index(self): @@ -225,8 +229,10 @@ def nest_2(self): List[Tuple[int, int, object]] An array describing the current stack of foreach steps """ - return [(frame.index, frame.num_splits, self._find_input(stack_index=i)) - for i, frame in enumerate(self._foreach_stack)] + return [ + (frame.index, frame.num_splits, self._find_input(stack_index=i)) + for i, frame in enumerate(self._foreach_stack) + ] def _find_input(self, stack_index=None): if stack_index is None: @@ -255,9 +261,9 @@ def _find_input(self, stack_index=None): self._cached_input[stack_index] = var[frame.index] except TypeError: # __getitem__ not supported, fall back to an iterator - self._cached_input[stack_index] = next(islice(var, - frame.index, - frame.index + 1)) + self._cached_input[stack_index] = next( + islice(var, frame.index, frame.index + 1) + ) return self._cached_input[stack_index] def merge_artifacts(self, inputs, exclude=[], include=[]): @@ -312,9 +318,11 @@ def merge_artifacts(self, inputs, exclude=[], include=[]): be found """ node = self._graph[self._current_step] - if node.type != 'join': - msg = "merge_artifacts can only be called in a join and step *{step}* "\ - "is not a join".format(step=self._current_step) + if node.type != "join": + msg = ( + "merge_artifacts can only be called in a join and step *{step}* " + "is not a join".format(step=self._current_step) + ) raise MetaflowException(msg) if len(exclude) > 0 and len(include) > 0: msg = "`exclude` and `include` are mutually exclusive in merge_artifacts" @@ -325,11 +333,17 @@ def merge_artifacts(self, inputs, exclude=[], include=[]): for inp in inputs: # available_vars is the list of variables from inp that should be considered if include: - available_vars = ((var, sha) for var, sha in inp._datastore.items() - if (var in include) and (not hasattr(self, var))) + available_vars = ( + (var, sha) + for var, sha in inp._datastore.items() + if (var in include) and (not hasattr(self, var)) + ) else: - available_vars = ((var, sha) for var, sha in inp._datastore.items() - if (var not in exclude) and (not hasattr(self, var))) + available_vars = ( + (var, sha) + for var, sha in inp._datastore.items() + if (var not in exclude) and (not hasattr(self, var)) + ) for var, sha in available_vars: _, previous_sha = to_merge.setdefault(var, (inp, sha)) if previous_sha != sha: @@ -342,20 +356,25 @@ def merge_artifacts(self, inputs, exclude=[], include=[]): missing.append(v) if unresolved: # We have unresolved conflicts so we do not set anything and error out - msg = "Step *{step}* cannot merge the following artifacts due to them "\ - "having conflicting values:\n[{artifacts}].\nTo remedy this issue, "\ - "be sure to explicitly set those artifacts (using "\ - "self. = ...) prior to calling merge_artifacts."\ - .format(step=self._current_step, artifacts=', '.join(unresolved)) + msg = ( + "Step *{step}* cannot merge the following artifacts due to them " + "having conflicting values:\n[{artifacts}].\nTo remedy this issue, " + "be sure to explicitly set those artifacts (using " + "self. = ...) prior to calling merge_artifacts.".format( + step=self._current_step, artifacts=", ".join(unresolved) + ) + ) raise UnhandledInMergeArtifactsException(msg, unresolved) if missing: - msg = "Step *{step}* specifies that [{include}] should be merged but "\ - "[{missing}] are not present.\nTo remedy this issue, make sure "\ - "that the values specified in only come from at least one branch"\ - .format( - step=self._current_step, - include=', '.join(include), - missing=', '.join(missing)) + msg = ( + "Step *{step}* specifies that [{include}] should be merged but " + "[{missing}] are not present.\nTo remedy this issue, make sure " + "that the values specified in only come from at least one branch".format( + step=self._current_step, + include=", ".join(include), + missing=", ".join(missing), + ) + ) raise MissingInMergeArtifactsException(msg, missing) # If things are resolved, we pass down the variables from the input datastores for var, (inp, _) in to_merge.items(): @@ -364,20 +383,21 @@ def merge_artifacts(self, inputs, exclude=[], include=[]): def _validate_ubf_step(self, step_name): join_list = self._graph[step_name].out_funcs if len(join_list) != 1: - msg = "UnboundedForeach is supported only over a single node, "\ - "not an arbitrary DAG. Specify a single `join` node"\ - " instead of multiple:{join_list}."\ - .format(join_list=join_list) + msg = ( + "UnboundedForeach is supported only over a single node, " + "not an arbitrary DAG. Specify a single `join` node" + " instead of multiple:{join_list}.".format(join_list=join_list) + ) raise InvalidNextException(msg) join_step = join_list[0] join_node = self._graph[join_step] join_type = join_node.type - if join_type != 'join': - msg = "UnboundedForeach found for:{node} -> {join}."\ - " The join type isn't valid."\ - .format(node=step_name, - join=join_step) + if join_type != "join": + msg = ( + "UnboundedForeach found for:{node} -> {join}." + " The join type isn't valid.".format(node=step_name, join=join_step) + ) raise InvalidNextException(msg) def next(self, *dsts, **kwargs): @@ -407,18 +427,22 @@ def next(self, *dsts, **kwargs): step = self._current_step - foreach = kwargs.pop('foreach', None) - condition = kwargs.pop('condition', None) + foreach = kwargs.pop("foreach", None) + condition = kwargs.pop("condition", None) if kwargs: kw = next(iter(kwargs)) - msg = "Step *{step}* passes an unknown keyword argument "\ - "'{invalid}' to self.next().".format(step=step, invalid=kw) + msg = ( + "Step *{step}* passes an unknown keyword argument " + "'{invalid}' to self.next().".format(step=step, invalid=kw) + ) raise InvalidNextException(msg) # check: next() is called only once if self._transition is not None: - msg = "Multiple self.next() calls detected in step *{step}*. "\ - "Call self.next() only once.".format(step=step) + msg = ( + "Multiple self.next() calls detected in step *{step}*. " + "Call self.next() only once.".format(step=step) + ) raise InvalidNextException(msg) # check: all destinations are methods of this object @@ -427,45 +451,53 @@ def next(self, *dsts, **kwargs): try: name = dst.__func__.__name__ except: - msg = "In step *{step}* the {arg}. argument in self.next() is "\ - "not a function. Make sure all arguments in self.next() "\ - "are methods of the Flow class."\ - .format(step=step, arg=i + 1) + msg = ( + "In step *{step}* the {arg}. argument in self.next() is " + "not a function. Make sure all arguments in self.next() " + "are methods of the Flow class.".format(step=step, arg=i + 1) + ) raise InvalidNextException(msg) if not hasattr(self, name): - msg = "Step *{step}* specifies a self.next() transition to an "\ - "unknown step, *{name}*.".format(step=step, - name=name) + msg = ( + "Step *{step}* specifies a self.next() transition to an " + "unknown step, *{name}*.".format(step=step, name=name) + ) raise InvalidNextException(msg) funcs.append(name) # check: foreach and condition are mutually exclusive if not (foreach is None or condition is None): - msg = "Step *{step}* has an invalid self.next() transition. "\ - "Specify either 'foreach' or 'condition', not both."\ - .format(step=step) + msg = ( + "Step *{step}* has an invalid self.next() transition. " + "Specify either 'foreach' or 'condition', not both.".format(step=step) + ) raise InvalidNextException(msg) # check: foreach is valid if foreach: if not isinstance(foreach, basestring): - msg = "Step *{step}* has an invalid self.next() transition. "\ - "The argument to 'foreach' must be a string."\ - .format(step=step) + msg = ( + "Step *{step}* has an invalid self.next() transition. " + "The argument to 'foreach' must be a string.".format(step=step) + ) raise InvalidNextException(msg) if len(dsts) != 1: - msg = "Step *{step}* has an invalid self.next() transition. "\ - "Specify exactly one target for 'foreach'."\ - .format(step=step) + msg = ( + "Step *{step}* has an invalid self.next() transition. " + "Specify exactly one target for 'foreach'.".format(step=step) + ) raise InvalidNextException(msg) try: foreach_iter = getattr(self, foreach) except: - msg = "Foreach variable *self.{var}* in step *{step}* "\ - "does not exist. Check your variable."\ - .format(step=step, var=foreach) + msg = ( + "Foreach variable *self.{var}* in step *{step}* " + "does not exist. Check your variable.".format( + step=step, var=foreach + ) + ) raise InvalidNextException(msg) if issubclass(type(foreach_iter), UnboundedForeachInput): @@ -476,15 +508,21 @@ def next(self, *dsts, **kwargs): try: self._foreach_num_splits = sum(1 for _ in foreach_iter) except TypeError: - msg = "Foreach variable *self.{var}* in step *{step}* "\ - "is not iterable. Check your variable."\ - .format(step=step, var=foreach) + msg = ( + "Foreach variable *self.{var}* in step *{step}* " + "is not iterable. Check your variable.".format( + step=step, var=foreach + ) + ) raise InvalidNextException(msg) if self._foreach_num_splits == 0: - msg = "Foreach iterator over *{var}* in step *{step}* "\ - "produced zero splits. Check your variable."\ - .format(step=step, var=foreach) + msg = ( + "Foreach iterator over *{var}* in step *{step}* " + "produced zero splits. Check your variable.".format( + step=step, var=foreach + ) + ) raise InvalidNextException(msg) self._foreach_var = foreach @@ -492,51 +530,61 @@ def next(self, *dsts, **kwargs): # check: condition is valid if condition: if not isinstance(condition, basestring): - msg = "Step *{step}* has an invalid self.next() transition. "\ - "The argument to 'condition' must be a string."\ - .format(step=step) + msg = ( + "Step *{step}* has an invalid self.next() transition. " + "The argument to 'condition' must be a string.".format(step=step) + ) raise InvalidNextException(msg) if len(dsts) != 2: - msg = "Step *{step}* has an invalid self.next() transition. "\ - "Specify two targets for 'condition': The first target "\ - "is used if the condition evaluates to true, the second "\ - "otherwise.".format(step=step) + msg = ( + "Step *{step}* has an invalid self.next() transition. " + "Specify two targets for 'condition': The first target " + "is used if the condition evaluates to true, the second " + "otherwise.".format(step=step) + ) raise InvalidNextException(msg) # check: non-keyword transitions are valid if foreach is None and condition is None: if len(dsts) < 1: - msg = "Step *{step}* has an invalid self.next() transition. "\ - "Specify at least one step function as an argument in "\ - "self.next().".format(step=step) + msg = ( + "Step *{step}* has an invalid self.next() transition. " + "Specify at least one step function as an argument in " + "self.next().".format(step=step) + ) raise InvalidNextException(msg) self._transition = (funcs, foreach, condition) def __str__(self): - step_name = getattr(self, '_current_step', None) + step_name = getattr(self, "_current_step", None) if step_name: - index = ','.join(str(idx) for idx, _, _ in self.foreach_stack()) + index = ",".join(str(idx) for idx, _, _ in self.foreach_stack()) if index: inp = self.input if inp is None: - return '' %\ - (self.name, step_name, index) + return "" % (self.name, step_name, index) else: inp = str(inp) if len(inp) > 20: - inp = inp[:20] + '...' - return '' %\ - (self.name, step_name, index, inp) + inp = inp[:20] + "..." + return "" % ( + self.name, + step_name, + index, + inp, + ) else: - return '' % (self.name, step_name) + return "" % (self.name, step_name) else: - return '' % self.name + return "" % self.name def __getstate__(self): - raise MetaflowException("Flows can't be serialized. Maybe you tried " - "to assign *self* or one of the *inputs* " - "to an attribute? Instead of serializing the " - "whole flow, you should choose specific " - "attributes, e.g. *input.some_var*, to be " - "stored.") + raise MetaflowException( + "Flows can't be serialized. Maybe you tried " + "to assign *self* or one of the *inputs* " + "to an attribute? Instead of serializing the " + "whole flow, you should choose specific " + "attributes, e.g. *input.some_var*, to be " + "stored." + ) diff --git a/metaflow/graph.py b/metaflow/graph.py index 7a94a1d22aa..ecf83a9ddd5 100644 --- a/metaflow/graph.py +++ b/metaflow/graph.py @@ -2,6 +2,7 @@ import ast import re + def deindent_docstring(doc): if doc: # Find the indent to remove from the doctring. We consider the following possibilities: @@ -28,16 +29,17 @@ def deindent_docstring(doc): matched_indent = None for line in doc.splitlines(): if line: - matched_indent = re.match('[\t ]+', line) + matched_indent = re.match("[\t ]+", line) if matched_indent is not None or saw_first_line: break saw_first_line = True if matched_indent: - return re.sub(r'\n' + matched_indent.group(), '\n', doc).strip() + return re.sub(r"\n" + matched_indent.group(), "\n", doc).strip() else: return doc else: - return '' + return "" + class DAGNode(object): def __init__(self, func_ast, decos, doc): @@ -66,7 +68,7 @@ def __init__(self, func_ast, decos, doc): self.is_inside_foreach = False def _expr_str(self, expr): - return '%s.%s' % (expr.value.id, expr.attr) + return "%s.%s" % (expr.value.id, expr.attr) def _parse(self, func_ast): @@ -74,9 +76,9 @@ def _parse(self, func_ast): tail = func_ast.body[-1] # end doesn't need a transition - if self.name == 'end': + if self.name == "end": # TYPE: end - self.type = 'end' + self.type = "end" # ensure that the tail an expression if not isinstance(tail, ast.Expr): @@ -84,7 +86,7 @@ def _parse(self, func_ast): # determine the type of self.next transition try: - if not self._expr_str(tail.value.func) == 'self.next': + if not self._expr_str(tail.value.func) == "self.next": return self.has_tail_next = True @@ -94,37 +96,36 @@ def _parse(self, func_ast): keywords = dict((k.arg, k.value.s) for k in tail.value.keywords) if len(keywords) == 1: - if 'foreach' in keywords: + if "foreach" in keywords: # TYPE: foreach - self.type = 'foreach' + self.type = "foreach" if len(self.out_funcs) == 1: - self.foreach_param = keywords['foreach'] + self.foreach_param = keywords["foreach"] self.invalid_tail_next = False - elif 'condition' in keywords: + elif "condition" in keywords: # TYPE: split-or - self.type = 'split-or' + self.type = "split-or" if len(self.out_funcs) == 2: - self.condition = keywords['condition'] + self.condition = keywords["condition"] self.invalid_tail_next = False elif len(keywords) == 0: if len(self.out_funcs) > 1: # TYPE: split-and - self.type = 'split-and' + self.type = "split-and" self.invalid_tail_next = False elif len(self.out_funcs) == 1: # TYPE: linear if self.num_args > 1: - self.type = 'join' + self.type = "join" else: - self.type = 'linear' + self.type = "linear" self.invalid_tail_next = False except AttributeError: return def __str__(self): - return\ -"""*[{0.name} {0.type} (line {0.func_lineno})]* + return """*[{0.name} {0.type} (line {0.func_lineno})]* in_funcs={in_funcs} out_funcs={out_funcs} split_parents={parents} @@ -136,18 +137,19 @@ def __str__(self): invalid_tail_next={0.invalid_tail_next} condition={0.condition} foreach_param={0.foreach_param} - -> {out}"""\ - .format(self, - matching_join=self.matching_join and '[%s]' % self.matching_join, + -> {out}""".format( + self, + matching_join=self.matching_join and "[%s]" % self.matching_join, is_inside_foreach=self.is_inside_foreach, - out_funcs=', '.join('[%s]' % x for x in self.out_funcs), - in_funcs=', '.join('[%s]' % x for x in self.in_funcs), - parents=', '.join('[%s]' % x for x in self.split_parents), - decos=' | '.join(map(str, self.decorators)), - out=', '.join('[%s]' % x for x in self.out_funcs)) + out_funcs=", ".join("[%s]" % x for x in self.out_funcs), + in_funcs=", ".join("[%s]" % x for x in self.in_funcs), + parents=", ".join("[%s]" % x for x in self.split_parents), + decos=" | ".join(map(str, self.decorators)), + out=", ".join("[%s]" % x for x in self.out_funcs), + ) -class StepVisitor(ast.NodeVisitor): +class StepVisitor(ast.NodeVisitor): def __init__(self, nodes, flow): self.nodes = nodes self.flow = flow @@ -155,11 +157,11 @@ def __init__(self, nodes, flow): def visit_FunctionDef(self, node): func = getattr(self.flow, node.name) - if hasattr(func, 'is_step'): + if hasattr(func, "is_step"): self.nodes[node.name] = DAGNode(node, func.decorators, func.__doc__) -class FlowGraph(object): +class FlowGraph(object): def __init__(self, flow): self.name = flow.__name__ self.nodes = self._create_nodes(flow) @@ -170,8 +172,9 @@ def __init__(self, flow): def _create_nodes(self, flow): module = __import__(flow.__module__) tree = ast.parse(inspect.getsource(module)).body - root = [n for n in tree\ - if isinstance(n, ast.ClassDef) and n.name == self.name][0] + root = [n for n in tree if isinstance(n, ast.ClassDef) and n.name == self.name][ + 0 + ] nodes = {} StepVisitor(nodes, flow).visit(root) return nodes @@ -181,20 +184,19 @@ def _postprocess(self): # has is_inside_foreach=True *unless* all of those foreaches # are joined by the node for node in self.nodes.values(): - foreaches = [p for p in node.split_parents - if self.nodes[p].type == 'foreach'] - if [f for f in foreaches - if self.nodes[f].matching_join != node.name]: + foreaches = [ + p for p in node.split_parents if self.nodes[p].type == "foreach" + ] + if [f for f in foreaches if self.nodes[f].matching_join != node.name]: node.is_inside_foreach = True def _traverse_graph(self): - def traverse(node, seen, split_parents): - if node.type in ('split-or', 'split-and', 'foreach'): + if node.type in ("split-or", "split-and", "foreach"): node.split_parents = split_parents split_parents = split_parents + [node.name] - elif node.type == 'join': + elif node.type == "join": # ignore joins without splits if split_parents: self[split_parents[-1]].matching_join = node.name @@ -212,8 +214,8 @@ def traverse(node, seen, split_parents): child.in_funcs.add(node.name) traverse(child, seen + [n], split_parents) - if 'start' in self: - traverse(self['start'], [], []) + if "start" in self: + traverse(self["start"], [], []) # fix the order of in_funcs for node in self.nodes.values(): @@ -229,28 +231,28 @@ def __iter__(self): return iter(self.nodes.values()) def __str__(self): - return '\n'.join(str(n) for _, n in sorted((n.func_lineno, n)\ - for n in self.nodes.values())) + return "\n".join( + str(n) for _, n in sorted((n.func_lineno, n) for n in self.nodes.values()) + ) def output_dot(self): - def edge_specs(): for node in self.nodes.values(): for edge in node.out_funcs: - yield '%s -> %s;' % (node.name, edge) + yield "%s -> %s;" % (node.name, edge) def node_specs(): for node in self.nodes.values(): - nodetype = 'join' if node.num_args > 1 else node.type - yield '"{0.name}"'\ - '[ label = <{0.name} | {type}> '\ - ' fontname = "Helvetica" '\ - ' shape = "record" ];'.format(node, type=nodetype) - - return "digraph {0.name} {{\n"\ - "{nodes}\n"\ - "{edges}\n"\ - "}}".format(self, - nodes='\n'.join(node_specs()), - edges='\n'.join(edge_specs())) - + nodetype = "join" if node.num_args > 1 else node.type + yield '"{0.name}"' '[ label = <{0.name} | {type}> ' ' fontname = "Helvetica" ' ' shape = "record" ];'.format( + node, type=nodetype + ) + + return ( + "digraph {0.name} {{\n" + "{nodes}\n" + "{edges}\n" + "}}".format( + self, nodes="\n".join(node_specs()), edges="\n".join(edge_specs()) + ) + ) diff --git a/metaflow/includefile.py b/metaflow/includefile.py index 524b2ebc78a..6b4f7b717bf 100644 --- a/metaflow/includefile.py +++ b/metaflow/includefile.py @@ -28,10 +28,12 @@ # TODO: This local "client" and the general notion of dataclients should probably # be moved somewhere else. Putting here to keep this change compact for now class MetaflowLocalURLException(MetaflowException): - headline = 'Invalid path' + headline = "Invalid path" + class MetaflowLocalNotFound(MetaflowException): - headline = 'Local object not found' + headline = "Local object not found" + class LocalObject(object): """ @@ -47,6 +49,7 @@ def __init__(self, url, path): # all fields of S3Object should return a unicode object def ensure_unicode(x): return None if x is None else to_unicode(x) + path = ensure_unicode(path) self._path = path @@ -104,6 +107,7 @@ def get_root_from_config(cls, echo, create_on_absent=True): result = DATATOOLS_LOCALROOT if result is None: from .datastore.local_storage import LocalStorage + result = LocalStorage.get_datastore_root_from_config(echo, create_on_absent) result = os.path.join(result, DATATOOLS_SUFFIX) if create_on_absent and not os.path.exists(result): @@ -125,24 +129,26 @@ def __exit__(self, *args): def _path(self, key): key = to_unicode(key) - if key.startswith(u'local://'): + if key.startswith(u"local://"): return key[8:] - elif key[0] != u'/': + elif key[0] != u"/": if current.is_running_flow: raise MetaflowLocalURLException( "Specify Local(run=self) when you use Local inside a running " "flow. Otherwise you have to use Local with full " - "local:// urls or absolute paths.") + "local:// urls or absolute paths." + ) else: raise MetaflowLocalURLException( "Initialize Local with an 'localroot' or 'run' if you don't " - "want to specify full local:// urls or absolute paths.") + "want to specify full local:// urls or absolute paths." + ) else: return key def get(self, key=None, return_missing=False): p = self._path(key) - url = u'local://%s' % p + url = u"local://%s" % p if not os.path.isfile(p): if return_missing: p = None @@ -154,18 +160,18 @@ def put(self, key, obj, overwrite=True): p = self._path(key) if overwrite or (not os.path.exists(p)): Local._makedirs(os.path.dirname(p)) - with open(p, 'wb') as f: + with open(p, "wb") as f: f.write(obj) - return u'local://%s' % p + return u"local://%s" % p # From here on out, this is the IncludeFile implementation. from .datatools import S3 -DATACLIENTS = {'local': Local, - 's3': S3} +DATACLIENTS = {"local": Local, "s3": S3} + -class LocalFile(): +class LocalFile: def __init__(self, is_text, encoding, path): self._is_text = is_text self._encoding = encoding @@ -175,16 +181,22 @@ def __init__(self, is_text, encoding, path): def is_file_handled(cls, path): if path: decoded_value = Uploader.decode_value(to_unicode(path)) - if decoded_value['type'] == 'self': - return True, LocalFile( - decoded_value['is_text'], decoded_value['encoding'], - decoded_value['url']), None - path = decoded_value['url'] + if decoded_value["type"] == "self": + return ( + True, + LocalFile( + decoded_value["is_text"], + decoded_value["encoding"], + decoded_value["url"], + ), + None, + ) + path = decoded_value["url"] for prefix, handler in DATACLIENTS.items(): if path.startswith(u"%s://" % prefix): return True, Uploader(handler), None try: - with open(path, mode='r') as _: + with open(path, mode="r") as _: pass except OSError: return False, None, "IncludeFile: could not open file '%s'" % path @@ -206,13 +218,15 @@ def __call__(self, ctx): client = DATACLIENTS.get(ctx.ds_type) if client: return Uploader(client).store( - ctx.flow_name, self._path, self._is_text, self._encoding, ctx.logger) + ctx.flow_name, self._path, self._is_text, self._encoding, ctx.logger + ) raise MetaflowException( - "IncludeFile: no client found for datastore type %s" % ctx.ds_type) + "IncludeFile: no client found for datastore type %s" % ctx.ds_type + ) class FilePathClass(click.ParamType): - name = 'FilePath' + name = "FilePath" # The logic for this class is as follows: # - It will always return a path that indicates the persisted path of the file. # + If the value is already such a string, nothing happens and it returns that same value @@ -237,78 +251,87 @@ def convert(self, value, param, ctx): self.fail(err) if file_type is None: # Here, we need to store the file - return lambda is_text=self._is_text, encoding=self._encoding,\ - value=value, ctx=parameters.context_proto: LocalFile( - is_text, encoding, value)(ctx) + return lambda is_text=self._is_text, encoding=self._encoding, value=value, ctx=parameters.context_proto: LocalFile( + is_text, encoding, value + )( + ctx + ) elif isinstance(file_type, LocalFile): # This is a default file that we evaluate now (to delay upload # until *after* the flow is checked) return lambda f=file_type, ctx=parameters.context_proto: f(ctx) else: # We will just store the URL in the datastore along with text/encoding info - return lambda is_text=self._is_text, encoding=self._encoding,\ - value=value: Uploader.encode_url( - 'external', value, is_text=is_text, encoding=encoding) + return lambda is_text=self._is_text, encoding=self._encoding, value=value: Uploader.encode_url( + "external", value, is_text=is_text, encoding=encoding + ) def __str__(self): return repr(self) def __repr__(self): - return 'FilePath' + return "FilePath" class IncludeFile(Parameter): - def __init__( - self, name, required=False, is_text=True, encoding=None, help=None, **kwargs): + self, name, required=False, is_text=True, encoding=None, help=None, **kwargs + ): # Defaults are DeployTimeField - v = kwargs.get('default') + v = kwargs.get("default") if v is not None: _, file_type, _ = LocalFile.is_file_handled(v) # Ignore error because we may never use the default if file_type is None: - o = { - 'type': 'self', - 'is_text': is_text, - 'encoding': encoding, - 'url': v - } - kwargs['default'] = DeployTimeField( + o = {"type": "self", "is_text": is_text, "encoding": encoding, "url": v} + kwargs["default"] = DeployTimeField( name, str, - 'default', - lambda ctx, full_evaluation, o=o: \ - LocalFile(o['is_text'], o['encoding'], o['url'])(ctx) \ - if full_evaluation else json.dumps(o), - print_representation=v) + "default", + lambda ctx, full_evaluation, o=o: LocalFile( + o["is_text"], o["encoding"], o["url"] + )(ctx) + if full_evaluation + else json.dumps(o), + print_representation=v, + ) else: - kwargs['default'] = DeployTimeField( + kwargs["default"] = DeployTimeField( name, str, - 'default', - lambda _, __, is_text=is_text, encoding=encoding, v=v: \ - Uploader.encode_url('external-default', v, - is_text=is_text, encoding=encoding), - print_representation=v) + "default", + lambda _, __, is_text=is_text, encoding=encoding, v=v: Uploader.encode_url( + "external-default", v, is_text=is_text, encoding=encoding + ), + print_representation=v, + ) super(IncludeFile, self).__init__( - name, required=required, help=help, - type=FilePathClass(is_text, encoding), **kwargs) + name, + required=required, + help=help, + type=FilePathClass(is_text, encoding), + **kwargs + ) def load_parameter(self, val): if val is None: return val ok, file_type, err = LocalFile.is_file_handled(val) if not ok: - raise MetaflowException("Parameter '%s' could not be loaded: %s" % (self.name, err)) + raise MetaflowException( + "Parameter '%s' could not be loaded: %s" % (self.name, err) + ) if file_type is None or isinstance(file_type, LocalFile): - raise MetaflowException("Parameter '%s' was not properly converted" % self.name) + raise MetaflowException( + "Parameter '%s' was not properly converted" % self.name + ) return file_type.load(val) -class Uploader(): +class Uploader: - file_type = 'uploader-v1' + file_type = "uploader-v1" def __init__(self, client_class): self._client_class = client_class @@ -316,67 +339,73 @@ def __init__(self, client_class): @staticmethod def encode_url(url_type, url, **kwargs): # Avoid encoding twice (default -> URL -> _convert method of FilePath for example) - if url is None or len(url) == 0 or url[0] == '{': + if url is None or len(url) == 0 or url[0] == "{": return url - return_value = {'type': url_type, 'url': url} + return_value = {"type": url_type, "url": url} return_value.update(kwargs) return json.dumps(return_value) @staticmethod def decode_value(value): - if value is None or len(value) == 0 or value[0] != '{': - return {'type': 'base', 'url': value} + if value is None or len(value) == 0 or value[0] != "{": + return {"type": "base", "url": value} return json.loads(value) def store(self, flow_name, path, is_text, encoding, echo): sz = os.path.getsize(path) - unit = ['B', 'KB', 'MB', 'GB', 'TB'] + unit = ["B", "KB", "MB", "GB", "TB"] pos = 0 while pos < len(unit) and sz >= 1024: sz = sz // 1024 pos += 1 if pos >= 3: - extra = '(this may take a while)' + extra = "(this may take a while)" else: - extra = '' - echo( - 'Including file %s of size %d%s %s' % (path, sz, unit[pos], extra)) + extra = "" + echo("Including file %s of size %d%s %s" % (path, sz, unit[pos], extra)) try: - input_file = io.open(path, mode='rb').read() + input_file = io.open(path, mode="rb").read() except IOError: # If we get an error here, since we know that the file exists already, # it means that read failed which happens with Python 2.7 for large files - raise MetaflowException('Cannot read file at %s -- this is likely because it is too ' - 'large to be properly handled by Python 2.7' % path) + raise MetaflowException( + "Cannot read file at %s -- this is likely because it is too " + "large to be properly handled by Python 2.7" % path + ) sha = sha1(input_file).hexdigest() - path = os.path.join(self._client_class.get_root_from_config(echo, True), - flow_name, - sha) + path = os.path.join( + self._client_class.get_root_from_config(echo, True), flow_name, sha + ) buf = io.BytesIO() - with gzip.GzipFile( - fileobj=buf, mode='wb', compresslevel=3) as f: + with gzip.GzipFile(fileobj=buf, mode="wb", compresslevel=3) as f: f.write(input_file) buf.seek(0) with self._client_class() as client: url = client.put(path, buf.getvalue(), overwrite=False) - echo('File persisted at %s' % url) - return Uploader.encode_url(Uploader.file_type, url, is_text=is_text, encoding=encoding) + echo("File persisted at %s" % url) + return Uploader.encode_url( + Uploader.file_type, url, is_text=is_text, encoding=encoding + ) def load(self, value): value_info = Uploader.decode_value(value) with self._client_class() as client: - obj = client.get(value_info['url'], return_missing=True) + obj = client.get(value_info["url"], return_missing=True) if obj.exists: - if value_info['type'] == Uploader.file_type: + if value_info["type"] == Uploader.file_type: # We saved this file directly so we know how to read it out - with gzip.GzipFile(filename=obj.path, mode='rb') as f: - if value_info['is_text']: - return io.TextIOWrapper(f, encoding=value_info.get('encoding')).read() + with gzip.GzipFile(filename=obj.path, mode="rb") as f: + if value_info["is_text"]: + return io.TextIOWrapper( + f, encoding=value_info.get("encoding") + ).read() return f.read() else: # We open this file according to the is_text and encoding information - if value_info['is_text']: - return io.open(obj.path, mode='rt', encoding=value_info.get('encoding')).read() + if value_info["is_text"]: + return io.open( + obj.path, mode="rt", encoding=value_info.get("encoding") + ).read() else: - return io.open(obj.path, mode='rb').read() - raise FileNotFoundError("File at %s does not exist" % value_info['url']) + return io.open(obj.path, mode="rb").read() + raise FileNotFoundError("File at %s does not exist" % value_info["url"]) diff --git a/metaflow/lint.py b/metaflow/lint.py index 6d59926b34e..6dd84fa8521 100644 --- a/metaflow/lint.py +++ b/metaflow/lint.py @@ -2,11 +2,12 @@ from .exception import MetaflowException from .util import all_equal + class LintWarn(MetaflowException): - headline="Validity checker found an issue" + headline = "Validity checker found an issue" -class FlowLinter(object): +class FlowLinter(object): def __init__(self): self.require_static_graph = True self.require_fundamentals = True @@ -19,16 +20,16 @@ def _decorate(self, setting, f): return f def ensure_static_graph(self, f): - return self._decorate('require_static_graph', f) + return self._decorate("require_static_graph", f) def ensure_fundamentals(self, f): - return self._decorate('require_fundamentals', f) + return self._decorate("require_fundamentals", f) def ensure_acyclicity(self, f): - return self._decorate('require_acyclicity', f) + return self._decorate("require_acyclicity", f) def ensure_non_nested_foreach(self, f): - return self._decorate('require_non_nested_foreach', f) + return self._decorate("require_non_nested_foreach", f) def check(self, f): self._checks.append(f) @@ -37,193 +38,218 @@ def check(self, f): def run_checks(self, graph, **kwargs): for check in self._checks: - if any(getattr(self, attr) or kwargs.get(attr) - for attr in check.attrs): + if any(getattr(self, attr) or kwargs.get(attr) for attr in check.attrs): check(graph) + linter = FlowLinter() + @linter.ensure_fundamentals @linter.check def check_reserved_words(graph): - RESERVED = {'name', - 'next', - 'input', - 'index', - 'cmd'} - msg = 'Step name *%s* is a reserved word. Choose another name for the '\ - 'step.' + RESERVED = {"name", "next", "input", "index", "cmd"} + msg = "Step name *%s* is a reserved word. Choose another name for the " "step." for node in graph: if node.name in RESERVED: raise LintWarn(msg % node.name) + @linter.ensure_fundamentals @linter.check def check_basic_steps(graph): - msg ="Add %s *%s* step in your flow." - for prefix, node in (('a', 'start'), ('an', 'end')): + msg = "Add %s *%s* step in your flow." + for prefix, node in (("a", "start"), ("an", "end")): if node not in graph: raise LintWarn(msg % (prefix, node)) + @linter.ensure_static_graph @linter.check def check_that_end_is_end(graph): - msg0="The *end* step should not have a step.next() transition. "\ - "Just remove it." - msg1="The *end* step should not be a join step (it gets an extra "\ - "argument). Add a join step before it." + msg0 = "The *end* step should not have a step.next() transition. " "Just remove it." + msg1 = ( + "The *end* step should not be a join step (it gets an extra " + "argument). Add a join step before it." + ) - node=graph['end'] + node = graph["end"] if node.has_tail_next or node.invalid_tail_next: raise LintWarn(msg0, node.tail_next_lineno) if node.num_args > 1: raise LintWarn(msg1, node.tail_next_lineno) + @linter.ensure_fundamentals @linter.check def check_step_names(graph): - msg =\ - "Step *{0.name}* has an invalid name. Only lowercase ascii "\ - "characters, underscores, and digits are allowed." + msg = ( + "Step *{0.name}* has an invalid name. Only lowercase ascii " + "characters, underscores, and digits are allowed." + ) for node in graph: - if re.search('[^a-z0-9_]', node.name) or node.name[0] == '_': + if re.search("[^a-z0-9_]", node.name) or node.name[0] == "_": raise LintWarn(msg.format(node), node.func_lineno) + @linter.ensure_fundamentals @linter.check def check_num_args(graph): - msg0 =\ - "Step {0.name} has too many arguments. Normal steps take only "\ - "'self' as an argument. Join steps take 'self' and 'inputs'." - msg1 =\ - "Step *{0.name}* is both a join step (it takes an extra argument) "\ - "and a split step (it transitions to multiple steps). This is not "\ - "allowed. Add a new step so that split and join become separate steps." + msg0 = ( + "Step {0.name} has too many arguments. Normal steps take only " + "'self' as an argument. Join steps take 'self' and 'inputs'." + ) + msg1 = ( + "Step *{0.name}* is both a join step (it takes an extra argument) " + "and a split step (it transitions to multiple steps). This is not " + "allowed. Add a new step so that split and join become separate steps." + ) msg2 = "Step *{0.name}* is missing the 'self' argument." for node in graph: if node.num_args > 2: raise LintWarn(msg0.format(node), node.func_lineno) - elif node.num_args == 2 and node.type != 'join': + elif node.num_args == 2 and node.type != "join": raise LintWarn(msg1.format(node), node.func_lineno) elif node.num_args == 0: raise LintWarn(msg2.format(node), node.func_lineno) + @linter.ensure_static_graph @linter.check def check_static_transitions(graph): - msg =\ - "Step *{0.name}* is missing a self.next() transition to "\ - "the next step. Add a self.next() as the last line in the "\ - "function." + msg = ( + "Step *{0.name}* is missing a self.next() transition to " + "the next step. Add a self.next() as the last line in the " + "function." + ) for node in graph: - if node.type != 'end' and not node.has_tail_next: + if node.type != "end" and not node.has_tail_next: raise LintWarn(msg.format(node), node.func_lineno) + @linter.ensure_static_graph @linter.check def check_valid_transitions(graph): - msg =\ - "Step *{0.name}* specifies an invalid self.next() transition. "\ - "Make sure the self.next() expression matches with one of the "\ - "supported transition types." + msg = ( + "Step *{0.name}* specifies an invalid self.next() transition. " + "Make sure the self.next() expression matches with one of the " + "supported transition types." + ) for node in graph: - if node.type != 'end' and\ - node.has_tail_next and\ - node.invalid_tail_next: + if node.type != "end" and node.has_tail_next and node.invalid_tail_next: raise LintWarn(msg.format(node), node.tail_next_lineno) + @linter.ensure_static_graph @linter.check def check_unknown_transitions(graph): - msg =\ - "Step *{0.name}* specifies a self.next() transition to "\ - "an unknown step, *{step}*." + msg = ( + "Step *{0.name}* specifies a self.next() transition to " + "an unknown step, *{step}*." + ) for node in graph: unknown = [n for n in node.out_funcs if n not in graph] if unknown: - raise LintWarn(msg.format(node, step=unknown[0]), - node.tail_next_lineno) + raise LintWarn(msg.format(node, step=unknown[0]), node.tail_next_lineno) + @linter.ensure_acyclicity @linter.ensure_static_graph @linter.check def check_for_acyclicity(graph): - msg = "There is a loop in your flow: *{0}*. Break the loop "\ - "by fixing self.next() transitions." + msg = ( + "There is a loop in your flow: *{0}*. Break the loop " + "by fixing self.next() transitions." + ) + def check_path(node, seen): for n in node.out_funcs: if n in seen: - path = '->'.join(seen + [n]) - raise LintWarn(msg.format(path), - node.tail_next_lineno) + path = "->".join(seen + [n]) + raise LintWarn(msg.format(path), node.tail_next_lineno) else: check_path(graph[n], seen + [n]) + for start in graph: check_path(start, []) + @linter.ensure_static_graph @linter.check def check_for_orphans(graph): - msg =\ - "Step *{0.name}* is unreachable from the start step. Add "\ - "self.next({0.name}) in another step or remove *{0.name}*." - seen = set(['start']) + msg = ( + "Step *{0.name}* is unreachable from the start step. Add " + "self.next({0.name}) in another step or remove *{0.name}*." + ) + seen = set(["start"]) + def traverse(node): for n in node.out_funcs: if n not in seen: seen.add(n) traverse(graph[n]) - traverse(graph['start']) + + traverse(graph["start"]) nodeset = frozenset(n.name for n in graph) orphans = nodeset - seen if orphans: orphan = graph[list(orphans)[0]] raise LintWarn(msg.format(orphan), orphan.func_lineno) + @linter.ensure_static_graph @linter.check def check_split_join_balance(graph): - msg0 = "Step *end* reached before a split started at step(s) *{roots}* "\ - "were joined. Add a join step before *end*." - msg1 = "Step *{0.name}* seems like a join step (it takes an extra input "\ - "argument) but an incorrect number of steps (*{paths}*) lead to "\ - "it. This join was expecting {num_roots} incoming paths, starting "\ - "from split step(s) *{roots}*." - msg2 = "Step *{0.name}* seems like a join step (it takes an extra input "\ - "argument) but it is not preceded by a split. Ensure that there is "\ - "a matching split for every join." - msg3 = "Step *{0.name}* joins steps from unrelated splits. Ensure that "\ - "there is a matching join for every split." + msg0 = ( + "Step *end* reached before a split started at step(s) *{roots}* " + "were joined. Add a join step before *end*." + ) + msg1 = ( + "Step *{0.name}* seems like a join step (it takes an extra input " + "argument) but an incorrect number of steps (*{paths}*) lead to " + "it. This join was expecting {num_roots} incoming paths, starting " + "from split step(s) *{roots}*." + ) + msg2 = ( + "Step *{0.name}* seems like a join step (it takes an extra input " + "argument) but it is not preceded by a split. Ensure that there is " + "a matching split for every join." + ) + msg3 = ( + "Step *{0.name}* joins steps from unrelated splits. Ensure that " + "there is a matching join for every split." + ) def traverse(node, split_stack): - if node.type == 'linear': + if node.type == "linear": new_stack = split_stack - elif node.type in ('split-or', 'split-and', 'foreach'): - new_stack = split_stack + [('split', node.out_funcs)] - elif node.type == 'end': + elif node.type in ("split-or", "split-and", "foreach"): + new_stack = split_stack + [("split", node.out_funcs)] + elif node.type == "end": if split_stack: split_type, split_roots = split_stack.pop() - roots = ', '.join(split_roots) + roots = ", ".join(split_roots) raise LintWarn(msg0.format(roots=roots)) - elif node.type == 'join': + elif node.type == "join": if split_stack: split_type, split_roots = split_stack[-1] new_stack = split_stack[:-1] if len(node.in_funcs) != len(split_roots): - paths = ', '.join(node.in_funcs) - roots = ', '.join(split_roots) - raise LintWarn(msg1.format(node, - paths=paths, - num_roots=len(split_roots), - roots=roots), - node.func_lineno) + paths = ", ".join(node.in_funcs) + roots = ", ".join(split_roots) + raise LintWarn( + msg1.format( + node, paths=paths, num_roots=len(split_roots), roots=roots + ), + node.func_lineno, + ) else: raise LintWarn(msg2.format(node), node.func_lineno) # check that incoming steps come from the same lineage # (no cross joins) def parents(n): - if graph[n].type == 'join': + if graph[n].type == "join": return tuple(graph[n].split_parents[:-1]) else: return tuple(graph[n].split_parents) @@ -234,27 +260,32 @@ def parents(n): for n in node.out_funcs: traverse(graph[n], new_stack) - traverse(graph['start'], []) + traverse(graph["start"], []) + @linter.ensure_static_graph @linter.check def check_empty_foreaches(graph): - msg = "Step *{0.name}* is a foreach split that has no children: "\ - "it is followed immediately by a join step, *{join}*. Add "\ - "at least one step between the split and the join." + msg = ( + "Step *{0.name}* is a foreach split that has no children: " + "it is followed immediately by a join step, *{join}*. Add " + "at least one step between the split and the join." + ) for node in graph: - if node.type == 'foreach': - joins = [n for n in node.out_funcs if graph[n].type == 'join'] + if node.type == "foreach": + joins = [n for n in node.out_funcs if graph[n].type == "join"] if joins: raise LintWarn(msg.format(node, join=joins[0])) + @linter.ensure_non_nested_foreach @linter.check def check_nested_foreach(graph): - msg = "Nested foreaches are not allowed: Step *{0.name}* is a foreach "\ - "split that is nested under another foreach split." + msg = ( + "Nested foreaches are not allowed: Step *{0.name}* is a foreach " + "split that is nested under another foreach split." + ) for node in graph: - if node.type == 'foreach': - if any(graph[p].type == 'foreach' for p in node.split_parents): + if node.type == "foreach": + if any(graph[p].type == "foreach" for p in node.split_parents): raise LintWarn(msg.format(node)) - diff --git a/metaflow/main_cli.py b/metaflow/main_cli.py index 09650507b63..b34084502d6 100644 --- a/metaflow/main_cli.py +++ b/metaflow/main_cli.py @@ -21,12 +21,15 @@ def makedirs(path): else: raise + def echo_dev_null(*args, **kwargs): pass + def echo_always(line, **kwargs): click.secho(line, **kwargs) + @click.group(invoke_without_command=True) @click.pass_context def main(ctx): @@ -34,176 +37,176 @@ def main(ctx): echo = echo_always import metaflow - echo('Metaflow ', - fg='magenta', - bold=True, - nl=False) + + echo("Metaflow ", fg="magenta", bold=True, nl=False) if ctx.invoked_subcommand is None: - echo('(%s): ' % metaflow.__version__, - fg='magenta', - bold=False, - nl=False) + echo("(%s): " % metaflow.__version__, fg="magenta", bold=False, nl=False) else: - echo('(%s)\n' % metaflow.__version__, - fg='magenta', - bold=False) + echo("(%s)\n" % metaflow.__version__, fg="magenta", bold=False) if ctx.invoked_subcommand is None: - echo("More data science, less engineering\n", - fg='magenta') + echo("More data science, less engineering\n", fg="magenta") # metaflow URL - echo('http://docs.metaflow.org', fg='cyan', nl=False) - echo(' - Read the documentation') + echo("http://docs.metaflow.org", fg="cyan", nl=False) + echo(" - Read the documentation") # metaflow chat - echo('http://chat.metaflow.org', fg='cyan', nl=False) - echo(' - Chat with us') + echo("http://chat.metaflow.org", fg="cyan", nl=False) + echo(" - Chat with us") # metaflow help email - echo('help@metaflow.org', fg='cyan', nl=False) - echo(' - Get help by email\n') + echo("help@metaflow.org", fg="cyan", nl=False) + echo(" - Get help by email\n") # print a short list of next steps. - short_help = {'tutorials': 'Browse and access metaflow tutorials.', - 'configure': 'Configure metaflow to access the cloud.', - 'status': 'Display the current working tree.', - 'help': 'Show all available commands to run.'} + short_help = { + "tutorials": "Browse and access metaflow tutorials.", + "configure": "Configure metaflow to access the cloud.", + "status": "Display the current working tree.", + "help": "Show all available commands to run.", + } - echo('Commands:', bold=False) + echo("Commands:", bold=False) for cmd, desc in short_help.items(): - echo(' metaflow {0:<10} '.format(cmd), - fg='cyan', - bold=False, - nl=False) + echo(" metaflow {0:<10} ".format(cmd), fg="cyan", bold=False, nl=False) - echo('%s' % desc) + echo("%s" % desc) -@main.command(help='Show all available commands.') + +@main.command(help="Show all available commands.") @click.pass_context def help(ctx): print(ctx.parent.get_help()) -@main.command(help='Show flows accessible from the current working tree.') + +@main.command(help="Show flows accessible from the current working tree.") def status(): from metaflow.client import get_metadata + res = get_metadata() if res: - res = res.split('@') + res = res.split("@") else: - raise click.ClickException('Unknown status: cannot find a Metadata provider') - if res[0] == 'service': - echo('Using Metadata provider at: ', nl=False) - echo('"%s"\n' % res[1], fg='cyan') - echo('To list available flows, type:\n') - echo('1. python') - echo('2. from metaflow import Metaflow') - echo('3. list(Metaflow())') + raise click.ClickException("Unknown status: cannot find a Metadata provider") + if res[0] == "service": + echo("Using Metadata provider at: ", nl=False) + echo('"%s"\n' % res[1], fg="cyan") + echo("To list available flows, type:\n") + echo("1. python") + echo("2. from metaflow import Metaflow") + echo("3. list(Metaflow())") return from metaflow.client import namespace, metadata, Metaflow # Get the local data store path - path = LocalStorage.get_datastore_root_from_config( - echo, create_on_absent=False) + path = LocalStorage.get_datastore_root_from_config(echo, create_on_absent=False) # Throw an exception if path is None: - raise click.ClickException("Could not find " +\ - click.style('"%s"' % DATASTORE_LOCAL_DIR, - fg='red') +\ - " in the current working tree.") + raise click.ClickException( + "Could not find " + + click.style('"%s"' % DATASTORE_LOCAL_DIR, fg="red") + + " in the current working tree." + ) stripped_path = os.path.dirname(path) namespace(None) - metadata('local@%s' % stripped_path) - echo('Working tree found at: ', nl=False) - echo('"%s"\n' % stripped_path, fg='cyan') - echo('Available flows:', fg='cyan', bold=True) + metadata("local@%s" % stripped_path) + echo("Working tree found at: ", nl=False) + echo('"%s"\n' % stripped_path, fg="cyan") + echo("Available flows:", fg="cyan", bold=True) for flow in Metaflow(): - echo('* %s' % flow, fg='cyan') + echo("* %s" % flow, fg="cyan") + @main.group(help="Browse and access the metaflow tutorial episodes.") def tutorials(): pass + def get_tutorials_dir(): metaflow_dir = os.path.dirname(__file__) package_dir = os.path.dirname(metaflow_dir) - tutorials_dir = os.path.join(package_dir, 'metaflow', 'tutorials') + tutorials_dir = os.path.join(package_dir, "metaflow", "tutorials") return tutorials_dir + def get_tutorial_metadata(tutorial_path): metadata = {} - with open(os.path.join(tutorial_path, 'README.md')) as readme: - content = readme.read() - - paragraphs = [paragraph.strip() \ - for paragraph \ - in content.split('#') if paragraph] - metadata['description'] = paragraphs[0].split('**')[1] - header = paragraphs[0].split('\n') - header = header[0].split(':') - metadata['episode'] = header[0].strip()[len('Episode '):] - metadata['title'] = header[1].strip() + with open(os.path.join(tutorial_path, "README.md")) as readme: + content = readme.read() + + paragraphs = [paragraph.strip() for paragraph in content.split("#") if paragraph] + metadata["description"] = paragraphs[0].split("**")[1] + header = paragraphs[0].split("\n") + header = header[0].split(":") + metadata["episode"] = header[0].strip()[len("Episode ") :] + metadata["title"] = header[1].strip() for paragraph in paragraphs[1:]: - if paragraph.startswith('Before playing'): - lines = '\n'.join(paragraph.split('\n')[1:]) - metadata['prereq'] = lines.replace('```', '') + if paragraph.startswith("Before playing"): + lines = "\n".join(paragraph.split("\n")[1:]) + metadata["prereq"] = lines.replace("```", "") - if paragraph.startswith('Showcasing'): - lines = '\n'.join(paragraph.split('\n')[1:]) - metadata['showcase'] = lines.replace('```', '') + if paragraph.startswith("Showcasing"): + lines = "\n".join(paragraph.split("\n")[1:]) + metadata["showcase"] = lines.replace("```", "") - if paragraph.startswith('To play'): - lines = '\n'.join(paragraph.split('\n')[1:]) - metadata['play'] = lines.replace('```', '') + if paragraph.startswith("To play"): + lines = "\n".join(paragraph.split("\n")[1:]) + metadata["play"] = lines.replace("```", "") return metadata + def get_all_episodes(): episodes = [] for name in sorted(os.listdir(get_tutorials_dir())): # Skip hidden files (like .gitignore) - if not name.startswith('.'): + if not name.startswith("."): episodes.append(name) return episodes + @tutorials.command(help="List the available episodes.") def list(): - echo('Episodes:', fg='cyan', bold=True) + echo("Episodes:", fg="cyan", bold=True) for name in get_all_episodes(): path = os.path.join(get_tutorials_dir(), name) metadata = get_tutorial_metadata(path) - echo('* {0: <20} '.format(metadata['episode']), - fg='cyan', - nl=False) - echo('- {0}'.format(metadata['title'])) + echo("* {0: <20} ".format(metadata["episode"]), fg="cyan", nl=False) + echo("- {0}".format(metadata["title"])) + + echo("\nTo pull the episodes, type: ") + echo("metaflow tutorials pull", fg="cyan") - echo('\nTo pull the episodes, type: ') - echo('metaflow tutorials pull', fg='cyan') def validate_episode(episode): src_dir = os.path.join(get_tutorials_dir(), episode) if not os.path.isdir(src_dir): - raise click.BadArgumentUsage("Episode " + \ - click.style("\"{0}\"".format(episode), - fg='red') + " does not exist."\ - " To see a list of available episodes, "\ - "type:\n" + \ - click.style("metaflow tutorials list", - fg='cyan')) + raise click.BadArgumentUsage( + "Episode " + + click.style('"{0}"'.format(episode), fg="red") + + " does not exist." + " To see a list of available episodes, " + "type:\n" + click.style("metaflow tutorials list", fg="cyan") + ) + def autocomplete_episodes(ctx, args, incomplete): return [k for k in get_all_episodes() if incomplete in k] -@tutorials.command(help="Pull episodes "\ - "into your current working directory.") -@click.option('--episode', default="", help="Optional episode name "\ - "to pull only a single episode.") + +@tutorials.command(help="Pull episodes " "into your current working directory.") +@click.option( + "--episode", + default="", + help="Optional episode name " "to pull only a single episode.", +) def pull(episode): tutorials_dir = get_tutorials_dir() if not episode: @@ -214,7 +217,7 @@ def pull(episode): for episode in episodes: validate_episode(episode) # Create destination `metaflow-tutorials` dir. - dst_parent = os.path.join(os.getcwd(), 'metaflow-tutorials') + dst_parent = os.path.join(os.getcwd(), "metaflow-tutorials") makedirs(dst_parent) # Pull specified episodes. @@ -222,67 +225,83 @@ def pull(episode): dst_dir = os.path.join(dst_parent, episode) # Check if episode has already been pulled before. if os.path.exists(dst_dir): - if click.confirm("Episode " + \ - click.style("\"{0}\"".format(episode), fg='red') +\ - " has already been pulled before. Do you wish "\ - "to delete the existing version?"): + if click.confirm( + "Episode " + + click.style('"{0}"'.format(episode), fg="red") + + " has already been pulled before. Do you wish " + "to delete the existing version?" + ): shutil.rmtree(dst_dir) else: continue - echo('Pulling episode ', nl=False) - echo('\"{0}\"'.format(episode), fg='cyan', nl=False) + echo("Pulling episode ", nl=False) + echo('"{0}"'.format(episode), fg="cyan", nl=False) # TODO: Is the following redundant? - echo(' into your current working directory.') + echo(" into your current working directory.") # Copy from (local) metaflow package dir to current. src_dir = os.path.join(tutorials_dir, episode) shutil.copytree(src_dir, dst_dir) - echo('\nTo know more about an episode, type:\n', nl=False) - echo('metaflow tutorials info [EPISODE]', fg='cyan') + echo("\nTo know more about an episode, type:\n", nl=False) + echo("metaflow tutorials info [EPISODE]", fg="cyan") + -@tutorials.command(help='Find out more about an episode.') -@click.argument('episode', autocompletion=autocomplete_episodes) +@tutorials.command(help="Find out more about an episode.") +@click.argument("episode", autocompletion=autocomplete_episodes) def info(episode): validate_episode(episode) src_dir = os.path.join(get_tutorials_dir(), episode) metadata = get_tutorial_metadata(src_dir) - echo('Synopsis:', fg='cyan', bold=True) - echo('%s' % metadata['description']) + echo("Synopsis:", fg="cyan", bold=True) + echo("%s" % metadata["description"]) - echo('\nShowcasing:', fg='cyan', bold=True, nl=True) - echo('%s' % metadata['showcase']) + echo("\nShowcasing:", fg="cyan", bold=True, nl=True) + echo("%s" % metadata["showcase"]) - if 'prereq' in metadata: - echo('\nBefore playing:', fg='cyan', bold=True, nl=True) - echo('%s' % metadata['prereq']) + if "prereq" in metadata: + echo("\nBefore playing:", fg="cyan", bold=True, nl=True) + echo("%s" % metadata["prereq"]) + + echo("\nTo play:", fg="cyan", bold=True) + echo("%s" % metadata["play"]) - echo('\nTo play:', fg='cyan', bold=True) - echo('%s' % metadata['play']) # NOTE: This code needs to be in sync with metaflow/metaflow_config.py. -METAFLOW_CONFIGURATION_DIR =\ - expanduser(os.environ.get('METAFLOW_HOME', '~/.metaflowconfig')) +METAFLOW_CONFIGURATION_DIR = expanduser( + os.environ.get("METAFLOW_HOME", "~/.metaflowconfig") +) + @main.group(help="Configure Metaflow to access the cloud.") def configure(): makedirs(METAFLOW_CONFIGURATION_DIR) + def get_config_path(profile): - config_file = 'config.json' if not profile else ('config_%s.json' % profile) + config_file = "config.json" if not profile else ("config_%s.json" % profile) path = os.path.join(METAFLOW_CONFIGURATION_DIR, config_file) return path + def overwrite_config(profile): path = get_config_path(profile) if os.path.exists(path): if not click.confirm( - click.style('We found an existing configuration for your ' + - 'profile. Do you want to modify the existing ' + - 'configuration?', fg='red', bold=True)): - echo('You can configure a different named profile by using the ' - '--profile argument. You can activate this profile by setting ' - 'the environment variable METAFLOW_PROFILE to the named ' - 'profile.', fg='yellow') + click.style( + "We found an existing configuration for your " + + "profile. Do you want to modify the existing " + + "configuration?", + fg="red", + bold=True, + ) + ): + echo( + "You can configure a different named profile by using the " + "--profile argument. You can activate this profile by setting " + "the environment variable METAFLOW_PROFILE to the named " + "profile.", + fg="yellow", + ) return False return True @@ -291,10 +310,13 @@ def check_for_missing_profile(profile): path = get_config_path(profile) # Absence of default config is equivalent to running locally. if profile and not os.path.exists(path): - raise click.ClickException("Couldn't find configuration for profile " + - click.style('"%s"' % profile, fg='red') + - " in " + - click.style('"%s"' % path, fg='red')) + raise click.ClickException( + "Couldn't find configuration for profile " + + click.style('"%s"' % profile, fg="red") + + " in " + + click.style('"%s"' % path, fg="red") + ) + def get_env(profile): path = get_config_path(profile) @@ -303,154 +325,182 @@ def get_env(profile): return json.load(f) return {} + def persist_env(env_dict, profile): # TODO: Should we persist empty env_dict or notify user differently? path = get_config_path(profile) - with open(path, 'w') as f: + with open(path, "w") as f: json.dump(env_dict, f, indent=4, sort_keys=True) - echo('\nConfiguration successfully written to ', nl=False, bold=True) - echo('"%s"' % path, fg='cyan') + echo("\nConfiguration successfully written to ", nl=False, bold=True) + echo('"%s"' % path, fg="cyan") + -@configure.command(help='Reset configuration to disable cloud access.') -@click.option('--profile', '-p', default='', - help="Optional named profile.") +@configure.command(help="Reset configuration to disable cloud access.") +@click.option("--profile", "-p", default="", help="Optional named profile.") def reset(profile): check_for_missing_profile(profile) path = get_config_path(profile) if os.path.exists(path): - if click.confirm('Do you really wish to reset the configuration in ' +\ - click.style('"%s"' % path, fg='cyan'), abort=True): + if click.confirm( + "Do you really wish to reset the configuration in " + + click.style('"%s"' % path, fg="cyan"), + abort=True, + ): os.remove(path) - echo('Configuration successfully reset to run locally.') + echo("Configuration successfully reset to run locally.") else: - echo('Configuration is already reset to run locally.') + echo("Configuration is already reset to run locally.") -@configure.command(help='Show existing configuration.') -@click.option('--profile', '-p', default='', - help="Optional named profile.") + +@configure.command(help="Show existing configuration.") +@click.option("--profile", "-p", default="", help="Optional named profile.") def show(profile): check_for_missing_profile(profile) path = get_config_path(profile) env_dict = {} if os.path.exists(path): - with open(path, 'r') as f: + with open(path, "r") as f: env_dict = json.load(f) if env_dict: - echo('Showing configuration in ', nl=False) - echo('"%s"\n' % path, fg='cyan') - for k,v in env_dict.items(): - echo('%s=%s' % (k, v)) + echo("Showing configuration in ", nl=False) + echo('"%s"\n' % path, fg="cyan") + for k, v in env_dict.items(): + echo("%s=%s" % (k, v)) else: - echo('Configuration is set to run locally.') + echo("Configuration is set to run locally.") + -@configure.command(help='Export configuration to a file.') -@click.option('--profile', '-p', default='', - help="Optional named profile whose configuration must be " - "exported.") -@click.argument('output_filename', type=click.Path(resolve_path=True)) +@configure.command(help="Export configuration to a file.") +@click.option( + "--profile", + "-p", + default="", + help="Optional named profile whose configuration must be " "exported.", +) +@click.argument("output_filename", type=click.Path(resolve_path=True)) def export(profile, output_filename): check_for_missing_profile(profile) # Export its contents to a new file. path = get_config_path(profile) env_dict = {} if os.path.exists(path): - with open(path, 'r') as f: + with open(path, "r") as f: env_dict = json.load(f) # resolve_path doesn't expand `~` in `path`. output_path = expanduser(output_filename) if os.path.exists(output_path): - if click.confirm('Do you wish to overwrite the contents in ' + - click.style('"%s"' % output_path, fg='cyan') + '?', - abort=True): + if click.confirm( + "Do you wish to overwrite the contents in " + + click.style('"%s"' % output_path, fg="cyan") + + "?", + abort=True, + ): pass # Write to file. - with open(output_path, 'w') as f: + with open(output_path, "w") as f: json.dump(env_dict, f, indent=4, sort_keys=True) - echo('Configuration successfully exported to: ', nl=False) - echo('"%s"' % output_path, fg='cyan') - -@configure.command(help='Import configuration from a file.', name='import') -@click.option('--profile', '-p', default='', - help="Optional named profile to which the configuration must be " - "imported into.") -@click.argument('input_filename', type=click.Path(exists=True, - resolve_path=True)) + echo("Configuration successfully exported to: ", nl=False) + echo('"%s"' % output_path, fg="cyan") + + +@configure.command(help="Import configuration from a file.", name="import") +@click.option( + "--profile", + "-p", + default="", + help="Optional named profile to which the configuration must be " "imported into.", +) +@click.argument("input_filename", type=click.Path(exists=True, resolve_path=True)) def import_from(profile, input_filename): check_for_missing_profile(profile) # Import configuration. input_path = expanduser(input_filename) env_dict = {} - with open(input_path, 'r') as f: + with open(input_path, "r") as f: env_dict = json.load(f) - echo('Configuration successfully read from: ', nl=False) - echo('"%s"' % input_path, fg='cyan') + echo("Configuration successfully read from: ", nl=False) + echo('"%s"' % input_path, fg="cyan") # Persist configuration. overwrite_config(profile) persist_env(env_dict, profile) -@configure.command(help='Configure metaflow to access hosted sandbox.') -@click.option('--profile', '-p', default='', - help='Configure a named profile. Activate the profile by setting ' - '`METAFLOW_PROFILE` environment variable.') + +@configure.command(help="Configure metaflow to access hosted sandbox.") +@click.option( + "--profile", + "-p", + default="", + help="Configure a named profile. Activate the profile by setting " + "`METAFLOW_PROFILE` environment variable.", +) def sandbox(profile): overwrite_config(profile) # Prompt for user input. - encoded_str = click.prompt('Following instructions from ' - 'https://metaflow.org/sandbox, ' - 'please paste the encoded magic string', - type=str) + encoded_str = click.prompt( + "Following instructions from " + "https://metaflow.org/sandbox, " + "please paste the encoded magic string", + type=str, + ) # Decode the bytes to env_dict. try: import base64, zlib from metaflow.util import to_bytes - env_dict =\ - json.loads(to_unicode(zlib.decompress(base64.b64decode(to_bytes(encoded_str))))) + + env_dict = json.loads( + to_unicode(zlib.decompress(base64.b64decode(to_bytes(encoded_str)))) + ) except: # TODO: Add the URL for contact us page in the error? - raise click.BadArgumentUsage('Could not decode the sandbox '\ - 'configuration. Please contact us.') + raise click.BadArgumentUsage( + "Could not decode the sandbox " "configuration. Please contact us." + ) # Persist to a file. persist_env(env_dict, profile) def cyan(string): - return click.style(string, fg='cyan') + return click.style(string, fg="cyan") + def yellow(string): - return click.style(string, fg='yellow') + return click.style(string, fg="yellow") + def red(string): - return click.style(string, fg='red') + return click.style(string, fg="red") + def configure_s3_datastore(existing_env): env = {} # Set Amazon S3 as default datastore. - env['METAFLOW_DEFAULT_DATASTORE'] = 's3' + env["METAFLOW_DEFAULT_DATASTORE"] = "s3" # Set Amazon S3 folder for datastore. - env['METAFLOW_DATASTORE_SYSROOT_S3'] =\ - click.prompt(cyan('[METAFLOW_DATASTORE_SYSROOT_S3]') + - ' Amazon S3 folder for Metaflow artifact storage ' + - '(s3:///).', - default=\ - existing_env.get('METAFLOW_DATASTORE_SYSROOT_S3'), - show_default=True) + env["METAFLOW_DATASTORE_SYSROOT_S3"] = click.prompt( + cyan("[METAFLOW_DATASTORE_SYSROOT_S3]") + + " Amazon S3 folder for Metaflow artifact storage " + + "(s3:///).", + default=existing_env.get("METAFLOW_DATASTORE_SYSROOT_S3"), + show_default=True, + ) # Set Amazon S3 folder for datatools. - env['METAFLOW_DATATOOLS_SYSROOT_S3'] =\ - click.prompt(cyan('[METAFLOW_DATATOOLS_SYSROOT_S3]') + - yellow(' (optional)') + - ' Amazon S3 folder for Metaflow datatools ' + - '(s3:///).', - default=\ - existing_env.get('METAFLOW_DATATOOLS_SYSROOT_S3', - os.path.join( - env['METAFLOW_DATASTORE_SYSROOT_S3'], - 'data')), - show_default=True) + env["METAFLOW_DATATOOLS_SYSROOT_S3"] = click.prompt( + cyan("[METAFLOW_DATATOOLS_SYSROOT_S3]") + + yellow(" (optional)") + + " Amazon S3 folder for Metaflow datatools " + + "(s3:///).", + default=existing_env.get( + "METAFLOW_DATATOOLS_SYSROOT_S3", + os.path.join(env["METAFLOW_DATASTORE_SYSROOT_S3"], "data"), + ), + show_default=True, + ) return env + def configure_metadata_service(existing_env): empty_profile = False if not existing_env: @@ -458,31 +508,32 @@ def configure_metadata_service(existing_env): env = {} # Set Metadata Service as default. - env['METAFLOW_DEFAULT_METADATA'] = 'service' + env["METAFLOW_DEFAULT_METADATA"] = "service" # Set URL for the Metadata Service. - env['METAFLOW_SERVICE_URL'] =\ - click.prompt(cyan('[METAFLOW_SERVICE_URL]') + - ' URL for Metaflow Service.', - default=existing_env.get('METAFLOW_SERVICE_URL'), - show_default=True) + env["METAFLOW_SERVICE_URL"] = click.prompt( + cyan("[METAFLOW_SERVICE_URL]") + " URL for Metaflow Service.", + default=existing_env.get("METAFLOW_SERVICE_URL"), + show_default=True, + ) # Set internal URL for the Metadata Service. - env['METAFLOW_SERVICE_INTERNAL_URL'] =\ - click.prompt(cyan('[METAFLOW_SERVICE_INTERNAL_URL]') + - yellow(' (optional)') + - ' URL for Metaflow Service ' + - '(Accessible only within VPC).', - default=\ - existing_env.get('METAFLOW_SERVICE_INTERNAL_URL', - env['METAFLOW_SERVICE_URL']), - show_default=True) + env["METAFLOW_SERVICE_INTERNAL_URL"] = click.prompt( + cyan("[METAFLOW_SERVICE_INTERNAL_URL]") + + yellow(" (optional)") + + " URL for Metaflow Service " + + "(Accessible only within VPC).", + default=existing_env.get( + "METAFLOW_SERVICE_INTERNAL_URL", env["METAFLOW_SERVICE_URL"] + ), + show_default=True, + ) # Set Auth Key for the Metadata Service. - env['METAFLOW_SERVICE_AUTH_KEY'] =\ - click.prompt(cyan('[METAFLOW_SERVICE_AUTH_KEY]') + - yellow(' (optional)') + - ' Auth Key for Metaflow Service.', - default=\ - existing_env.get('METAFLOW_SERVICE_AUTH_KEY', ''), - show_default=True) + env["METAFLOW_SERVICE_AUTH_KEY"] = click.prompt( + cyan("[METAFLOW_SERVICE_AUTH_KEY]") + + yellow(" (optional)") + + " Auth Key for Metaflow Service.", + default=existing_env.get("METAFLOW_SERVICE_AUTH_KEY", ""), + show_default=True, + ) return env @@ -493,33 +544,35 @@ def configure_datastore_and_metadata(existing_env): env = {} # Configure Amazon S3 as the datastore. - use_s3_as_datastore = click.confirm('\nMetaflow can use ' + - yellow('Amazon S3 as the storage backend') + - ' for all code and data artifacts on ' + - 'AWS.\nAmazon S3 is a strict requirement if you ' + - 'intend to execute your flows on AWS Batch ' + - 'and/or schedule them on AWS Step ' + - 'Functions.\nWould you like to configure Amazon ' + - 'S3 as the default storage backend?', - default=empty_profile or \ - existing_env.get( - 'METAFLOW_DEFAULT_DATASTORE', '') == 's3', - abort=False) + use_s3_as_datastore = click.confirm( + "\nMetaflow can use " + + yellow("Amazon S3 as the storage backend") + + " for all code and data artifacts on " + + "AWS.\nAmazon S3 is a strict requirement if you " + + "intend to execute your flows on AWS Batch " + + "and/or schedule them on AWS Step " + + "Functions.\nWould you like to configure Amazon " + + "S3 as the default storage backend?", + default=empty_profile + or existing_env.get("METAFLOW_DEFAULT_DATASTORE", "") == "s3", + abort=False, + ) if use_s3_as_datastore: env.update(configure_s3_datastore(existing_env)) # Configure Metadata service for tracking. - if click.confirm('\nMetaflow can use a ' + - yellow('remote Metadata Service to track') + - ' and persist flow execution metadata.\nConfiguring the ' - 'service is a requirement if you intend to schedule your ' - 'flows with AWS Step Functions.\nWould you like to ' - 'configure the Metadata Service?', - default=empty_profile or\ - existing_env.get('METAFLOW_DEFAULT_METADATA', '') ==\ - 'service' or\ - 'METAFLOW_SFN_IAM_ROLE' in env, - abort=False): + if click.confirm( + "\nMetaflow can use a " + + yellow("remote Metadata Service to track") + + " and persist flow execution metadata.\nConfiguring the " + "service is a requirement if you intend to schedule your " + "flows with AWS Step Functions.\nWould you like to " + "configure the Metadata Service?", + default=empty_profile + or existing_env.get("METAFLOW_DEFAULT_METADATA", "") == "service" + or "METAFLOW_SFN_IAM_ROLE" in env, + abort=False, + ): env.update(configure_metadata_service(existing_env)) return env @@ -530,81 +583,79 @@ def configure_aws_batch(existing_env): empty_profile = True env = {} - # Set AWS Batch Job Queue. - env['METAFLOW_BATCH_JOB_QUEUE'] =\ - click.prompt(cyan('[METAFLOW_BATCH_JOB_QUEUE]') + - ' AWS Batch Job Queue.', - default=\ - existing_env.get('METAFLOW_BATCH_JOB_QUEUE'), - show_default=True) + env["METAFLOW_BATCH_JOB_QUEUE"] = click.prompt( + cyan("[METAFLOW_BATCH_JOB_QUEUE]") + " AWS Batch Job Queue.", + default=existing_env.get("METAFLOW_BATCH_JOB_QUEUE"), + show_default=True, + ) # Set IAM role for AWS Batch jobs to assume. - env['METAFLOW_ECS_S3_ACCESS_IAM_ROLE'] =\ - click.prompt(cyan('[METAFLOW_ECS_S3_ACCESS_IAM_ROLE]') + - ' IAM role for AWS Batch jobs to access AWS ' + - 'resources (Amazon S3 etc.).', - default=\ - existing_env.get('METAFLOW_ECS_S3_ACCESS_IAM_ROLE'), - show_default=True) + env["METAFLOW_ECS_S3_ACCESS_IAM_ROLE"] = click.prompt( + cyan("[METAFLOW_ECS_S3_ACCESS_IAM_ROLE]") + + " IAM role for AWS Batch jobs to access AWS " + + "resources (Amazon S3 etc.).", + default=existing_env.get("METAFLOW_ECS_S3_ACCESS_IAM_ROLE"), + show_default=True, + ) # Set default Docker repository for AWS Batch jobs. - env['METAFLOW_BATCH_CONTAINER_REGISTRY'] =\ - click.prompt(cyan('[METAFLOW_BATCH_CONTAINER_REGISTRY]') + - yellow(' (optional)') + - ' Default Docker image repository for AWS ' + - 'Batch jobs. If nothing is specified, ' + - 'dockerhub (hub.docker.com/) is ' + - 'used as default.', - default=\ - existing_env.get('METAFLOW_BATCH_CONTAINER_REGISTRY', ''), - show_default=True) + env["METAFLOW_BATCH_CONTAINER_REGISTRY"] = click.prompt( + cyan("[METAFLOW_BATCH_CONTAINER_REGISTRY]") + + yellow(" (optional)") + + " Default Docker image repository for AWS " + + "Batch jobs. If nothing is specified, " + + "dockerhub (hub.docker.com/) is " + + "used as default.", + default=existing_env.get("METAFLOW_BATCH_CONTAINER_REGISTRY", ""), + show_default=True, + ) # Set default Docker image for AWS Batch jobs. - env['METAFLOW_BATCH_CONTAINER_IMAGE'] =\ - click.prompt(cyan('[METAFLOW_BATCH_CONTAINER_IMAGE]') + - yellow(' (optional)') + - ' Default Docker image for AWS Batch jobs. ' + - 'If nothing is specified, an appropriate ' + - 'python image is used as default.', - default=\ - existing_env.get('METAFLOW_BATCH_CONTAINER_IMAGE', ''), - show_default=True) + env["METAFLOW_BATCH_CONTAINER_IMAGE"] = click.prompt( + cyan("[METAFLOW_BATCH_CONTAINER_IMAGE]") + + yellow(" (optional)") + + " Default Docker image for AWS Batch jobs. " + + "If nothing is specified, an appropriate " + + "python image is used as default.", + default=existing_env.get("METAFLOW_BATCH_CONTAINER_IMAGE", ""), + show_default=True, + ) # Configure AWS Step Functions for scheduling. - if click.confirm('\nMetaflow can ' + - yellow('schedule your flows on AWS Step ' - 'Functions') + - ' and trigger them at a specific cadence using ' - 'Amazon EventBridge.\nTo support flows involving ' - 'foreach steps, you would need access to AWS ' - 'DynamoDB.\nWould you like to configure AWS Step ' - 'Functions for scheduling?', - default=empty_profile or - 'METAFLOW_SFN_IAM_ROLE' in existing_env, - abort=False): + if click.confirm( + "\nMetaflow can " + + yellow("schedule your flows on AWS Step " "Functions") + + " and trigger them at a specific cadence using " + "Amazon EventBridge.\nTo support flows involving " + "foreach steps, you would need access to AWS " + "DynamoDB.\nWould you like to configure AWS Step " + "Functions for scheduling?", + default=empty_profile or "METAFLOW_SFN_IAM_ROLE" in existing_env, + abort=False, + ): # Configure IAM role for AWS Step Functions. - env['METAFLOW_SFN_IAM_ROLE'] =\ - click.prompt(cyan('[METAFLOW_SFN_IAM_ROLE]') + - ' IAM role for AWS Step Functions to ' + - 'access AWS resources (AWS Batch, ' + - 'AWS DynamoDB).', - default=\ - existing_env.get('METAFLOW_SFN_IAM_ROLE'), - show_default=True) + env["METAFLOW_SFN_IAM_ROLE"] = click.prompt( + cyan("[METAFLOW_SFN_IAM_ROLE]") + + " IAM role for AWS Step Functions to " + + "access AWS resources (AWS Batch, " + + "AWS DynamoDB).", + default=existing_env.get("METAFLOW_SFN_IAM_ROLE"), + show_default=True, + ) # Configure IAM role for AWS Events Bridge. - env['METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE'] =\ - click.prompt(cyan('[METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE]') + - ' IAM role for Amazon EventBridge to ' + - 'access AWS Step Functions.', - default=\ - existing_env.get('METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE'), - show_default=True) + env["METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE"] = click.prompt( + cyan("[METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE]") + + " IAM role for Amazon EventBridge to " + + "access AWS Step Functions.", + default=existing_env.get("METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE"), + show_default=True, + ) # Configure AWS DynamoDB Table for AWS Step Functions. - env['METAFLOW_SFN_DYNAMO_DB_TABLE'] =\ - click.prompt(cyan('[METAFLOW_SFN_DYNAMO_DB_TABLE]') + - ' AWS DynamoDB table name for tracking '+ - 'AWS Step Functions execution metadata.', - default=\ - existing_env.get('METAFLOW_SFN_DYNAMO_DB_TABLE'), - show_default=True) + env["METAFLOW_SFN_DYNAMO_DB_TABLE"] = click.prompt( + cyan("[METAFLOW_SFN_DYNAMO_DB_TABLE]") + + " AWS DynamoDB table name for tracking " + + "AWS Step Functions execution metadata.", + default=existing_env.get("METAFLOW_SFN_DYNAMO_DB_TABLE"), + show_default=True, + ) return env @@ -612,32 +663,42 @@ def check_kubernetes_client(ctx): try: import kubernetes except ImportError: - echo("Please install python kubernetes client first " + \ - "(run " + yellow('pip install kubernetes') + \ - " or equivalent in your favorite python package manager)" + echo( + "Please install python kubernetes client first " + + "(run " + + yellow("pip install kubernetes") + + " or equivalent in your favorite python package manager)" ) ctx.abort() def check_kubernetes_config(ctx): from kubernetes import config + try: all_contexts, current_context = config.list_kube_config_contexts() - click.confirm("You have a valid kubernetes configuration. The current context is set to " + \ - yellow(current_context["name"]) + " " + \ - "Proceed?", + click.confirm( + "You have a valid kubernetes configuration. The current context is set to " + + yellow(current_context["name"]) + + " " + + "Proceed?", default=True, - abort=True + abort=True, ) except config.config_exception.ConfigException as e: - click.confirm("\nYou don't seem to have a valid kubernetes configuration file. " + \ - "The error from kubernetes client library: " + \ - red(str(e)) + "." + \ - "To create a kubernetes configuration for EKS, you typically need to run " + yellow("aws eks update-kubeconfig --name ") + \ - ". For further details, refer to AWS Documentation at https://docs.aws.amazon.com/eks/latest/userguide/create-kubeconfig.html\n" - 'Do you want to proceed with configuring Metaflow for EKS anyway?', - default=False, - abort=True) + click.confirm( + "\nYou don't seem to have a valid kubernetes configuration file. " + + "The error from kubernetes client library: " + + red(str(e)) + + "." + + "To create a kubernetes configuration for EKS, you typically need to run " + + yellow("aws eks update-kubeconfig --name ") + + ". For further details, refer to AWS Documentation at https://docs.aws.amazon.com/eks/latest/userguide/create-kubeconfig.html\n" + "Do you want to proceed with configuring Metaflow for EKS anyway?", + default=False, + abort=True, + ) + def configure_eks(existing_env): empty_profile = False @@ -646,75 +707,88 @@ def configure_eks(existing_env): env = {} # Set K8S Namespace - env['METAFLOW_KUBERNETES_NAMESPACE'] =\ - click.prompt(cyan('[METAFLOW_KUBERNETES_NAMESPACE]') + - yellow(' (optional)') + - ' Kubernetes Namespace ', - default="default", - show_default=True) + env["METAFLOW_KUBERNETES_NAMESPACE"] = click.prompt( + cyan("[METAFLOW_KUBERNETES_NAMESPACE]") + + yellow(" (optional)") + + " Kubernetes Namespace ", + default="default", + show_default=True, + ) # Set K8S SA - env['METAFLOW_KUBERNETES_SERVICE_ACCOUNT'] =\ - click.prompt(cyan('[METAFLOW_KUBERNETES_SERVICE_ACCOUNT]') + - yellow(' (optional)') + - ' Kubernetes Service Account ', - default="default", - show_default=True) + env["METAFLOW_KUBERNETES_SERVICE_ACCOUNT"] = click.prompt( + cyan("[METAFLOW_KUBERNETES_SERVICE_ACCOUNT]") + + yellow(" (optional)") + + " Kubernetes Service Account ", + default="default", + show_default=True, + ) # Set default Docker repository for K8S jobs. - env['METAFLOW_KUBERNETES_CONTAINER_REGISTRY'] =\ - click.prompt(cyan('[METAFLOW_KUBERNETES_CONTAINER_REGISTRY]') + - yellow(' (optional)') + - ' Default Docker image repository for K8S ' + - 'jobs. If nothing is specified, ' + - 'dockerhub (hub.docker.com/) is ' + - 'used as default.', - default=\ - existing_env.get('METAFLOW_KUBERNETES_CONTAINER_REGISTRY', ''), - show_default=True) + env["METAFLOW_KUBERNETES_CONTAINER_REGISTRY"] = click.prompt( + cyan("[METAFLOW_KUBERNETES_CONTAINER_REGISTRY]") + + yellow(" (optional)") + + " Default Docker image repository for K8S " + + "jobs. If nothing is specified, " + + "dockerhub (hub.docker.com/) is " + + "used as default.", + default=existing_env.get("METAFLOW_KUBERNETES_CONTAINER_REGISTRY", ""), + show_default=True, + ) # Set default Docker image for K8S jobs. - env['METAFLOW_KUBERNETES_CONTAINER_IMAGE'] =\ - click.prompt(cyan('[METAFLOW_KUBERNETES_CONTAINER_IMAGE]') + - yellow(' (optional)') + - ' Default Docker image for K8S jobs. ' + - 'If nothing is specified, an appropriate ' + - 'python image is used as default.', - default=\ - existing_env.get('METAFLOW_KUBERNETES_CONTAINER_IMAGE', ''), - show_default=True) + env["METAFLOW_KUBERNETES_CONTAINER_IMAGE"] = click.prompt( + cyan("[METAFLOW_KUBERNETES_CONTAINER_IMAGE]") + + yellow(" (optional)") + + " Default Docker image for K8S jobs. " + + "If nothing is specified, an appropriate " + + "python image is used as default.", + default=existing_env.get("METAFLOW_KUBERNETES_CONTAINER_IMAGE", ""), + show_default=True, + ) return env def verify_aws_credentials(ctx): # Verify that the user has configured AWS credentials on their computer. - if not click.confirm('\nMetaflow relies on ' + - yellow('AWS access credentials') + - ' present on your computer to access resources on AWS.' - '\nBefore proceeding further, please confirm that you ' - 'have already configured these access credentials on ' - 'this computer.', - default=True): - echo('There are many ways to setup your AWS access credentials. You ' - 'can get started by following this guide: ', - nl=False, - fg='yellow') - echo('https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html', - fg='cyan') + if not click.confirm( + "\nMetaflow relies on " + + yellow("AWS access credentials") + + " present on your computer to access resources on AWS." + "\nBefore proceeding further, please confirm that you " + "have already configured these access credentials on " + "this computer.", + default=True, + ): + echo( + "There are many ways to setup your AWS access credentials. You " + "can get started by following this guide: ", + nl=False, + fg="yellow", + ) + echo( + "https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html", + fg="cyan", + ) ctx.abort() -@configure.command(help='Configure metaflow to access self-managed AWS resources.') -@click.option('--profile', '-p', default='', - help='Configure a named profile. Activate the profile by setting ' - '`METAFLOW_PROFILE` environment variable.') +@configure.command(help="Configure metaflow to access self-managed AWS resources.") +@click.option( + "--profile", + "-p", + default="", + help="Configure a named profile. Activate the profile by setting " + "`METAFLOW_PROFILE` environment variable.", +) @click.pass_context def aws(ctx, profile): # Greet the user! - echo('Welcome to Metaflow! Follow the prompts to configure your ' - 'installation.\n', - bold=True) + echo( + "Welcome to Metaflow! Follow the prompts to configure your " "installation.\n", + bold=True, + ) # Check for existing configuration. if not overwrite_config(profile): @@ -731,34 +805,40 @@ def aws(ctx, profile): env.update(configure_datastore_and_metadata(existing_env)) # Configure AWS Batch for compute if using S3 - if env.get('METAFLOW_DEFAULT_DATASTORE') == 's3': - if click.confirm('\nMetaflow can scale your flows by ' + - yellow('executing your steps on AWS Batch') + - '.\nAWS Batch is a strict requirement if you intend ' - 'to schedule your flows on AWS Step Functions.\nWould ' - 'you like to configure AWS Batch as your compute ' - 'backend?', - default=empty_profile or - 'METAFLOW_BATCH_JOB_QUEUE' in existing_env, - abort=False): + if env.get("METAFLOW_DEFAULT_DATASTORE") == "s3": + if click.confirm( + "\nMetaflow can scale your flows by " + + yellow("executing your steps on AWS Batch") + + ".\nAWS Batch is a strict requirement if you intend " + "to schedule your flows on AWS Step Functions.\nWould " + "you like to configure AWS Batch as your compute " + "backend?", + default=empty_profile or "METAFLOW_BATCH_JOB_QUEUE" in existing_env, + abort=False, + ): env.update(configure_aws_batch(existing_env)) persist_env({k: v for k, v in env.items() if v}, profile) -@configure.command(help='Configure metaflow to use AWS EKS.') -@click.option('--profile', '-p', default='', - help='Configure a named profile. Activate the profile by setting ' - '`METAFLOW_PROFILE` environment variable.') +@configure.command(help="Configure metaflow to use AWS EKS.") +@click.option( + "--profile", + "-p", + default="", + help="Configure a named profile. Activate the profile by setting " + "`METAFLOW_PROFILE` environment variable.", +) @click.pass_context def eks(ctx, profile): check_kubernetes_client(ctx) # Greet the user! - echo('Welcome to Metaflow! Follow the prompts to configure your ' - 'installation.\n', - bold=True) + echo( + "Welcome to Metaflow! Follow the prompts to configure your " "installation.\n", + bold=True, + ) check_kubernetes_config(ctx) @@ -772,31 +852,35 @@ def eks(ctx, profile): env = existing_env.copy() - if existing_env.get('METAFLOW_DEFAULT_DATASTORE') == 's3': + if existing_env.get("METAFLOW_DEFAULT_DATASTORE") == "s3": # Skip S3 configuration if it is already configured pass - elif not existing_env.get('METAFLOW_DEFAULT_DATASTORE'): + elif not existing_env.get("METAFLOW_DEFAULT_DATASTORE"): env.update(configure_s3_datastore(existing_env)) else: # If configured to use something else, offer to switch to S3 - click.confirm('\nMetaflow on EKS needs to use S3 as a datastore, ' + - "but your existing configuration is not using S3. " + - 'Would you like to reconfigure it to use S3?', - default=True, - abort=True) + click.confirm( + "\nMetaflow on EKS needs to use S3 as a datastore, " + + "but your existing configuration is not using S3. " + + "Would you like to reconfigure it to use S3?", + default=True, + abort=True, + ) env.update(configure_s3_datastore(existing_env)) # Configure remote metadata. - if existing_env.get('METAFLOW_DEFAULT_METADATA') == 'service': + if existing_env.get("METAFLOW_DEFAULT_METADATA") == "service": # Skip metadata service configuration if it is already configured pass else: - if click.confirm('\nMetaflow can use a ' + - yellow('remote Metadata Service to track') + - ' and persist flow execution metadata. \nWould you like to ' - 'configure the Metadata Service?', - default=True, - abort=False): + if click.confirm( + "\nMetaflow can use a " + + yellow("remote Metadata Service to track") + + " and persist flow execution metadata. \nWould you like to " + "configure the Metadata Service?", + default=True, + abort=False, + ): env.update(configure_metadata_service(existing_env)) # Configure AWS EKS for compute. @@ -804,4 +888,5 @@ def eks(ctx, profile): persist_env({k: v for k, v in env.items() if v}, profile) -main() \ No newline at end of file + +main() diff --git a/metaflow/metadata/__init__.py b/metaflow/metadata/__init__.py index 60d965ca034..7baffa9f79f 100644 --- a/metaflow/metadata/__init__.py +++ b/metaflow/metadata/__init__.py @@ -1 +1 @@ -from .metadata import DataArtifact, MetadataProvider, MetaDatum \ No newline at end of file +from .metadata import DataArtifact, MetadataProvider, MetaDatum diff --git a/metaflow/metadata/heartbeat.py b/metaflow/metadata/heartbeat.py index 28f7af3c8dd..d2f01f2fb4b 100644 --- a/metaflow/metadata/heartbeat.py +++ b/metaflow/metadata/heartbeat.py @@ -7,17 +7,17 @@ from metaflow.metaflow_config import METADATA_SERVICE_HEADERS from metaflow.exception import MetaflowException -HB_URL_KEY = 'hb_url' +HB_URL_KEY = "hb_url" class HeartBeatException(MetaflowException): - headline = 'Metaflow heart beat error' + headline = "Metaflow heart beat error" def __init__(self, msg): super(HeartBeatException, self).__init__(msg) -class MetadataHeartBeat(object): +class MetadataHeartBeat(object): def __init__(self): self.headers = METADATA_SERVICE_HEADERS self.req_thread = Thread(target=self.ping) @@ -30,8 +30,7 @@ def process_message(self, msg): if msg.msg_type == MessageTypes.SHUTDOWN: # todo shutdown doesnt do anything yet? should it still be called self.shutdown() - if (not self.req_thread.is_alive()) and \ - msg.msg_type == MessageTypes.LOG_EVENT: + if (not self.req_thread.is_alive()) and msg.msg_type == MessageTypes.LOG_EVENT: # set post url self.hb_url = msg.payload[HB_URL_KEY] # start thread @@ -50,24 +49,24 @@ def ping(self): retry_counter = 0 except HeartBeatException as e: retry_counter = retry_counter + 1 - time.sleep(4**retry_counter) + time.sleep(4 ** retry_counter) def heartbeat(self): if self.hb_url is not None: - response = \ - requests.post(url=self.hb_url, data='{}', headers=self.headers) + response = requests.post(url=self.hb_url, data="{}", headers=self.headers) # Unfortunately, response.json() returns a string that we need # to cast to json; however when the request encounters an error # the return type is a json blob :/ if response.status_code == 200: - return json.loads(response.json()).get('wait_time_in_seconds') + return json.loads(response.json()).get("wait_time_in_seconds") else: - raise HeartBeatException('HeartBeat request (%s) failed' - ' (code %s): %s' % - (self.hb_url, response.status_code, - response.text)) + raise HeartBeatException( + "HeartBeat request (%s) failed" + " (code %s): %s" + % (self.hb_url, response.status_code, response.text) + ) return None def shutdown(self): # attempts sending one last heartbeat - self.heartbeat() \ No newline at end of file + self.heartbeat() diff --git a/metaflow/metadata/metadata.py b/metaflow/metadata/metadata.py index 9fea610bf53..c20b441f09e 100644 --- a/metaflow/metadata/metadata.py +++ b/metaflow/metadata/metadata.py @@ -9,14 +9,13 @@ from metaflow.util import get_username, resolve_identity -DataArtifact = namedtuple('DataArtifact', - 'name ds_type ds_root url type sha') +DataArtifact = namedtuple("DataArtifact", "name ds_type ds_root url type sha") -MetaDatum = namedtuple('MetaDatum', - 'field value type tags') +MetaDatum = namedtuple("MetaDatum", "field value type tags") attempt_id_re = re.compile(r"attempt_id:([0-9]+)") + class MetadataProviderMeta(type): def __new__(metaname, classname, bases, attrs): return type.__new__(metaname, classname, bases, attrs) @@ -41,18 +40,18 @@ def with_metaclass(mcls): def decorator(cls): body = vars(cls).copy() # clean out class body - body.pop('__dict__', None) - body.pop('__weakref__', None) + body.pop("__dict__", None) + body.pop("__weakref__", None) return mcls(cls.__name__, cls.__bases__, body) + return decorator @with_metaclass(MetadataProviderMeta) class MetadataProvider(object): - @classmethod def compute_info(cls, val): - ''' + """ Compute the new information for this provider The computed value should be returned and will then be accessible directly as cls.INFO. @@ -68,12 +67,12 @@ def compute_info(cls, val): ------- str : Value to be set to INFO - ''' - return '' + """ + return "" @classmethod def default_info(cls): - ''' + """ Returns the default information for this provider This should compute and return the default value for the information regarding this provider. @@ -83,22 +82,22 @@ def default_info(cls): ------- str Value to be set by default in INFO - ''' - return '' + """ + return "" def version(self): - ''' + """ Returns the version of this provider Returns ------- str Version of the provider - ''' - return '' + """ + return "" def new_run_id(self, tags=None, sys_tags=None): - ''' + """ Creates an ID and registers this new run. The run ID will be unique within a given flow. @@ -114,11 +113,11 @@ def new_run_id(self, tags=None, sys_tags=None): ------- int Run ID for the run - ''' + """ raise NotImplementedError() def register_run_id(self, run_id, tags=None, sys_tags=None): - ''' + """ No-op operation in this implementation. Parameters @@ -129,11 +128,11 @@ def register_run_id(self, run_id, tags=None, sys_tags=None): Tags to apply to this particular run, by default None sys_tags : list, optional System tags to apply to this particular run, by default None - ''' + """ raise NotImplementedError() def new_task_id(self, run_id, step_name, tags=None, sys_tags=None): - ''' + """ Creates an ID and registers this new task. The task ID will be unique within a flow, run and step @@ -153,12 +152,13 @@ def new_task_id(self, run_id, step_name, tags=None, sys_tags=None): ------- int Task ID for the task - ''' + """ raise NotImplementedError() def register_task_id( - self, run_id, step_name, task_id, attempt=0, tags=None, sys_tags=None): - ''' + self, run_id, step_name, task_id, attempt=0, tags=None, sys_tags=None + ): + """ No-op operation in this implementation. Parameters @@ -173,11 +173,11 @@ def register_task_id( Tags to apply to this particular run, by default [] sys_tags : list, optional System tags to apply to this particular run, by default [] - ''' + """ raise NotImplementedError() def get_runtime_environment(self, runtime_name): - ''' + """ Returns a dictionary of environment variables to be set Parameters @@ -189,17 +189,13 @@ def get_runtime_environment(self, runtime_name): ------- dict[string] -> string Environment variables from this metadata provider - ''' - return {'METAFLOW_RUNTIME_NAME': runtime_name, - 'USER': get_username()} - - def register_data_artifacts(self, - run_id, - step_name, - task_id, - attempt_id, - artifacts): - ''' + """ + return {"METAFLOW_RUNTIME_NAME": runtime_name, "USER": get_username()} + + def register_data_artifacts( + self, run_id, step_name, task_id, attempt_id, artifacts + ): + """ Registers the fact that the data-artifacts are associated with the particular task. @@ -218,11 +214,11 @@ def register_data_artifacts(self, Attempt for the task artifacts : List of DataArtifact Artifacts associated with this task - ''' + """ raise NotImplementedError() def register_metadata(self, run_id, step_name, task_id, metadata): - ''' + """ Registers metadata with a task. Note that the same metadata can be registered multiple times for the same task (for example @@ -239,7 +235,7 @@ def register_metadata(self, run_id, step_name, task_id, metadata): Task ID for the task metadata : List of MetaDatum Metadata associated with this task - ''' + """ raise NotImplementedError() def start_task_heartbeat(self, flow_id, run_id, step_name, task_id): @@ -253,8 +249,9 @@ def stop_heartbeat(self): @classmethod def _get_object_internal( - cls, obj_type, obj_order, sub_type, sub_order, filters, attempt, *args): - ''' + cls, obj_type, obj_order, sub_type, sub_order, filters, attempt, *args + ): + """ Return objects for the implementation of this class See get_object_internal for the description of what this function does @@ -285,11 +282,11 @@ def _get_object_internal( ------ object or list : Depending on the call, the type of object return varies - ''' + """ raise NotImplementedError() def add_sticky_tags(self, tags=None, sys_tags=None): - ''' + """ Adds tags to be added to every run and task Tags can be added to record information about a run/task. Such tags can be specified on a @@ -303,7 +300,7 @@ def add_sticky_tags(self, tags=None, sys_tags=None): Tags to add to every run/task, by default None sys_tags : list, optional System tags to add to every run/task, by default None - ''' + """ if tags: self.sticky_tags.update(tags) if sys_tags: @@ -311,7 +308,7 @@ def add_sticky_tags(self, tags=None, sys_tags=None): @classmethod def get_object(cls, obj_type, sub_type, filters, attempt, *args): - '''Returns the requested object depending on obj_type and sub_type + """Returns the requested object depending on obj_type and sub_type obj_type can be one of 'root', 'flow', 'run', 'step', 'task', or 'artifact' @@ -357,33 +354,38 @@ def get_object(cls, obj_type, sub_type, filters, attempt, *args): ------ object or list : Depending on the call, the type of object return varies - ''' + """ obj_order = { - 'root': 0, - 'flow': 1, - 'run': 2, - 'step': 3, - 'task': 4, - 'artifact': 5, - 'metadata': 6, - 'self': 7} + "root": 0, + "flow": 1, + "run": 2, + "step": 3, + "task": 4, + "artifact": 5, + "metadata": 6, + "self": 7, + } type_order = obj_order.get(obj_type) sub_order = obj_order.get(sub_type) if type_order is None: - raise MetaflowInternalError(msg='Cannot find type %s' % obj_type) + raise MetaflowInternalError(msg="Cannot find type %s" % obj_type) if type_order > 5: - raise MetaflowInternalError(msg='Type %s is not allowed' % obj_type) + raise MetaflowInternalError(msg="Type %s is not allowed" % obj_type) if sub_order is None: - raise MetaflowInternalError(msg='Cannot find subtype %s' % sub_type) + raise MetaflowInternalError(msg="Cannot find subtype %s" % sub_type) if type_order >= sub_order: - raise MetaflowInternalError(msg='Subtype %s not allowed for %s' % (sub_type, obj_type)) + raise MetaflowInternalError( + msg="Subtype %s not allowed for %s" % (sub_type, obj_type) + ) # Metadata is always only at the task level - if sub_type == 'metadata' and obj_type != 'task': - raise MetaflowInternalError(msg='Metadata can only be retrieved at the task level') + if sub_type == "metadata" and obj_type != "task": + raise MetaflowInternalError( + msg="Metadata can only be retrieved at the task level" + ) if attempt is not None: try: @@ -396,61 +398,65 @@ def get_object(cls, obj_type, sub_type, filters, attempt, *args): attempt_int = None pre_filter = cls._get_object_internal( - obj_type, type_order, sub_type, sub_order, filters, attempt_int, *args) + obj_type, type_order, sub_type, sub_order, filters, attempt_int, *args + ) if attempt_int is None or sub_order != 6: # If no attempt or not for metadata, just return as is return pre_filter return MetadataProvider._reconstruct_metadata_for_attempt( - pre_filter, attempt_int) + pre_filter, attempt_int + ) def _all_obj_elements(self, tags=None, sys_tags=None): user = get_username() return { - 'flow_id': self._flow_name, - 'user_name': user, - 'tags': list(tags) if tags else [], - 'system_tags': list(sys_tags) if sys_tags else [], - 'ts_epoch': int(round(time.time() * 1000))} + "flow_id": self._flow_name, + "user_name": user, + "tags": list(tags) if tags else [], + "system_tags": list(sys_tags) if sys_tags else [], + "ts_epoch": int(round(time.time() * 1000)), + } def _flow_to_json(self): # No need to store tags, sys_tags or username at the flow level # since runs are the top level logical concept, which is where we # store tags, sys_tags and username - return { - 'flow_id': self._flow_name, - 'ts_epoch': int(round(time.time() * 1000))} + return {"flow_id": self._flow_name, "ts_epoch": int(round(time.time() * 1000))} def _run_to_json(self, run_id=None, tags=None, sys_tags=None): if run_id is not None: - d = {'run_number': run_id} + d = {"run_number": run_id} else: d = {} d.update(self._all_obj_elements(tags, sys_tags)) return d def _step_to_json(self, run_id, step_name, tags=None, sys_tags=None): - d = { - 'run_number': run_id, - 'step_name': step_name} + d = {"run_number": run_id, "step_name": step_name} d.update(self._all_obj_elements(tags, sys_tags)) return d def _task_to_json(self, run_id, step_name, task_id=None, tags=None, sys_tags=None): - d = { - 'run_number': run_id, - 'step_name': step_name} + d = {"run_number": run_id, "step_name": step_name} if task_id is not None: - d['task_id'] = task_id + d["task_id"] = task_id d.update(self._all_obj_elements(tags, sys_tags)) return d def _object_to_json( - self, obj_type, run_id=None, step_name=None, task_id=None, tags=None, sys_tags=None): - if obj_type == 'task': + self, + obj_type, + run_id=None, + step_name=None, + task_id=None, + tags=None, + sys_tags=None, + ): + if obj_type == "task": return self._task_to_json(run_id, step_name, task_id, tags, sys_tags) - if obj_type == 'step': + if obj_type == "step": return self._step_to_json(run_id, step_name, tags, sys_tags) - if obj_type == 'run': + if obj_type == "run": return self._run_to_json(run_id, tags, sys_tags) return self._flow_to_json() @@ -458,60 +464,71 @@ def _artifacts_to_json(self, run_id, step_name, task_id, attempt_id, artifacts): result = [] for art in artifacts: d = { - 'run_number': run_id, - 'step_name': step_name, - 'task_id': task_id, - 'attempt_id': attempt_id, - 'name': art.name, - 'content_type': art.type, - 'type': 'metaflow.artifact', - 'sha': art.sha, - 'ds_type': art.ds_type, - 'location': art.url if art.url else ':root:%s' % art.ds_root} + "run_number": run_id, + "step_name": step_name, + "task_id": task_id, + "attempt_id": attempt_id, + "name": art.name, + "content_type": art.type, + "type": "metaflow.artifact", + "sha": art.sha, + "ds_type": art.ds_type, + "location": art.url if art.url else ":root:%s" % art.ds_root, + } d.update(self._all_obj_elements(self.sticky_tags, self.sticky_sys_tags)) result.append(d) return result def _metadata_to_json(self, run_id, step_name, task_id, metadata): user = get_username() - return [{ - 'flow_id': self._flow_name, - 'run_number': run_id, - 'step_name': step_name, - 'task_id': task_id, - 'field_name': datum.field, - 'type': datum.type, - 'value': datum.value, - 'tags': list(set(datum.tags)) if datum.tags else [], - 'user_name': user, - 'ts_epoch': int(round(time.time() * 1000))} for datum in metadata] + return [ + { + "flow_id": self._flow_name, + "run_number": run_id, + "step_name": step_name, + "task_id": task_id, + "field_name": datum.field, + "type": datum.type, + "value": datum.value, + "tags": list(set(datum.tags)) if datum.tags else [], + "user_name": user, + "ts_epoch": int(round(time.time() * 1000)), + } + for datum in metadata + ] def _tags(self): env = self._environment.get_environment_info() tags = [ resolve_identity(), - 'runtime:' + env['runtime'], - 'python_version:' + env['python_version_code'], - 'date:' + datetime.utcnow().strftime('%Y-%m-%d')] - if env['metaflow_version']: - tags.append('metaflow_version:' + env['metaflow_version']) - if 'metaflow_r_version' in env: - tags.append('metaflow_r_version:' + env['metaflow_r_version']) - if 'r_version_code' in env: - tags.append('r_version:' + env['r_version_code']) + "runtime:" + env["runtime"], + "python_version:" + env["python_version_code"], + "date:" + datetime.utcnow().strftime("%Y-%m-%d"), + ] + if env["metaflow_version"]: + tags.append("metaflow_version:" + env["metaflow_version"]) + if "metaflow_r_version" in env: + tags.append("metaflow_r_version:" + env["metaflow_r_version"]) + if "r_version_code" in env: + tags.append("r_version:" + env["r_version_code"]) return tags def _register_code_package_metadata(self, run_id, step_name, task_id, attempt): metadata = [] - code_sha = os.environ.get('METAFLOW_CODE_SHA') - code_url = os.environ.get('METAFLOW_CODE_URL') - code_ds = os.environ.get('METAFLOW_CODE_DS') + code_sha = os.environ.get("METAFLOW_CODE_SHA") + code_url = os.environ.get("METAFLOW_CODE_URL") + code_ds = os.environ.get("METAFLOW_CODE_DS") if code_sha: - metadata.append(MetaDatum( - field='code-package', - value=json.dumps({'ds_type': code_ds, 'sha': code_sha, 'location': code_url}), - type='code-package', - tags=["attempt_id:{0}".format(attempt)])) + metadata.append( + MetaDatum( + field="code-package", + value=json.dumps( + {"ds_type": code_ds, "sha": code_sha, "location": code_url} + ), + type="code-package", + tags=["attempt_id:{0}".format(attempt)], + ) + ) # We don't tag with attempt_id here because not readily available; this # is ok though as this doesn't change from attempt to attempt. if metadata: @@ -524,17 +541,19 @@ def _apply_filter(elts, filters): starting_point = elts result = [] for key, value in filters.items(): - if key == 'any_tags': + if key == "any_tags": for obj in starting_point: - if value in obj.get('tags', []) or value in obj.get('system_tags', []): + if value in obj.get("tags", []) or value in obj.get( + "system_tags", [] + ): result.append(obj) - if key == 'tags': + if key == "tags": for obj in starting_point: - if value in obj.get('tags', []): + if value in obj.get("tags", []): result.append(obj) - if key == 'system_tags': + if key == "system_tags": for obj in starting_point: - if value in obj.get('system_tags', []): + if value in obj.get("system_tags", []): result.append(obj) starting_point = result result = [] @@ -546,9 +565,9 @@ def _reconstruct_metadata_for_attempt(all_metadata, attempt_id): attempts_start = {} post_filter = [] for v in all_metadata: - if v['field_name'] == 'attempt': - attempts_start[int(v['value'])] = v['ts_epoch'] - all_tags = v.get('tags') + if v["field_name"] == "attempt": + attempts_start[int(v["value"])] = v["ts_epoch"] + all_tags = v.get("tags") if all_tags is None: all_tags = [] for t in all_tags: @@ -565,11 +584,14 @@ def _reconstruct_metadata_for_attempt(all_metadata, attempt_id): # We reconstruct base on the attempts_start start_ts = attempts_start.get(attempt_id, -1) if start_ts < 0: - return [] # No metadata since the attempt hasn't started + return [] # No metadata since the attempt hasn't started # Doubt we will be using Python in year 3000 end_ts = attempts_start.get(attempt_id + 1, 32503680000000) - post_filter = [v for v in all_metadata - if v['ts_epoch'] >= start_ts and v['ts_epoch'] < end_ts] + post_filter = [ + v + for v in all_metadata + if v["ts_epoch"] >= start_ts and v["ts_epoch"] < end_ts + ] return post_filter @@ -581,6 +603,5 @@ def __init__(self, environment, flow, event_logger, monitor): self._event_logger = event_logger self._monitor = monitor self._environment = environment - self._runtime = os.environ.get( - 'METAFLOW_RUNTIME_NAME', 'dev') + self._runtime = os.environ.get("METAFLOW_RUNTIME_NAME", "dev") self.add_sticky_tags(sys_tags=self._tags()) diff --git a/metaflow/metadata/util.py b/metaflow/metadata/util.py index e7ec022f282..af16ad86ef6 100644 --- a/metaflow/metadata/util.py +++ b/metaflow/metadata/util.py @@ -10,24 +10,26 @@ def sync_local_metadata_to_datastore(metadata_local_dir, task_ds): with util.TempDir() as td: - tar_file_path = os.path.join(td, 'metadata.tgz') + tar_file_path = os.path.join(td, "metadata.tgz") buf = BytesIO() - with tarfile.open(name=tar_file_path, mode='w:gz', fileobj=buf) as tar: + with tarfile.open(name=tar_file_path, mode="w:gz", fileobj=buf) as tar: tar.add(metadata_local_dir) blob = buf.getvalue() _, key = task_ds.parent_datastore.save_data([blob], len_hint=1)[0] - task_ds.save_metadata({'local_metadata': key}) + task_ds.save_metadata({"local_metadata": key}) def sync_local_metadata_from_datastore(metadata_local_dir, task_ds): def echo_none(*args, **kwargs): pass - key_to_load = task_ds.load_metadata(['local_metadata'])['local_metadata'] + + key_to_load = task_ds.load_metadata(["local_metadata"])["local_metadata"] _, tarball = next(task_ds.parent_datastore.load_data([key_to_load])) with util.TempDir() as td: - with tarfile.open(fileobj=BytesIO(tarball), mode='r:gz') as tar: + with tarfile.open(fileobj=BytesIO(tarball), mode="r:gz") as tar: tar.extractall(td) copy_tree( os.path.join(td, metadata_local_dir), LocalStorage.get_datastore_root_from_config(echo_none), - update=True) \ No newline at end of file + update=True, + ) diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py index 0a4c77bbf40..05325aa092c 100644 --- a/metaflow/metaflow_config.py +++ b/metaflow/metaflow_config.py @@ -15,19 +15,20 @@ def init_config(): # Read configuration from $METAFLOW_HOME/config_.json. - home = os.environ.get('METAFLOW_HOME', '~/.metaflowconfig') - profile = os.environ.get('METAFLOW_PROFILE') - path_to_config = os.path.join(home, 'config.json') + home = os.environ.get("METAFLOW_HOME", "~/.metaflowconfig") + profile = os.environ.get("METAFLOW_PROFILE") + path_to_config = os.path.join(home, "config.json") if profile: - path_to_config = os.path.join(home, 'config_%s.json' % profile) + path_to_config = os.path.join(home, "config_%s.json" % profile) path_to_config = os.path.expanduser(path_to_config) config = {} if os.path.exists(path_to_config): with open(path_to_config) as f: return json.load(f) elif profile: - raise MetaflowException('Unable to locate METAFLOW_PROFILE \'%s\' in \'%s\')' % - (profile, home)) + raise MetaflowException( + "Unable to locate METAFLOW_PROFILE '%s' in '%s')" % (profile, home) + ) return config @@ -42,86 +43,96 @@ def from_conf(name, default=None): ### # Default configuration ### -DEFAULT_DATASTORE = from_conf('METAFLOW_DEFAULT_DATASTORE', 'local') -DEFAULT_ENVIRONMENT = from_conf('METAFLOW_DEFAULT_ENVIRONMENT', 'local') -DEFAULT_EVENT_LOGGER = from_conf('METAFLOW_DEFAULT_EVENT_LOGGER', 'nullSidecarLogger') -DEFAULT_METADATA = from_conf('METAFLOW_DEFAULT_METADATA', 'local') -DEFAULT_MONITOR = from_conf('METAFLOW_DEFAULT_MONITOR', 'nullSidecarMonitor') -DEFAULT_PACKAGE_SUFFIXES = from_conf('METAFLOW_DEFAULT_PACKAGE_SUFFIXES', '.py,.R,.RDS') -DEFAULT_AWS_CLIENT_PROVIDER = from_conf('METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER', 'boto3') +DEFAULT_DATASTORE = from_conf("METAFLOW_DEFAULT_DATASTORE", "local") +DEFAULT_ENVIRONMENT = from_conf("METAFLOW_DEFAULT_ENVIRONMENT", "local") +DEFAULT_EVENT_LOGGER = from_conf("METAFLOW_DEFAULT_EVENT_LOGGER", "nullSidecarLogger") +DEFAULT_METADATA = from_conf("METAFLOW_DEFAULT_METADATA", "local") +DEFAULT_MONITOR = from_conf("METAFLOW_DEFAULT_MONITOR", "nullSidecarMonitor") +DEFAULT_PACKAGE_SUFFIXES = from_conf("METAFLOW_DEFAULT_PACKAGE_SUFFIXES", ".py,.R,.RDS") +DEFAULT_AWS_CLIENT_PROVIDER = from_conf("METAFLOW_DEFAULT_AWS_CLIENT_PROVIDER", "boto3") ### # Datastore configuration ### # Path to the local directory to store artifacts for 'local' datastore. -DATASTORE_LOCAL_DIR = '.metaflow' -DATASTORE_SYSROOT_LOCAL = from_conf('METAFLOW_DATASTORE_SYSROOT_LOCAL') +DATASTORE_LOCAL_DIR = ".metaflow" +DATASTORE_SYSROOT_LOCAL = from_conf("METAFLOW_DATASTORE_SYSROOT_LOCAL") # S3 bucket and prefix to store artifacts for 's3' datastore. -DATASTORE_SYSROOT_S3 = from_conf('METAFLOW_DATASTORE_SYSROOT_S3') +DATASTORE_SYSROOT_S3 = from_conf("METAFLOW_DATASTORE_SYSROOT_S3") # S3 datatools root location -DATATOOLS_SUFFIX = from_conf('METAFLOW_DATATOOLS_SUFFIX', 'data') +DATATOOLS_SUFFIX = from_conf("METAFLOW_DATATOOLS_SUFFIX", "data") DATATOOLS_S3ROOT = from_conf( - 'METAFLOW_DATATOOLS_S3ROOT', - '%s/%s' % (from_conf('METAFLOW_DATASTORE_SYSROOT_S3'), DATATOOLS_SUFFIX) - if from_conf('METAFLOW_DATASTORE_SYSROOT_S3') else None) + "METAFLOW_DATATOOLS_S3ROOT", + "%s/%s" % (from_conf("METAFLOW_DATASTORE_SYSROOT_S3"), DATATOOLS_SUFFIX) + if from_conf("METAFLOW_DATASTORE_SYSROOT_S3") + else None, +) # Local datatools root location DATATOOLS_LOCALROOT = from_conf( - 'METAFLOW_DATATOOLS_LOCALROOT', - '%s/%s' % (from_conf('METAFLOW_DATASTORE_SYSROOT_LOCAL'), DATATOOLS_SUFFIX) - if from_conf('METAFLOW_DATASTORE_SYSROOT_LOCAL') else None) + "METAFLOW_DATATOOLS_LOCALROOT", + "%s/%s" % (from_conf("METAFLOW_DATASTORE_SYSROOT_LOCAL"), DATATOOLS_SUFFIX) + if from_conf("METAFLOW_DATASTORE_SYSROOT_LOCAL") + else None, +) -# S3 endpoint url -S3_ENDPOINT_URL = from_conf('METAFLOW_S3_ENDPOINT_URL', None) -S3_VERIFY_CERTIFICATE = from_conf('METAFLOW_S3_VERIFY_CERTIFICATE', None) +# S3 endpoint url +S3_ENDPOINT_URL = from_conf("METAFLOW_S3_ENDPOINT_URL", None) +S3_VERIFY_CERTIFICATE = from_conf("METAFLOW_S3_VERIFY_CERTIFICATE", None) # S3 retry configuration # This is useful if you want to "fail fast" on S3 operations; use with caution # though as this may increase failures. Note that this is the number of *retries* # so setting it to 0 means each operation will be tried once. -S3_RETRY_COUNT = int(from_conf('METAFLOW_S3_RETRY_COUNT', 7)) +S3_RETRY_COUNT = int(from_conf("METAFLOW_S3_RETRY_COUNT", 7)) ### # Datastore local cache ### # Path to the client cache -CLIENT_CACHE_PATH = from_conf('METAFLOW_CLIENT_CACHE_PATH', '/tmp/metaflow_client') +CLIENT_CACHE_PATH = from_conf("METAFLOW_CLIENT_CACHE_PATH", "/tmp/metaflow_client") # Maximum size (in bytes) of the cache -CLIENT_CACHE_MAX_SIZE = int(from_conf('METAFLOW_CLIENT_CACHE_MAX_SIZE', 10000)) +CLIENT_CACHE_MAX_SIZE = int(from_conf("METAFLOW_CLIENT_CACHE_MAX_SIZE", 10000)) # Maximum number of cached Flow and TaskDatastores in the cache -CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT = int(from_conf( - 'METAFLOW_CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT', 50)) -CLIENT_CACHE_MAX_TASKDATASTORE_COUNT = int(from_conf( - 'METAFLOW_CLIENT_CACHE_MAX_TASKDATASTORE_COUNT', - CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT * 100)) +CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT = int( + from_conf("METAFLOW_CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT", 50) +) +CLIENT_CACHE_MAX_TASKDATASTORE_COUNT = int( + from_conf( + "METAFLOW_CLIENT_CACHE_MAX_TASKDATASTORE_COUNT", + CLIENT_CACHE_MAX_FLOWDATASTORE_COUNT * 100, + ) +) ### # Metadata configuration ### -METADATA_SERVICE_URL = from_conf('METAFLOW_SERVICE_URL') -METADATA_SERVICE_NUM_RETRIES = from_conf('METAFLOW_SERVICE_RETRY_COUNT', 5) -METADATA_SERVICE_AUTH_KEY = from_conf('METAFLOW_SERVICE_AUTH_KEY') -METADATA_SERVICE_HEADERS = json.loads(from_conf('METAFLOW_SERVICE_HEADERS', '{}')) +METADATA_SERVICE_URL = from_conf("METAFLOW_SERVICE_URL") +METADATA_SERVICE_NUM_RETRIES = from_conf("METAFLOW_SERVICE_RETRY_COUNT", 5) +METADATA_SERVICE_AUTH_KEY = from_conf("METAFLOW_SERVICE_AUTH_KEY") +METADATA_SERVICE_HEADERS = json.loads(from_conf("METAFLOW_SERVICE_HEADERS", "{}")) if METADATA_SERVICE_AUTH_KEY is not None: - METADATA_SERVICE_HEADERS['x-api-key'] = METADATA_SERVICE_AUTH_KEY + METADATA_SERVICE_HEADERS["x-api-key"] = METADATA_SERVICE_AUTH_KEY ### # AWS Batch configuration ### -# IAM role for AWS Batch container with Amazon S3 access +# IAM role for AWS Batch container with Amazon S3 access # (and AWS DynamoDb access for AWS StepFunctions, if enabled) -ECS_S3_ACCESS_IAM_ROLE = from_conf('METAFLOW_ECS_S3_ACCESS_IAM_ROLE') +ECS_S3_ACCESS_IAM_ROLE = from_conf("METAFLOW_ECS_S3_ACCESS_IAM_ROLE") # IAM role for AWS Batch container for AWS Fargate -ECS_FARGATE_EXECUTION_ROLE = from_conf('METAFLOW_ECS_FARGATE_EXECUTION_ROLE') +ECS_FARGATE_EXECUTION_ROLE = from_conf("METAFLOW_ECS_FARGATE_EXECUTION_ROLE") # Job queue for AWS Batch -BATCH_JOB_QUEUE = from_conf('METAFLOW_BATCH_JOB_QUEUE') +BATCH_JOB_QUEUE = from_conf("METAFLOW_BATCH_JOB_QUEUE") # Default container image for AWS Batch BATCH_CONTAINER_IMAGE = from_conf("METAFLOW_BATCH_CONTAINER_IMAGE") # Default container registry for AWS Batch BATCH_CONTAINER_REGISTRY = from_conf("METAFLOW_BATCH_CONTAINER_REGISTRY") # Metadata service URL for AWS Batch -BATCH_METADATA_SERVICE_URL = from_conf('METAFLOW_SERVICE_INTERNAL_URL', METADATA_SERVICE_URL) +BATCH_METADATA_SERVICE_URL = from_conf( + "METAFLOW_SERVICE_INTERNAL_URL", METADATA_SERVICE_URL +) BATCH_METADATA_SERVICE_HEADERS = METADATA_SERVICE_HEADERS # Assign resource tags to AWS Batch jobs. Set to False by default since @@ -145,7 +156,7 @@ def from_conf(name, default=None): # sandbox. SFN_STATE_MACHINE_PREFIX = from_conf("METAFLOW_SFN_STATE_MACHINE_PREFIX") # Optional AWS CloudWatch Log Group ARN for emitting AWS Step Functions state -# machine execution logs. This needs to be available when using the +# machine execution logs. This needs to be available when using the # `step-functions create --log-execution-history` command. SFN_EXECUTION_LOG_GROUP_ARN = from_conf("METAFLOW_SFN_EXECUTION_LOG_GROUP_ARN") @@ -166,38 +177,43 @@ def from_conf(name, default=None): ### # Conda package root location on S3 CONDA_PACKAGE_S3ROOT = from_conf( - 'METAFLOW_CONDA_PACKAGE_S3ROOT', - '%s/conda' % from_conf('METAFLOW_DATASTORE_SYSROOT_S3')) + "METAFLOW_CONDA_PACKAGE_S3ROOT", + "%s/conda" % from_conf("METAFLOW_DATASTORE_SYSROOT_S3"), +) ### # Debug configuration ### -DEBUG_OPTIONS = ['subcommand', 'sidecar', 's3client'] +DEBUG_OPTIONS = ["subcommand", "sidecar", "s3client"] for typ in DEBUG_OPTIONS: - vars()['METAFLOW_DEBUG_%s' % typ.upper()] = from_conf('METAFLOW_DEBUG_%s' % typ.upper()) + vars()["METAFLOW_DEBUG_%s" % typ.upper()] = from_conf( + "METAFLOW_DEBUG_%s" % typ.upper() + ) ### # AWS Sandbox configuration ### # Boolean flag for metaflow AWS sandbox access -AWS_SANDBOX_ENABLED = bool(from_conf('METAFLOW_AWS_SANDBOX_ENABLED', False)) +AWS_SANDBOX_ENABLED = bool(from_conf("METAFLOW_AWS_SANDBOX_ENABLED", False)) # Metaflow AWS sandbox auth endpoint -AWS_SANDBOX_STS_ENDPOINT_URL = from_conf('METAFLOW_SERVICE_URL') +AWS_SANDBOX_STS_ENDPOINT_URL = from_conf("METAFLOW_SERVICE_URL") # Metaflow AWS sandbox API auth key -AWS_SANDBOX_API_KEY = from_conf('METAFLOW_AWS_SANDBOX_API_KEY') +AWS_SANDBOX_API_KEY = from_conf("METAFLOW_AWS_SANDBOX_API_KEY") # Internal Metadata URL -AWS_SANDBOX_INTERNAL_SERVICE_URL = from_conf('METAFLOW_AWS_SANDBOX_INTERNAL_SERVICE_URL') +AWS_SANDBOX_INTERNAL_SERVICE_URL = from_conf( + "METAFLOW_AWS_SANDBOX_INTERNAL_SERVICE_URL" +) # AWS region -AWS_SANDBOX_REGION = from_conf('METAFLOW_AWS_SANDBOX_REGION') +AWS_SANDBOX_REGION = from_conf("METAFLOW_AWS_SANDBOX_REGION") # Finalize configuration if AWS_SANDBOX_ENABLED: - os.environ['AWS_DEFAULT_REGION'] = AWS_SANDBOX_REGION + os.environ["AWS_DEFAULT_REGION"] = AWS_SANDBOX_REGION BATCH_METADATA_SERVICE_URL = AWS_SANDBOX_INTERNAL_SERVICE_URL - METADATA_SERVICE_HEADERS['x-api-key'] = AWS_SANDBOX_API_KEY - SFN_STATE_MACHINE_PREFIX = from_conf('METAFLOW_AWS_SANDBOX_STACK_NAME') + METADATA_SERVICE_HEADERS["x-api-key"] = AWS_SANDBOX_API_KEY + SFN_STATE_MACHINE_PREFIX = from_conf("METAFLOW_AWS_SANDBOX_STACK_NAME") # MAX_ATTEMPTS is the maximum number of attempts, including the first @@ -218,8 +234,7 @@ def from_conf(name, default=None): # to silence it: class Filter(logging.Filter): def filter(self, record): - if record.pathname.endswith('driver.py') and \ - 'grammar' in record.msg: + if record.pathname.endswith("driver.py") and "grammar" in record.msg: return False return True @@ -237,19 +252,19 @@ def get_version(pkg): def get_pinned_conda_libs(python_version): if python_version.startswith("3.5"): return { - 'click': '7.1.2', - 'requests': '2.24.0', - 'boto3': '1.9.88', - 'coverage': '4.5.1' + "click": "7.1.2", + "requests": "2.24.0", + "boto3": "1.9.88", + "coverage": "4.5.1", } else: return { - 'click': '7.1.2', - 'requests': '2.24.0', - 'boto3': '1.14.47', - 'coverage': '4.5.4' + "click": "7.1.2", + "requests": "2.24.0", + "boto3": "1.14.47", + "coverage": "4.5.4", } - + # Check if there is a an extension to Metaflow to load and override everything try: @@ -260,28 +275,32 @@ def get_pinned_conda_libs(python_version): # e.name is set to the name of the package that fails to load # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) - if not (isinstance(e, ModuleNotFoundError) and \ - e.name in ['metaflow_extensions', 'metaflow_extensions.config']): + if not ( + isinstance(e, ModuleNotFoundError) + and e.name in ["metaflow_extensions", "metaflow_extensions.config"] + ): print( "Cannot load metaflow_extensions configuration -- " - "if you want to ignore, uninstall metaflow_extensions package") + "if you want to ignore, uninstall metaflow_extensions package" + ) raise else: # We load into globals whatever we have in extension_module # We specifically exclude any modules that may be included (like sys, os, etc) for n, o in extension_module.__dict__.items(): - if n == 'DEBUG_OPTIONS': + if n == "DEBUG_OPTIONS": DEBUG_OPTIONS.extend(o) for typ in o: - vars()['METAFLOW_DEBUG_%s' % typ.upper()] = \ - from_conf('METAFLOW_DEBUG_%s' % typ.upper()) - elif not n.startswith('__') and not isinstance(o, types.ModuleType): + vars()["METAFLOW_DEBUG_%s" % typ.upper()] = from_conf( + "METAFLOW_DEBUG_%s" % typ.upper() + ) + elif not n.startswith("__") and not isinstance(o, types.ModuleType): globals()[n] = o finally: # Erase all temporary names to avoid leaking things - for _n in ['ver', 'n', 'o', 'e', 'extension_module']: + for _n in ["ver", "n", "o", "e", "extension_module"]: try: del globals()[_n] except KeyError: pass - del globals()['_n'] + del globals()["_n"] diff --git a/metaflow/metaflow_environment.py b/metaflow/metaflow_environment.py index d941043c9ad..2f2e7583a01 100644 --- a/metaflow/metaflow_environment.py +++ b/metaflow/metaflow_environment.py @@ -12,11 +12,11 @@ class InvalidEnvironmentException(MetaflowException): - headline = 'Incompatible environment' + headline = "Incompatible environment" class MetaflowEnvironment(object): - TYPE = 'local' + TYPE = "local" def __init__(self, flow): pass @@ -79,26 +79,26 @@ def get_client_info(cls, flow_name, metadata): return "Local environment" def get_package_commands(self, code_package_url): - cmds = [BASH_MFLOG, - "mflog \'Setting up task environment.\'", - "%s -m pip install awscli click requests boto3 -qqq" - % self._python(), - "mkdir metaflow", - "cd metaflow", - "mkdir .metaflow", # mute local datastore creation log - "i=0; while [ $i -le 5 ]; do " - "mflog \'Downloading code package...\'; " - "%s -m awscli s3 cp %s job.tar >/dev/null && \ - mflog \'Code package downloaded.\' && break; " - "sleep 10; i=$((i+1)); " - "done" % (self._python(), code_package_url), - "if [ $i -gt 5 ]; then " - "mflog \'Failed to download code package from %s " - "after 6 tries. Exiting...\' && exit 1; " - "fi" % code_package_url, - "TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar", - "mflog \'Task is starting.\'", - ] + cmds = [ + BASH_MFLOG, + "mflog 'Setting up task environment.'", + "%s -m pip install awscli click requests boto3 -qqq" % self._python(), + "mkdir metaflow", + "cd metaflow", + "mkdir .metaflow", # mute local datastore creation log + "i=0; while [ $i -le 5 ]; do " + "mflog 'Downloading code package...'; " + "%s -m awscli s3 cp %s job.tar >/dev/null && \ + mflog 'Code package downloaded.' && break; " + "sleep 10; i=$((i+1)); " + "done" % (self._python(), code_package_url), + "if [ $i -gt 5 ]; then " + "mflog 'Failed to download code package from %s " + "after 6 tries. Exiting...' && exit 1; " + "fi" % code_package_url, + "TAR_OPTIONS='--warning=no-timestamp' tar xf job.tar", + "mflog 'Task is starting.'", + ] return cmds def get_environment_info(self): @@ -109,21 +109,23 @@ def get_environment_info(self): # note that this dict goes into the code package # so variables here should be relatively stable (no # timestamps) so the hash won't change all the time - env = {'platform': platform.system(), - 'username': get_username(), - 'production_token': os.environ.get('METAFLOW_PRODUCTION_TOKEN'), - 'runtime': os.environ.get('METAFLOW_RUNTIME_NAME', 'dev'), - 'app': os.environ.get('APP'), - 'environment_type': self.TYPE, - 'use_r': R.use_r(), - 'python_version': sys.version, - 'python_version_code': '%d.%d.%d' % sys.version_info[:3], - 'metaflow_version': version_cache, - 'script': os.path.basename(os.path.abspath(sys.argv[0]))} + env = { + "platform": platform.system(), + "username": get_username(), + "production_token": os.environ.get("METAFLOW_PRODUCTION_TOKEN"), + "runtime": os.environ.get("METAFLOW_RUNTIME_NAME", "dev"), + "app": os.environ.get("APP"), + "environment_type": self.TYPE, + "use_r": R.use_r(), + "python_version": sys.version, + "python_version_code": "%d.%d.%d" % sys.version_info[:3], + "metaflow_version": version_cache, + "script": os.path.basename(os.path.abspath(sys.argv[0])), + } if R.use_r(): - env['metaflow_r_version'] = R.metaflow_r_version() - env['r_version'] = R.r_version() - env['r_version_code'] = R.r_version_code() + env["metaflow_r_version"] = R.metaflow_r_version() + env["r_version"] = R.r_version() + env["r_version_code"] = R.r_version_code() return env def executable(self, step_name): @@ -133,4 +135,4 @@ def _python(self): if R.use_r(): return "python3" else: - return "python" \ No newline at end of file + return "python" diff --git a/metaflow/metaflow_profile.py b/metaflow/metaflow_profile.py index a87a24cd9c2..39ecf42cdc3 100644 --- a/metaflow/metaflow_profile.py +++ b/metaflow/metaflow_profile.py @@ -2,14 +2,15 @@ from contextlib import contextmanager + @contextmanager def profile(label, stats_dict=None): if stats_dict is None: - print('PROFILE: %s starting' % label) + print("PROFILE: %s starting" % label) start = time.time() yield took = int((time.time() - start) * 1000) if stats_dict is None: - print('PROFILE: %s completed in %dms' % (label, took)) + print("PROFILE: %s completed in %dms" % (label, took)) else: stats_dict[label] = stats_dict.get(label, 0) + took diff --git a/metaflow/metaflow_version.py b/metaflow/metaflow_version.py index ad66b0f5553..d83193e50cd 100644 --- a/metaflow/metaflow_version.py +++ b/metaflow/metaflow_version.py @@ -19,6 +19,7 @@ GIT_COMMAND = "git" if name == "nt": + def find_git_on_windows(): """find the path to the git executable on windows""" # first see if git is in the path @@ -33,11 +34,11 @@ def find_git_on_windows(): possible_locations = [] # look in program files for msysgit if "PROGRAMFILES(X86)" in environ: - possible_locations.append("%s/Git/cmd/git.exe" % - environ["PROGRAMFILES(X86)"]) + possible_locations.append( + "%s/Git/cmd/git.exe" % environ["PROGRAMFILES(X86)"] + ) if "PROGRAMFILES" in environ: - possible_locations.append("%s/Git/cmd/git.exe" % - environ["PROGRAMFILES"]) + possible_locations.append("%s/Git/cmd/git.exe" % environ["PROGRAMFILES"]) # look for the github version of git if "LOCALAPPDATA" in environ: github_dir = "%s/GitHub" % environ["LOCALAPPDATA"] @@ -45,8 +46,9 @@ def find_git_on_windows(): for subdir in listdir(github_dir): if not subdir.startswith("PortableGit"): continue - possible_locations.append("%s/%s/bin/git.exe" % - (github_dir, subdir)) + possible_locations.append( + "%s/%s/bin/git.exe" % (github_dir, subdir) + ) for possible_location in possible_locations: if path.isfile(possible_location): return possible_location @@ -62,18 +64,23 @@ def call_git_describe(abbrev=7): # first, make sure we are actually in a Metaflow repo, # not some other repo - with open(devnull, 'w') as fnull: + with open(devnull, "w") as fnull: arguments = [GIT_COMMAND, "rev-parse", "--show-toplevel"] - reponame = check_output(arguments, cwd=CURRENT_DIRECTORY, - stderr=fnull).decode("ascii").strip() - if path.basename(reponame) != 'metaflow': + reponame = ( + check_output(arguments, cwd=CURRENT_DIRECTORY, stderr=fnull) + .decode("ascii") + .strip() + ) + if path.basename(reponame) != "metaflow": return None with open(devnull, "w") as fnull: - arguments = [GIT_COMMAND, "describe", "--tags", - "--abbrev=%d" % abbrev] - return check_output(arguments, cwd=CURRENT_DIRECTORY, - stderr=fnull).decode("ascii").strip() + arguments = [GIT_COMMAND, "describe", "--tags", "--abbrev=%d" % abbrev] + return ( + check_output(arguments, cwd=CURRENT_DIRECTORY, stderr=fnull) + .decode("ascii") + .strip() + ) except (OSError, CalledProcessError): return None @@ -99,7 +106,7 @@ def read_info_version(): """Read version information from INFO file""" try: with open(INFO_FILE, "r") as contents: - return json.load(contents).get('metaflow_version') + return json.load(contents).get("metaflow_version") except IOError: return None @@ -117,7 +124,7 @@ def get_version(pep440=False): Otherwise, the version logged by package installer is returned. - If even that information isn't available (likely when executing on a + If even that information isn't available (likely when executing on a remote cloud instance), the version information is returned from INFO file in the current directory. @@ -127,10 +134,11 @@ def get_version(pep440=False): version_addl = None if version is None: # not a git repository import metaflow + version = metaflow.__version__ version_addl = metaflow.__version_addl__ - if version is None: # not a proper python package + if version is None: # not a proper python package version = read_info_version() if version and version_addl: - return '+'.join([version, version_addl]) + return "+".join([version, version_addl]) return version diff --git a/metaflow/mflog/__init__.py b/metaflow/mflog/__init__.py index 85902e9dea4..5a9076320b4 100644 --- a/metaflow/mflog/__init__.py +++ b/metaflow/mflog/__init__.py @@ -15,47 +15,49 @@ # or an empty string when trying to access these new-style files. # This is deliberate, so the users won't see partial files with older # clients. -RUNTIME_LOG_SOURCE = 'runtime' -TASK_LOG_SOURCE = 'task' +RUNTIME_LOG_SOURCE = "runtime" +TASK_LOG_SOURCE = "task" # Loglines from all sources need to be merged together to # produce a complete view of logs. Hence keep this list short # since every items takes a DataStore access. -LOG_SOURCES = [ - RUNTIME_LOG_SOURCE, - TASK_LOG_SOURCE -] +LOG_SOURCES = [RUNTIME_LOG_SOURCE, TASK_LOG_SOURCE] # BASH_MFLOG defines a bash function that outputs valid mflog # structured loglines. We use this to output properly timestamped # loglined prior to Metaflow package has been downloaded. # Note that MFLOG_STDOUT is defined by mflog_export_env_vars() function. -BASH_MFLOG =\ - 'mflog(){ '\ - 'T=$(date -u -Ins|tr , .); '\ - 'echo \\"[MFLOG|0|${T:0:26}Z|%s|$T]$1\\"'\ - ' >> $MFLOG_STDOUT; echo $1; '\ - ' }' % TASK_LOG_SOURCE +BASH_MFLOG = ( + "mflog(){ " + "T=$(date -u -Ins|tr , .); " + 'echo \\"[MFLOG|0|${T:0:26}Z|%s|$T]$1\\"' + " >> $MFLOG_STDOUT; echo $1; " + " }" % TASK_LOG_SOURCE +) -BASH_SAVE_LOGS_ARGS = ['python', '-m', 'metaflow.mflog.save_logs'] -BASH_SAVE_LOGS = ' '.join(BASH_SAVE_LOGS_ARGS) +BASH_SAVE_LOGS_ARGS = ["python", "-m", "metaflow.mflog.save_logs"] +BASH_SAVE_LOGS = " ".join(BASH_SAVE_LOGS_ARGS) # this function returns a bash expression that redirects stdout # and stderr of the given bash expression to mflog.tee def bash_capture_logs(bash_expr, var_transform=None): if var_transform is None: - var_transform = lambda s: '$%s' % s - cmd = 'python -m metaflow.mflog.tee %s %s' - parts = (bash_expr, - cmd % (TASK_LOG_SOURCE, var_transform('MFLOG_STDOUT')), - cmd % (TASK_LOG_SOURCE, var_transform('MFLOG_STDERR'))) - return '(%s) 1>> >(%s) 2>> >(%s >&2)' % parts + var_transform = lambda s: "$%s" % s + cmd = "python -m metaflow.mflog.tee %s %s" + parts = ( + bash_expr, + cmd % (TASK_LOG_SOURCE, var_transform("MFLOG_STDOUT")), + cmd % (TASK_LOG_SOURCE, var_transform("MFLOG_STDERR")), + ) + return "(%s) 1>> >(%s) 2>> >(%s >&2)" % parts + # update_delay determines how often logs should be uploaded to S3 # as a function of the task execution time -MIN_UPDATE_DELAY = 1. # the most frequent update interval -MAX_UPDATE_DELAY = 30. # the least frequent update interval +MIN_UPDATE_DELAY = 1.0 # the most frequent update interval +MAX_UPDATE_DELAY = 30.0 # the least frequent update interval + def update_delay(secs_since_start): # this sigmoid function reaches @@ -64,33 +66,36 @@ def update_delay(secs_since_start): # - 1.0 after 23 minutes # in other words, the user will see very frequent updates # during the first 10 minutes - sigmoid = 1. / (1. + math.exp(-0.01 * secs_since_start + 9.)) + sigmoid = 1.0 / (1.0 + math.exp(-0.01 * secs_since_start + 9.0)) return MIN_UPDATE_DELAY + sigmoid * MAX_UPDATE_DELAY + # this function is used to generate a Bash 'export' expression that # sets environment variables that are used by 'tee' and 'save_logs'. # Note that we can't set the env vars statically, as some of them # may need to be evaluated during runtime -def export_mflog_env_vars(flow_name=None, - run_id=None, - step_name=None, - task_id=None, - retry_count=None, - datastore_type=None, - datastore_root=None, - stdout_path=None, - stderr_path=None): +def export_mflog_env_vars( + flow_name=None, + run_id=None, + step_name=None, + task_id=None, + retry_count=None, + datastore_type=None, + datastore_root=None, + stdout_path=None, + stderr_path=None, +): - pathspec = '/'.join((flow_name, str(run_id), step_name, str(task_id))) + pathspec = "/".join((flow_name, str(run_id), step_name, str(task_id))) env_vars = { - 'PYTHONUNBUFFERED': 'x', - 'MF_PATHSPEC': pathspec, - 'MF_DATASTORE': datastore_type, - 'MF_ATTEMPT': retry_count, - 'MFLOG_STDOUT': stdout_path, - 'MFLOG_STDERR': stderr_path + "PYTHONUNBUFFERED": "x", + "MF_PATHSPEC": pathspec, + "MF_DATASTORE": datastore_type, + "MF_ATTEMPT": retry_count, + "MFLOG_STDOUT": stdout_path, + "MFLOG_STDERR": stderr_path, } if datastore_root is not None: - env_vars['MF_DATASTORE_ROOT'] = datastore_root + env_vars["MF_DATASTORE_ROOT"] = datastore_root - return 'export ' + ' '.join('%s=%s' % kv for kv in env_vars.items()) \ No newline at end of file + return "export " + " ".join("%s=%s" % kv for kv in env_vars.items()) diff --git a/metaflow/mflog/mflog.py b/metaflow/mflog/mflog.py index 53555c03d95..1f63377281b 100644 --- a/metaflow/mflog/mflog.py +++ b/metaflow/mflog/mflog.py @@ -7,25 +7,24 @@ from metaflow.exception import MetaflowException from metaflow.util import to_bytes, to_fileobj, to_unicode -VERSION = b'0' +VERSION = b"0" -RE = b'(\[!)?'\ - b'\[MFLOG\|'\ - b'(0)\|'\ - b'(.+?)Z\|'\ - b'(.+?)\|'\ - b'(.+?)\]'\ - b'(.*)' +RE = b"(\[!)?" b"\[MFLOG\|" b"(0)\|" b"(.+?)Z\|" b"(.+?)\|" b"(.+?)\]" b"(.*)" # the RE groups defined above must match the MFLogline fields below # except utc_timestamp, which is filled in by the parser based on utc_tstamp_str -MFLogline = namedtuple('MFLogline', ['should_persist', - 'version', - 'utc_tstamp_str', - 'logsource', - 'id', - 'msg', - 'utc_tstamp']) +MFLogline = namedtuple( + "MFLogline", + [ + "should_persist", + "version", + "utc_tstamp_str", + "logsource", + "id", + "msg", + "utc_tstamp", + ], +) LINE_PARSER = re.compile(RE) @@ -46,16 +45,20 @@ try: # python3 from datetime import timezone + def utc_to_local(utc_dt): return utc_dt.replace(tzinfo=timezone.utc).astimezone(tz=None) + except ImportError: # python2 import calendar + def utc_to_local(utc_dt): timestamp = calendar.timegm(utc_dt.timetuple()) local_dt = datetime.fromtimestamp(timestamp) return local_dt.replace(microsecond=utc_dt.microsecond) + def decorate(source, line, version=VERSION, now=None, lineid=None): if now is None: now = datetime.utcnow() @@ -64,20 +67,15 @@ def decorate(source, line, version=VERSION, now=None, lineid=None): lineid = to_bytes(str(uuid.uuid4())) line = to_bytes(line) source = to_bytes(source) - return b''.join((b'[MFLOG|', - version, - b'|', - tstamp, - b'Z|', - source, - b'|', - lineid, - b']', - line)) + return b"".join( + (b"[MFLOG|", version, b"|", tstamp, b"Z|", source, b"|", lineid, b"]", line) + ) + def is_structured(line): line = to_bytes(line) - return line.startswith(b'[MFLOG|') or line.startswith(b'[![MFLOG|') + return line.startswith(b"[MFLOG|") or line.startswith(b"[![MFLOG|") + def parse(line): line = to_bytes(line) @@ -90,35 +88,39 @@ def parse(line): except: pass + def set_should_persist(line): # this marker indicates that the logline should be persisted by # the receiver line = to_bytes(line) - if is_structured(line) and not line.startswith(b'[!['): - return b'[!' + line + if is_structured(line) and not line.startswith(b"[!["): + return b"[!" + line else: return line + def unset_should_persist(line): # prior to persisting, the should_persist marker should be removed # from the logline using this function line = to_bytes(line) - if is_structured(line) and line.startswith(b'[!['): + if is_structured(line) and line.startswith(b"[!["): return line[2:] else: return line + def refine(line, prefix=None, suffix=None): line = to_bytes(line) - prefix = to_bytes(prefix) if prefix else b'' - suffix = to_bytes(suffix) if suffix else b'' - parts = line.split(b']', 1) + prefix = to_bytes(prefix) if prefix else b"" + suffix = to_bytes(suffix) if suffix else b"" + parts = line.split(b"]", 1) if len(parts) == 2: header, body = parts - return b''.join((header, b']', prefix, body, suffix)) + return b"".join((header, b"]", prefix, body, suffix)) else: return line + def merge_logs(logs): def line_iter(logblob): # all valid timestamps are guaranteed to be smaller than @@ -132,13 +134,9 @@ def line_iter(logblob): else: missing.append(line) for line in missing: - res = MFLogline(False, - None, - MISSING_TIMESTAMP_STR, - None, - None, - line, - MISSING_TIMESTAMP) + res = MFLogline( + False, None, MISSING_TIMESTAMP_STR, None, None, line, MISSING_TIMESTAMP + ) yield res.utc_tstamp_str, res # note that sorted() below should be a very cheap, often a O(n) operation diff --git a/metaflow/mflog/save_logs.py b/metaflow/mflog/save_logs.py index 4e87053ce97..4931766ae94 100644 --- a/metaflow/mflog/save_logs.py +++ b/metaflow/mflog/save_logs.py @@ -7,42 +7,43 @@ from metaflow.util import Path from . import TASK_LOG_SOURCE -SMALL_FILE_LIMIT = 1024*1024 +SMALL_FILE_LIMIT = 1024 * 1024 + def save_logs(): def _read_file(path): - with open(path, 'rb') as f: + with open(path, "rb") as f: return f.read() # these env vars are set by mflog.mflog_env - pathspec = os.environ['MF_PATHSPEC'] - attempt = os.environ['MF_ATTEMPT'] - ds_type = os.environ['MF_DATASTORE'] - ds_root = os.environ.get('MF_DATASTORE_ROOT') - paths = (os.environ['MFLOG_STDOUT'],\ - os.environ['MFLOG_STDERR']) + pathspec = os.environ["MF_PATHSPEC"] + attempt = os.environ["MF_ATTEMPT"] + ds_type = os.environ["MF_DATASTORE"] + ds_root = os.environ.get("MF_DATASTORE_ROOT") + paths = (os.environ["MFLOG_STDOUT"], os.environ["MFLOG_STDERR"]) - flow_name, run_id, step_name, task_id = pathspec.split('/') + flow_name, run_id, step_name, task_id = pathspec.split("/") storage_impl = DATASTORES[ds_type] if ds_root is None: + def print_clean(line, **kwargs): pass + ds_root = storage_impl.get_datastore_root_from_config(print_clean) - flow_datastore = FlowDataStore(flow_name, - None, - storage_impl=storage_impl, - ds_root=ds_root) - task_datastore = flow_datastore.get_task_datastore(run_id, - step_name, - task_id, - int(attempt), - mode='w') + flow_datastore = FlowDataStore( + flow_name, None, storage_impl=storage_impl, ds_root=ds_root + ) + task_datastore = flow_datastore.get_task_datastore( + run_id, step_name, task_id, int(attempt), mode="w" + ) try: - streams = ('stdout', 'stderr') - sizes = [(stream, path, os.path.getsize(path)) - for stream, path in zip(streams, paths) - if os.path.exists(path)] + streams = ("stdout", "stderr") + sizes = [ + (stream, path, os.path.getsize(path)) + for stream, path in zip(streams, paths) + if os.path.exists(path) + ] if max(size for _, _, size in sizes) < SMALL_FILE_LIMIT: op = _read_file @@ -57,10 +58,11 @@ def print_clean(line, **kwargs): # for transient errors. pass -if __name__ == '__main__': + +if __name__ == "__main__": save_logs() - #to debug delays in logs, comment the line above and uncomment - #this snippet: + # to debug delays in logs, comment the line above and uncomment + # this snippet: """ import sys from metaflow.metaflow_profile import profile @@ -68,4 +70,4 @@ def print_clean(line, **kwargs): with profile('save_logs', stats_dict=d): save_logs() print('Save logs took %dms' % d['save_logs'], file=sys.stderr) - """ \ No newline at end of file + """ diff --git a/metaflow/mflog/save_logs_periodically.py b/metaflow/mflog/save_logs_periodically.py index 4452801ed7b..32fc3d4c98e 100644 --- a/metaflow/mflog/save_logs_periodically.py +++ b/metaflow/mflog/save_logs_periodically.py @@ -8,8 +8,8 @@ from metaflow.sidecar import SidecarSubProcess from . import update_delay, BASH_SAVE_LOGS_ARGS -class SaveLogsPeriodicallySidecar(object): +class SaveLogsPeriodicallySidecar(object): def __init__(self): self._thread = Thread(target=self._update_loop) self.is_alive = True @@ -29,7 +29,7 @@ def _file_size(path): return 0 # these env vars are set by mflog.mflog_env - FILES = [os.environ['MFLOG_STDOUT'], os.environ['MFLOG_STDERR']] + FILES = [os.environ["MFLOG_STDOUT"], os.environ["MFLOG_STDERR"]] start_time = time.time() sizes = [0 for _ in FILES] while self.is_alive: @@ -41,4 +41,3 @@ def _file_size(path): except: pass time.sleep(update_delay(time.time() - start_time)) - diff --git a/metaflow/mflog/tee.py b/metaflow/mflog/tee.py index b29b8d4da68..ca362bb4b85 100644 --- a/metaflow/mflog/tee.py +++ b/metaflow/mflog/tee.py @@ -6,13 +6,13 @@ # and a file. In contrast to 'tee', this script formats each # line with mflog-style structure. -if __name__ == '__main__': - SOURCE = sys.argv[1].encode('ascii') +if __name__ == "__main__": + SOURCE = sys.argv[1].encode("ascii") - with open(sys.argv[2], mode='ab', buffering=0) as f: + with open(sys.argv[2], mode="ab", buffering=0) as f: if sys.version_info < (3, 0): # Python 2 - for line in iter(sys.stdin.readline, ''): + for line in iter(sys.stdin.readline, ""): # https://bugs.python.org/issue3907 decorated = decorate(SOURCE, line) f.write(decorated) @@ -22,4 +22,4 @@ for line in sys.stdin.buffer: decorated = decorate(SOURCE, line) f.write(decorated) - sys.stdout.buffer.write(line) \ No newline at end of file + sys.stdout.buffer.write(line) diff --git a/metaflow/monitor.py b/metaflow/monitor.py index c69e3724bcd..ea33b5cd123 100644 --- a/metaflow/monitor.py +++ b/metaflow/monitor.py @@ -10,8 +10,8 @@ MEASURE_TYPE = "MEASURE" TIMER_TYPE = "TIMER" -class NullMonitor(object): +class NullMonitor(object): def __init__(self, *args, **kwargs): pass @@ -32,8 +32,8 @@ def gauge(self, gauge): def terminate(self): pass -class Monitor(NullMonitor): +class Monitor(NullMonitor): def __init__(self, monitor_type, env, flow_name): # type: (str) -> None self.sidecar_process = None @@ -50,9 +50,7 @@ def count(self, name): if self.sidecar_process is not None: counter = Counter(name, self.env_info) counter.increment() - payload = { - 'counter': counter.to_dict() - } + payload = {"counter": counter.to_dict()} msg = Message(MessageTypes.LOG_EVENT, payload) yield self.sidecar_process.msg_handler(msg) @@ -68,10 +66,7 @@ def measure(self, name): counter.increment() yield timer.end() - payload = { - 'counter': counter.to_dict(), - 'timer': timer.to_dict() - } + payload = {"counter": counter.to_dict(), "timer": timer.to_dict()} msg = Message(MessageTypes.LOG_EVENT, payload) self.sidecar_process.msg_handler(msg) else: @@ -79,9 +74,7 @@ def measure(self, name): def gauge(self, gauge): if self.sidecar_process is not None: - payload = { - 'gauge': gauge.to_dict() - } + payload = {"gauge": gauge.to_dict()} msg = Message(MessageTypes.LOG_EVENT, payload) self.sidecar_process.msg_handler(msg) @@ -92,7 +85,7 @@ def terminate(self): class Metric(object): """ - Abstract base class + Abstract base class """ def __init__(self, type, env): @@ -105,7 +98,7 @@ def name(self): @property def flow_name(self): - return self._env['flow_name'] + return self._env["flow_name"] @property def env(self): @@ -120,8 +113,8 @@ def set_env(self, env): def to_dict(self): return { - '_env': self._env, - '_type': self._type, + "_env": self._env, + "_type": self._type, } @@ -157,9 +150,9 @@ def value(self): def to_dict(self): parent_dict = super(Timer, self).to_dict() - parent_dict['_name'] = self.name - parent_dict['_start'] = self._start - parent_dict['_end'] = self._end + parent_dict["_name"] = self.name + parent_dict["_start"] = self._start + parent_dict["_end"] = self._end return parent_dict @@ -185,8 +178,8 @@ def value(self): def to_dict(self): parent_dict = super(Counter, self).to_dict() - parent_dict['_name'] = self.name - parent_dict['_count'] = self._count + parent_dict["_name"] = self.name + parent_dict["_count"] = self._count return parent_dict @@ -212,8 +205,8 @@ def value(self): def to_dict(self): parent_dict = super(Gauge, self).to_dict() - parent_dict['_name'] = self.name - parent_dict['_value'] = self.value + parent_dict["_name"] = self.name + parent_dict["_value"] = self.value return parent_dict @@ -221,36 +214,36 @@ def deserialize_metric(metrics_dict): if metrics_dict is None: return - type = metrics_dict.get('_type') - name = metrics_dict.get('_name') + type = metrics_dict.get("_type") + name = metrics_dict.get("_name") if type == COUNTER_TYPE: try: counter = Counter(name, None) - counter.set_env(metrics_dict.get('_env')) + counter.set_env(metrics_dict.get("_env")) except Exception as ex: return - counter.set_count(metrics_dict.get('_count')) + counter.set_count(metrics_dict.get("_count")) return counter elif type == TIMER_TYPE: timer = Timer(name, None) - timer.set_start(metrics_dict.get('_start')) - timer.set_end(metrics_dict.get('_end')) - timer.set_env(metrics_dict.get('_env')) + timer.set_start(metrics_dict.get("_start")) + timer.set_end(metrics_dict.get("_end")) + timer.set_env(metrics_dict.get("_env")) return timer elif type == GAUGE_TYPE: gauge = Gauge(name, None) - gauge.set_env(metrics_dict.get('_env')) - gauge.set_value(metrics_dict.get('_value')) + gauge.set_env(metrics_dict.get("_env")) + gauge.set_value(metrics_dict.get("_value")) return gauge else: raise NotImplementedError("UNSUPPORTED MESSAGE TYPE IN MONITOR") def get_monitor_msg_type(msg): - if msg.payload.get('gauge') is not None: + if msg.payload.get("gauge") is not None: return GAUGE_TYPE - if msg.payload.get('counter') is not None: - if msg.payload.get('timer') is not None: + if msg.payload.get("counter") is not None: + if msg.payload.get("timer") is not None: return MEASURE_TYPE - return COUNTER_TYPE \ No newline at end of file + return COUNTER_TYPE diff --git a/metaflow/multicore_utils.py b/metaflow/multicore_utils.py index c56de2c9eea..6bd8836abd5 100644 --- a/metaflow/multicore_utils.py +++ b/metaflow/multicore_utils.py @@ -22,13 +22,13 @@ # introducing an external dependency like joblib. # 3) Supports closures and lambdas in contrast to multiprocessing. + class MulticoreException(Exception): pass + def _spawn(func, arg, dir): - with NamedTemporaryFile(prefix='parallel_map_', - dir=dir, - delete=False) as tmpfile: + with NamedTemporaryFile(prefix="parallel_map_", dir=dir, delete=False) as tmpfile: output_file = tmpfile.name # make sure stdout and stderr are flushed before forking. Otherwise @@ -42,7 +42,7 @@ def _spawn(func, arg, dir): try: exit_code = 1 ret = func(arg) - with open(output_file, 'wb') as f: + with open(output_file, "wb") as f: pickle.dump(ret, f, protocol=pickle.HIGHEST_PROTOCOL) exit_code = 0 except: @@ -57,6 +57,7 @@ def _spawn(func, arg, dir): # finally blocks). os._exit(exit_code) + def parallel_imap_unordered(func, iterable, max_parallel=None, dir=None): if max_parallel is None: @@ -64,16 +65,15 @@ def parallel_imap_unordered(func, iterable, max_parallel=None, dir=None): ret = [] args_iter = iter(iterable) - pids = [_spawn(func, arg, dir) - for arg in islice(args_iter, max_parallel)] + pids = [_spawn(func, arg, dir) for arg in islice(args_iter, max_parallel)] while pids: pid, output_file = pids.pop() if os.waitpid(pid, 0)[1]: - raise MulticoreException('Child failed') + raise MulticoreException("Child failed") - with open(output_file, 'rb') as f: + with open(output_file, "rb") as f: yield pickle.load(f) os.remove(output_file) @@ -81,8 +81,8 @@ def parallel_imap_unordered(func, iterable, max_parallel=None, dir=None): if arg: pids.insert(0, _spawn(func, arg[0], dir)) -def parallel_map(func, iterable, **kwargs): +def parallel_map(func, iterable, **kwargs): def wrapper(arg_with_idx): idx, arg = arg_with_idx return idx, func(arg) diff --git a/metaflow/package.py b/metaflow/package.py index dc3d18d8339..d181705284e 100644 --- a/metaflow/package.py +++ b/metaflow/package.py @@ -10,11 +10,10 @@ from .util import to_unicode from . import R -DEFAULT_SUFFIXES_LIST = DEFAULT_PACKAGE_SUFFIXES.split(',') +DEFAULT_SUFFIXES_LIST = DEFAULT_PACKAGE_SUFFIXES.split(",") class MetaflowPackage(object): - def __init__(self, flow, environment, echo, suffixes=DEFAULT_SUFFIXES_LIST): self.suffixes = list(set().union(suffixes, DEFAULT_SUFFIXES_LIST)) self.environment = environment @@ -24,37 +23,38 @@ def __init__(self, flow, environment, echo, suffixes=DEFAULT_SUFFIXES_LIST): except ImportError: self.metaflow_extensions_root = None else: - self.metaflow_extensions_root = os.path.dirname(metaflow_extensions.__file__) + self.metaflow_extensions_root = os.path.dirname( + metaflow_extensions.__file__ + ) self.metaflow_extensions_addl_suffixes = getattr( - metaflow_extensions, - 'METAFLOW_EXTENSIONS_PACKAGE_SUFFIXES', - None) + metaflow_extensions, "METAFLOW_EXTENSIONS_PACKAGE_SUFFIXES", None + ) self.flow_name = flow.name self.create_time = time.time() environment.init_environment(echo) for step in flow: for deco in step.decorators: - deco.package_init(flow, - step.__name__, - environment) + deco.package_init(flow, step.__name__, environment) self.blob = self._make() def _walk(self, root, exclude_hidden=True, addl_suffixes=None): if addl_suffixes is None: addl_suffixes = [] root = to_unicode(root) # handle files/folder with non ascii chars - prefixlen = len('%s/' % os.path.dirname(root)) + prefixlen = len("%s/" % os.path.dirname(root)) for path, dirs, files in os.walk(root): - if exclude_hidden and '/.' in path: + if exclude_hidden and "/." in path: continue # path = path[2:] # strip the ./ prefix # if path and (path[0] == '.' or './' in path): # continue for fname in files: - if fname[0] == '.': + if fname[0] == ".": continue - if any(fname.endswith(suffix) for suffix in self.suffixes + addl_suffixes): + if any( + fname.endswith(suffix) for suffix in self.suffixes + addl_suffixes + ): p = os.path.join(path, fname) yield p, p[prefixlen:] @@ -70,31 +70,32 @@ def path_tuples(self): # Metaflow customization if any if self.metaflow_extensions_root: for path_tuple in self._walk( - self.metaflow_extensions_root, - exclude_hidden=False, - addl_suffixes=self.metaflow_extensions_addl_suffixes): + self.metaflow_extensions_root, + exclude_hidden=False, + addl_suffixes=self.metaflow_extensions_addl_suffixes, + ): yield path_tuple # the package folders for environment for path_tuple in self.environment.add_to_package(): yield path_tuple if R.use_r(): # the R working directory - for path_tuple in self._walk('%s/' % R.working_dir()): + for path_tuple in self._walk("%s/" % R.working_dir()): yield path_tuple # the R package for path_tuple in R.package_paths(): yield path_tuple else: # the user's working directory - flowdir = os.path.dirname(os.path.abspath(sys.argv[0])) + '/' + flowdir = os.path.dirname(os.path.abspath(sys.argv[0])) + "/" for path_tuple in self._walk(flowdir): yield path_tuple def _add_info(self, tar): - info = tarfile.TarInfo('INFO') + info = tarfile.TarInfo("INFO") env = self.environment.get_environment_info() buf = BytesIO() - buf.write(json.dumps(env).encode('utf-8')) + buf.write(json.dumps(env).encode("utf-8")) buf.seek(0) info.size = len(buf.getvalue()) tar.addfile(info, buf) @@ -107,16 +108,17 @@ def no_mtime(tarinfo): return tarinfo buf = BytesIO() - with tarfile.open(fileobj=buf, mode='w:gz', compresslevel=3) as tar: + with tarfile.open(fileobj=buf, mode="w:gz", compresslevel=3) as tar: self._add_info(tar) for path, arcname in self.path_tuples(): - tar.add(path, arcname=arcname, - recursive=False, filter=no_mtime) + tar.add(path, arcname=arcname, recursive=False, filter=no_mtime) blob = bytearray(buf.getvalue()) - blob[4:8] = [0] * 4 # Reset 4 bytes from offset 4 to account for ts + blob[4:8] = [0] * 4 # Reset 4 bytes from offset 4 to account for ts return blob def __str__(self): - return '' % \ - (self.flow_name, time.strftime("%a, %d %b %Y %H:%M:%S", self.create_time)) + return "" % ( + self.flow_name, + time.strftime("%a, %d %b %Y %H:%M:%S", self.create_time), + ) diff --git a/metaflow/parameters.py b/metaflow/parameters.py index 7f0df8088ca..d84e362c3d7 100644 --- a/metaflow/parameters.py +++ b/metaflow/parameters.py @@ -4,9 +4,11 @@ import click from .util import get_username, is_stringish -from .exception import ParameterFieldFailed,\ - ParameterFieldTypeMismatch,\ - MetaflowException +from .exception import ( + ParameterFieldFailed, + ParameterFieldTypeMismatch, + MetaflowException, +) try: # Python2 @@ -18,12 +20,10 @@ # ParameterContext allows deploy-time functions modify their # behavior based on the context. We can add fields here without # breaking backwards compatibility but don't remove any fields! -ParameterContext = namedtuple('ParameterContext', - ['flow_name', - 'user_name', - 'parameter_name', - 'logger', - 'ds_type']) +ParameterContext = namedtuple( + "ParameterContext", + ["flow_name", "user_name", "parameter_name", "logger", "ds_type"], +) # currently we execute only one flow per process, so we can treat # Parameters globally. If this was to change, it should/might be @@ -32,8 +32,9 @@ parameters = [] context_proto = None + class JSONTypeClass(click.ParamType): - name = 'JSON' + name = "JSON" def convert(self, value, param, ctx): if not isinstance(value, strtype): @@ -48,7 +49,8 @@ def __str__(self): return repr(self) def __repr__(self): - return 'JSON' + return "JSON" + class DeployTimeField(object): """ @@ -59,20 +61,25 @@ class DeployTimeField(object): object curries the context argument for the function, and pretty prints any exceptions that occur during evaluation. """ - def __init__(self, - parameter_name, - parameter_type, - field, - fun, - return_str=True, - print_representation=None): + + def __init__( + self, + parameter_name, + parameter_type, + field, + fun, + return_str=True, + print_representation=None, + ): self.fun = fun self.field = field self.parameter_name = parameter_name self.parameter_type = parameter_type self.return_str = return_str - self.print_representation = self.user_print_representation = print_representation + self.print_representation = ( + self.user_print_representation + ) = print_representation if self.print_representation is None: self.print_representation = str(self.fun) @@ -99,23 +106,22 @@ def _check_type(self, val): # note: this doesn't work with long in Python2 or types defined as # click types, e.g. click.INT - TYPES = {bool: 'bool', - int: 'int', - float: 'float', - list: 'list'} + TYPES = {bool: "bool", int: "int", float: "float", list: "list"} - msg = "The value returned by the deploy-time function for "\ - "the parameter *%s* field *%s* has a wrong type. " %\ - (self.parameter_name, self.field) + msg = ( + "The value returned by the deploy-time function for " + "the parameter *%s* field *%s* has a wrong type. " + % (self.parameter_name, self.field) + ) if self.parameter_type in TYPES: if type(val) != self.parameter_type: - msg += 'Expected a %s.' % TYPES[self.parameter_type] + msg += "Expected a %s." % TYPES[self.parameter_type] raise ParameterFieldTypeMismatch(msg) return str(val) if self.return_str else val else: if not is_stringish(val): - msg += 'Expected a string.' + msg += "Expected a string." raise ParameterFieldTypeMismatch(msg) return val @@ -140,66 +146,70 @@ def deploy_time_eval(value): else: return value + # this is called by cli.main def set_parameter_context(flow_name, echo, datastore): global context_proto - context_proto = ParameterContext(flow_name=flow_name, - user_name=get_username(), - parameter_name=None, - logger=echo, - ds_type=datastore.TYPE) + context_proto = ParameterContext( + flow_name=flow_name, + user_name=get_username(), + parameter_name=None, + logger=echo, + ds_type=datastore.TYPE, + ) + class Parameter(object): def __init__(self, name, **kwargs): self.name = name self.kwargs = kwargs # TODO: check that the type is one of the supported types - param_type = self.kwargs['type'] = self._get_type(kwargs) + param_type = self.kwargs["type"] = self._get_type(kwargs) - if self.name == 'params': - raise MetaflowException("Parameter name 'params' is a reserved " - "word. Please use a different " - "name for your parameter.") + if self.name == "params": + raise MetaflowException( + "Parameter name 'params' is a reserved " + "word. Please use a different " + "name for your parameter." + ) # make sure the user is not trying to pass a function in one of the # fields that don't support function-values yet - for field in ('show_default', - 'separator', - 'required'): + for field in ("show_default", "separator", "required"): if callable(kwargs.get(field)): - raise MetaflowException("Parameter *%s*: Field '%s' cannot " - "have a function as its value"\ - % (name, field)) + raise MetaflowException( + "Parameter *%s*: Field '%s' cannot " + "have a function as its value" % (name, field) + ) - self.kwargs['show_default'] = self.kwargs.get('show_default', True) + self.kwargs["show_default"] = self.kwargs.get("show_default", True) # default can be defined as a function - default_field = self.kwargs.get('default') + default_field = self.kwargs.get("default") if callable(default_field) and not isinstance(default_field, DeployTimeField): - self.kwargs['default'] = DeployTimeField(name, - param_type, - 'default', - self.kwargs['default'], - return_str=True) + self.kwargs["default"] = DeployTimeField( + name, param_type, "default", self.kwargs["default"], return_str=True + ) # note that separator doesn't work with DeployTimeFields unless you # specify type=str - self.separator = self.kwargs.pop('separator', None) + self.separator = self.kwargs.pop("separator", None) if self.separator and not self.is_string_type: - raise MetaflowException("Parameter *%s*: Separator is only allowed " - "for string parameters." % name) + raise MetaflowException( + "Parameter *%s*: Separator is only allowed " + "for string parameters." % name + ) parameters.append(self) def option_kwargs(self, deploy_mode): kwargs = self.kwargs - if isinstance(kwargs.get('default'), DeployTimeField) and not deploy_mode: + if isinstance(kwargs.get("default"), DeployTimeField) and not deploy_mode: ret = dict(kwargs) - help_msg = kwargs.get('help') - help_msg = '' if help_msg is None else help_msg - ret['help'] = help_msg + \ - "[default: deploy-time value of '%s']" % self.name - ret['default'] = None - ret['required'] = False + help_msg = kwargs.get("help") + help_msg = "" if help_msg is None else help_msg + ret["help"] = help_msg + "[default: deploy-time value of '%s']" % self.name + ret["default"] = None + ret["required"] = False return ret else: return kwargs @@ -210,22 +220,24 @@ def load_parameter(self, v): def _get_type(self, kwargs): default_type = str - default = kwargs.get('default') + default = kwargs.get("default") if default is not None and not callable(default): default_type = type(default) - return kwargs.get('type', default_type) + return kwargs.get("type", default_type) @property def is_string_type(self): - return self.kwargs.get('type', str) == str and\ - isinstance(self.kwargs.get('default', ''), strtype) + return self.kwargs.get("type", str) == str and isinstance( + self.kwargs.get("default", ""), strtype + ) # this is needed to appease Pylint for JSONType'd parameters, # which may do self.param['foobar'] def __getitem__(self, x): pass + def add_custom_parameters(deploy_mode=False): # deploy_mode determines whether deploy-time functions should or should # not be evaluated for this command @@ -234,23 +246,27 @@ def wrapper(cmd): # in the order they are defined in the FlowSpec subclass for arg in parameters[::-1]: kwargs = arg.option_kwargs(deploy_mode) - cmd.params.insert(0, click.Option(('--' + arg.name,), **kwargs)) + cmd.params.insert(0, click.Option(("--" + arg.name,), **kwargs)) return cmd + return wrapper + def set_parameters(flow, kwargs): seen = set() for var, param in flow._get_parameters(): norm = param.name.lower() if norm in seen: - raise MetaflowException("Parameter *%s* is specified twice. " - "Note that parameter names are " - "case-insensitive." % param.name) + raise MetaflowException( + "Parameter *%s* is specified twice. " + "Note that parameter names are " + "case-insensitive." % param.name + ) seen.add(norm) flow._success = True for var, param in flow._get_parameters(): - val = kwargs[param.name.replace('-', '_').lower()] + val = kwargs[param.name.replace("-", "_").lower()] # Support for delayed evaluation of parameters. This is used for # includefile in particular if callable(val): diff --git a/metaflow/plugins/__init__.py b/metaflow/plugins/__init__.py index 00352a69b88..086a0c57dae 100644 --- a/metaflow/plugins/__init__.py +++ b/metaflow/plugins/__init__.py @@ -2,15 +2,15 @@ import types _expected_extensions = { - 'FLOW_DECORATORS': [], - 'STEP_DECORATORS': [], - 'ENVIRONMENTS': [], - 'METADATA_PROVIDERS': [], - 'SIDECARS': {}, - 'LOGGING_SIDECARS': {}, - 'MONITOR_SIDECARS': {}, - 'AWS_CLIENT_PROVIDERS': [], - 'get_plugin_cli': lambda : [] + "FLOW_DECORATORS": [], + "STEP_DECORATORS": [], + "ENVIRONMENTS": [], + "METADATA_PROVIDERS": [], + "SIDECARS": {}, + "LOGGING_SIDECARS": {}, + "MONITOR_SIDECARS": {}, + "AWS_CLIENT_PROVIDERS": [], + "get_plugin_cli": lambda: [], } try: @@ -21,12 +21,16 @@ # e.name is set to the name of the package that fails to load # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) - if not (isinstance(e, ModuleNotFoundError) and \ - e.name in ['metaflow_extensions', 'metaflow_extensions.plugins']): + if not ( + isinstance(e, ModuleNotFoundError) + and e.name in ["metaflow_extensions", "metaflow_extensions.plugins"] + ): print( "Cannot load metaflow_extensions plugins -- " - "if you want to ignore, uninstall metaflow_extensions package") + "if you want to ignore, uninstall metaflow_extensions package" + ) raise + class _fake(object): def __getattr__(self, name): if name in _expected_extensions: @@ -40,19 +44,23 @@ def __getattr__(self, name): # *except* for ones that are part of metaflow_extensions (basically providing # an aliasing mechanism) lazy_load_custom_modules = {} - addl_modules = _ext_plugins.__dict__.get('__mf_promote_submodules__') + addl_modules = _ext_plugins.__dict__.get("__mf_promote_submodules__") if addl_modules: # We make an alias for these modules which the metaflow_extensions author # wants to expose but that may not be loaded yet lazy_load_custom_modules = { - 'metaflow.plugins.%s' % k: 'metaflow_extensions.plugins.%s' % k - for k in addl_modules} + "metaflow.plugins.%s" % k: "metaflow_extensions.plugins.%s" % k + for k in addl_modules + } for n, o in _ext_plugins.__dict__.items(): - if not n.startswith('__') and not isinstance(o, types.ModuleType): + if not n.startswith("__") and not isinstance(o, types.ModuleType): globals()[n] = o - elif isinstance(o, types.ModuleType) and o.__package__ and \ - o.__package__.startswith('metaflow_extensions'): - lazy_load_custom_modules['metaflow.plugins.%s' % n] = o + elif ( + isinstance(o, types.ModuleType) + and o.__package__ + and o.__package__.startswith("metaflow_extensions") + ): + lazy_load_custom_modules["metaflow.plugins.%s" % n] = o if lazy_load_custom_modules: # NOTE: We load things first to have metaflow_extensions override things here. # This does mean that for modules that have the same name (for example, @@ -64,8 +72,9 @@ def __getattr__(self, name): # load the non metaflow_extensions modules providing for possible confusion. # This keeps it cleaner. from metaflow import _LazyLoader + sys.meta_path = [_LazyLoader(lazy_load_custom_modules)] + sys.meta_path - + class _wrap(object): def __init__(self, obj): self.__dict__ = obj.__dict__ @@ -78,7 +87,6 @@ def __getattr__(self, name): _ext_plugins = _wrap(_ext_plugins) - def get_plugin_cli(): # it is important that CLIs are not imported when # __init__ is imported. CLIs may use e.g. @@ -95,7 +103,8 @@ def get_plugin_cli(): package_cli.cli, batch_cli.cli, kubernetes_cli.cli, - step_functions_cli.cli] + step_functions_cli.cli, + ] def _merge_lists(base, overrides, attr): @@ -108,6 +117,7 @@ def _merge_lists(base, overrides, attr): l.extend([d for d in base if getattr(d, attr) not in existing]) return l + # Add new decorators in this list from .catch_decorator import CatchDecorator from .timeout_decorator import TimeoutDecorator @@ -116,33 +126,43 @@ def _merge_lists(base, overrides, attr): from .resources_decorator import ResourcesDecorator from .aws.batch.batch_decorator import BatchDecorator from .aws.eks.kubernetes_decorator import KubernetesDecorator -from .aws.step_functions.step_functions_decorator \ - import StepFunctionsInternalDecorator -from .test_unbounded_foreach_decorator\ - import InternalTestUnboundedForeachDecorator, InternalTestUnboundedForeachInput +from .aws.step_functions.step_functions_decorator import StepFunctionsInternalDecorator +from .test_unbounded_foreach_decorator import ( + InternalTestUnboundedForeachDecorator, + InternalTestUnboundedForeachInput, +) from .conda.conda_step_decorator import CondaStepDecorator -STEP_DECORATORS = _merge_lists([CatchDecorator, - TimeoutDecorator, - EnvironmentDecorator, - ResourcesDecorator, - RetryDecorator, - BatchDecorator, - KubernetesDecorator, - StepFunctionsInternalDecorator, - CondaStepDecorator, - InternalTestUnboundedForeachDecorator], - _ext_plugins.STEP_DECORATORS, 'name') +STEP_DECORATORS = _merge_lists( + [ + CatchDecorator, + TimeoutDecorator, + EnvironmentDecorator, + ResourcesDecorator, + RetryDecorator, + BatchDecorator, + KubernetesDecorator, + StepFunctionsInternalDecorator, + CondaStepDecorator, + InternalTestUnboundedForeachDecorator, + ], + _ext_plugins.STEP_DECORATORS, + "name", +) # Add Conda environment from .conda.conda_environment import CondaEnvironment -ENVIRONMENTS = _merge_lists([CondaEnvironment], _ext_plugins.ENVIRONMENTS, 'TYPE') + +ENVIRONMENTS = _merge_lists([CondaEnvironment], _ext_plugins.ENVIRONMENTS, "TYPE") # Metadata providers from .metadata import LocalMetadataProvider, ServiceMetadataProvider METADATA_PROVIDERS = _merge_lists( - [LocalMetadataProvider, ServiceMetadataProvider], _ext_plugins.METADATA_PROVIDERS, 'TYPE') + [LocalMetadataProvider, ServiceMetadataProvider], + _ext_plugins.METADATA_PROVIDERS, + "TYPE", +) # Every entry in this list becomes a class-level flow decorator. # Add an entry here if you need a new flow-level annotation. Be @@ -151,46 +171,62 @@ def _merge_lists(base, overrides, attr): from .conda.conda_flow_decorator import CondaFlowDecorator from .aws.step_functions.schedule_decorator import ScheduleDecorator from .project_decorator import ProjectDecorator -FLOW_DECORATORS = _merge_lists([CondaFlowDecorator, - ScheduleDecorator, - ProjectDecorator], - _ext_plugins.FLOW_DECORATORS, 'name') + +FLOW_DECORATORS = _merge_lists( + [CondaFlowDecorator, ScheduleDecorator, ProjectDecorator], + _ext_plugins.FLOW_DECORATORS, + "name", +) # Sidecars from ..mflog.save_logs_periodically import SaveLogsPeriodicallySidecar from metaflow.metadata.heartbeat import MetadataHeartBeat -SIDECARS = {'save_logs_periodically': SaveLogsPeriodicallySidecar, - 'heartbeat': MetadataHeartBeat} +SIDECARS = { + "save_logs_periodically": SaveLogsPeriodicallySidecar, + "heartbeat": MetadataHeartBeat, +} SIDECARS.update(_ext_plugins.SIDECARS) # Add logger from .debug_logger import DebugEventLogger -LOGGING_SIDECARS = {'debugLogger': DebugEventLogger, - 'nullSidecarLogger': None} + +LOGGING_SIDECARS = {"debugLogger": DebugEventLogger, "nullSidecarLogger": None} LOGGING_SIDECARS.update(_ext_plugins.LOGGING_SIDECARS) # Add monitor from .debug_monitor import DebugMonitor -MONITOR_SIDECARS = {'debugMonitor': DebugMonitor, - 'nullSidecarMonitor': None} + +MONITOR_SIDECARS = {"debugMonitor": DebugMonitor, "nullSidecarMonitor": None} MONITOR_SIDECARS.update(_ext_plugins.MONITOR_SIDECARS) SIDECARS.update(LOGGING_SIDECARS) SIDECARS.update(MONITOR_SIDECARS) from .aws.aws_client import Boto3ClientProvider + AWS_CLIENT_PROVIDERS = _merge_lists( - [Boto3ClientProvider], _ext_plugins.AWS_CLIENT_PROVIDERS, 'name') + [Boto3ClientProvider], _ext_plugins.AWS_CLIENT_PROVIDERS, "name" +) # Erase all temporary names to avoid leaking things # We leave '_ext_plugins' and '_expected_extensions' because they are used in # a function (so they need to stick around) -for _n in ['ver', 'n', 'o', 'e', 'lazy_load_custom_modules', '_LazyLoader', - '_merge_lists', '_fake', '_wrap', 'addl_modules']: +for _n in [ + "ver", + "n", + "o", + "e", + "lazy_load_custom_modules", + "_LazyLoader", + "_merge_lists", + "_fake", + "_wrap", + "addl_modules", +]: try: del globals()[_n] except KeyError: pass -del globals()['_n'] +del globals()["_n"] diff --git a/metaflow/plugins/aws/aws_client.py b/metaflow/plugins/aws/aws_client.py index d3f4ac7eaed..275f236213f 100644 --- a/metaflow/plugins/aws/aws_client.py +++ b/metaflow/plugins/aws/aws_client.py @@ -8,24 +8,27 @@ class Boto3ClientProvider(object): @staticmethod def get_client(module, with_error=False, params={}): from metaflow.exception import MetaflowException - from metaflow.metaflow_config import AWS_SANDBOX_ENABLED, \ - AWS_SANDBOX_STS_ENDPOINT_URL, AWS_SANDBOX_API_KEY + from metaflow.metaflow_config import ( + AWS_SANDBOX_ENABLED, + AWS_SANDBOX_STS_ENDPOINT_URL, + AWS_SANDBOX_API_KEY, + ) import requests + try: import boto3 from botocore.exceptions import ClientError except (NameError, ImportError): raise MetaflowException( - "Could not import module 'boto3'. Install boto3 first.") + "Could not import module 'boto3'. Install boto3 first." + ) if AWS_SANDBOX_ENABLED: global cached_aws_sandbox_creds if cached_aws_sandbox_creds is None: # authenticate using STS url = "%s/auth/token" % AWS_SANDBOX_STS_ENDPOINT_URL - headers = { - 'x-api-key': AWS_SANDBOX_API_KEY - } + headers = {"x-api-key": AWS_SANDBOX_API_KEY} try: r = requests.get(url, headers=headers) r.raise_for_status() @@ -33,10 +36,15 @@ def get_client(module, with_error=False, params={}): except requests.exceptions.HTTPError as e: raise MetaflowException(repr(e)) if with_error: - return boto3.session.Session( - **cached_aws_sandbox_creds).client(module, **params), ClientError - return boto3.session.Session( - **cached_aws_sandbox_creds).client(module, **params) + return ( + boto3.session.Session(**cached_aws_sandbox_creds).client( + module, **params + ), + ClientError, + ) + return boto3.session.Session(**cached_aws_sandbox_creds).client( + module, **params + ) if with_error: return boto3.client(module, **params), ClientError return boto3.client(module, **params) @@ -47,11 +55,13 @@ def get_aws_client(module, with_error=False, params={}): if cached_provider_class is None: from metaflow.metaflow_config import DEFAULT_AWS_CLIENT_PROVIDER from metaflow.plugins import AWS_CLIENT_PROVIDERS + for p in AWS_CLIENT_PROVIDERS: if p.name == DEFAULT_AWS_CLIENT_PROVIDER: cached_provider_class = p break else: - raise ValueError("Cannot find AWS Client provider %s" - % DEFAULT_AWS_CLIENT_PROVIDER) + raise ValueError( + "Cannot find AWS Client provider %s" % DEFAULT_AWS_CLIENT_PROVIDER + ) return cached_provider_class.get_client(module, with_error, params) diff --git a/metaflow/plugins/aws/aws_utils.py b/metaflow/plugins/aws/aws_utils.py index 2ff547c9860..09a6ddc9023 100644 --- a/metaflow/plugins/aws/aws_utils.py +++ b/metaflow/plugins/aws/aws_utils.py @@ -1,5 +1,6 @@ import re + def get_docker_registry(image_uri): """ Explanation: @@ -48,4 +49,4 @@ def get_docker_registry(image_uri): registry, repository, tag = pattern.match(image_uri).groups() if registry is not None: registry = registry.rstrip("/") - return registry \ No newline at end of file + return registry diff --git a/metaflow/plugins/aws/batch/batch.py b/metaflow/plugins/aws/batch/batch.py index 07eac64e259..833e97801a7 100644 --- a/metaflow/plugins/aws/batch/batch.py +++ b/metaflow/plugins/aws/batch/batch.py @@ -15,7 +15,7 @@ DATASTORE_SYSROOT_S3, DEFAULT_METADATA, BATCH_METADATA_SERVICE_HEADERS, - BATCH_EMIT_TAGS + BATCH_EMIT_TAGS, ) from metaflow.mflog.mflog import refine, set_should_persist from metaflow.mflog import ( @@ -48,13 +48,9 @@ def __init__(self, metadata, environment): self.metadata = metadata self.environment = environment self._client = BatchClient() - atexit.register( - lambda: self.job.kill() if hasattr(self, "job") else None - ) + atexit.register(lambda: self.job.kill() if hasattr(self, "job") else None) - def _command( - self, environment, code_package_url, step_name, step_cmds, task_spec - ): + def _command(self, environment, code_package_url, step_name, step_cmds, task_spec): mflog_expr = export_mflog_env_vars( datastore_type="s3", stdout_path=STDOUT_PATH, @@ -111,9 +107,7 @@ def _search_jobs(self, flow_name, run_id, user): if match: yield job - def _job_name( - self, user, flow_name, run_id, step_name, task_id, retry_count - ): + def _job_name(self, user, flow_name, run_id, step_name, task_id, retry_count): return "{user}-{flow_name}-{run_id}-{step_name}-{task_id}-{retry_count}".format( user=user, flow_name=flow_name, @@ -194,36 +188,48 @@ def create_job( .job_name(job_name) .job_queue(queue) .command( - self._command(self.environment, code_package_url, - step_name, [step_cli], task_spec)) \ - .image(image) \ - .iam_role(iam_role) \ - .execution_role(execution_role) \ - .job_def(image, iam_role, - queue, execution_role, shared_memory, - max_swap, swappiness, host_volumes=host_volumes) \ - .cpu(cpu) \ - .gpu(gpu) \ - .memory(memory) \ - .shared_memory(shared_memory) \ - .max_swap(max_swap) \ - .swappiness(swappiness) \ - .timeout_in_secs(run_time_limit) \ - .environment_variable('AWS_DEFAULT_REGION', self._client.region()) \ - .environment_variable('METAFLOW_CODE_SHA', code_package_sha) \ - .environment_variable('METAFLOW_CODE_URL', code_package_url) \ - .environment_variable('METAFLOW_CODE_DS', code_package_ds) \ - .environment_variable('METAFLOW_USER', attrs['metaflow.user']) \ - .environment_variable('METAFLOW_SERVICE_URL', BATCH_METADATA_SERVICE_URL) \ - .environment_variable('METAFLOW_SERVICE_HEADERS', json.dumps(BATCH_METADATA_SERVICE_HEADERS)) \ - .environment_variable('METAFLOW_DATASTORE_SYSROOT_S3', DATASTORE_SYSROOT_S3) \ - .environment_variable('METAFLOW_DATATOOLS_S3ROOT', DATATOOLS_S3ROOT) \ - .environment_variable('METAFLOW_DEFAULT_DATASTORE', 's3') \ - .environment_variable('METAFLOW_DEFAULT_METADATA', DEFAULT_METADATA)) - # Skip setting METAFLOW_DATASTORE_SYSROOT_LOCAL because metadata sync between the local user - # instance and the remote AWS Batch instance assumes metadata is stored in DATASTORE_LOCAL_DIR - # on the remote AWS Batch instance; this happens when METAFLOW_DATASTORE_SYSROOT_LOCAL - # is NOT set (see get_datastore_root_from_config in datastore/local.py). + self._command( + self.environment, code_package_url, step_name, [step_cli], task_spec + ) + ) + .image(image) + .iam_role(iam_role) + .execution_role(execution_role) + .job_def( + image, + iam_role, + queue, + execution_role, + shared_memory, + max_swap, + swappiness, + host_volumes=host_volumes, + ) + .cpu(cpu) + .gpu(gpu) + .memory(memory) + .shared_memory(shared_memory) + .max_swap(max_swap) + .swappiness(swappiness) + .timeout_in_secs(run_time_limit) + .environment_variable("AWS_DEFAULT_REGION", self._client.region()) + .environment_variable("METAFLOW_CODE_SHA", code_package_sha) + .environment_variable("METAFLOW_CODE_URL", code_package_url) + .environment_variable("METAFLOW_CODE_DS", code_package_ds) + .environment_variable("METAFLOW_USER", attrs["metaflow.user"]) + .environment_variable("METAFLOW_SERVICE_URL", BATCH_METADATA_SERVICE_URL) + .environment_variable( + "METAFLOW_SERVICE_HEADERS", json.dumps(BATCH_METADATA_SERVICE_HEADERS) + ) + .environment_variable("METAFLOW_DATASTORE_SYSROOT_S3", DATASTORE_SYSROOT_S3) + .environment_variable("METAFLOW_DATATOOLS_S3ROOT", DATATOOLS_S3ROOT) + .environment_variable("METAFLOW_DEFAULT_DATASTORE", "s3") + .environment_variable("METAFLOW_DEFAULT_METADATA", DEFAULT_METADATA) + ) + # Skip setting METAFLOW_DATASTORE_SYSROOT_LOCAL because metadata sync between the local user + # instance and the remote AWS Batch instance assumes metadata is stored in DATASTORE_LOCAL_DIR + # on the remote AWS Batch instance; this happens when METAFLOW_DATASTORE_SYSROOT_LOCAL + # is NOT set (see get_datastore_root_from_config in datastore/local.py). for name, value in env.items(): job.environment_variable(name, value) if attrs: @@ -231,10 +237,16 @@ def create_job( job.parameter(key, value) # Tags for AWS Batch job (for say cost attribution) if BATCH_EMIT_TAGS: - for key in ['metaflow.flow_name', 'metaflow.run_id', - 'metaflow.step_name', 'metaflow.version', - 'metaflow.run_id.$', 'metaflow.user', - 'metaflow.owner', 'metaflow.production_token']: + for key in [ + "metaflow.flow_name", + "metaflow.run_id", + "metaflow.step_name", + "metaflow.version", + "metaflow.run_id.$", + "metaflow.user", + "metaflow.owner", + "metaflow.production_token", + ]: if key in attrs: job.tag(key, attrs.get(key)) return job @@ -270,26 +282,26 @@ def launch_job( " specified and no valid & enabled queue found." ) job = self.create_job( - step_name, - step_cli, - task_spec, - code_package_sha, - code_package_url, - code_package_ds, - image, - queue, - iam_role, - execution_role, - cpu, - gpu, - memory, - run_time_limit, - shared_memory, - max_swap, - swappiness, - env=env, - attrs=attrs, - host_volumes=host_volumes + step_name, + step_cli, + task_spec, + code_package_sha, + code_package_url, + code_package_ds, + image, + queue, + iam_role, + execution_role, + cpu, + gpu, + memory, + run_time_limit, + shared_memory, + max_swap, + swappiness, + env=env, + attrs=attrs, + host_volumes=host_volumes, ) self.job = job.execute() @@ -385,9 +397,7 @@ def _print_available(tail, stream, should_persist=False): if msg is not None ) raise BatchException( - "%s " - "This could be a transient error. " - "Use @retry to retry." % msg + "%s " "This could be a transient error. " "Use @retry to retry." % msg ) else: if self.job.is_running: diff --git a/metaflow/plugins/aws/batch/batch_cli.py b/metaflow/plugins/aws/batch/batch_cli.py index b5640874a0f..a838472e2a2 100644 --- a/metaflow/plugins/aws/batch/batch_cli.py +++ b/metaflow/plugins/aws/batch/batch_cli.py @@ -15,6 +15,7 @@ from .batch import Batch, BatchKilledException + @click.group() def cli(): pass @@ -43,9 +44,7 @@ def _execute_cmd(func, flow_name, run_id, user, my_runs, echo): if not run_id and latest_run: run_id = util.get_latest_run_id(echo, flow_name) if run_id is None: - raise CommandException( - "A previous run id was not found. Specify --run-id." - ) + raise CommandException("A previous run id was not found. Specify --run-id.") func(flow_name, run_id, user, echo) @@ -57,9 +56,7 @@ def _execute_cmd(func, flow_name, run_id, user, my_runs, echo): is_flag=True, help="List all my unfinished tasks.", ) -@click.option( - "--user", default=None, help="List unfinished tasks for the given user." -) +@click.option("--user", default=None, help="List unfinished tasks for the given user.") @click.option( "--run-id", default=None, @@ -130,12 +127,8 @@ def kill(ctx, run_id, user, my_runs): @click.option( "--tag", multiple=True, default=None, help="Passed to the top-level 'step'." ) -@click.option( - "--namespace", default=None, help="Passed to the top-level 'step'." -) -@click.option( - "--retry-count", default=0, help="Passed to the top-level 'step'." -) +@click.option("--namespace", default=None, help="Passed to the top-level 'step'.") +@click.option("--retry-count", default=0, help="Passed to the top-level 'step'.") @click.option( "--max-user-code-retries", default=0, help="Passed to the top-level 'step'." ) @@ -144,14 +137,12 @@ def kill(ctx, run_id, user, my_runs): default=5 * 24 * 60 * 60, help="Run time limit in seconds for the AWS Batch job. Default is 5 days.", ) -@click.option( - "--shared-memory", help="Shared Memory requirement for AWS Batch." -) +@click.option("--shared-memory", help="Shared Memory requirement for AWS Batch.") @click.option("--max-swap", help="Max Swap requirement for AWS Batch.") @click.option("--swappiness", help="Swappiness requirement for AWS Batch.") -#TODO: Maybe remove it altogether since it's not used here -@click.option('--ubf-context', default=None, type=click.Choice([None])) -@click.option('--host-volumes', multiple=True) +# TODO: Maybe remove it altogether since it's not used here +@click.option("--ubf-context", default=None, type=click.Choice([None])) +@click.option("--host-volumes", multiple=True) @click.pass_context def step( ctx, @@ -193,8 +184,7 @@ def echo(msg, stream="stderr", batch_id=None): if input_paths: max_size = 30 * 1024 split_vars = { - "METAFLOW_INPUT_PATHS_%d" - % (i // max_size): input_paths[i : i + max_size] + "METAFLOW_INPUT_PATHS_%d" % (i // max_size): input_paths[i : i + max_size] for i in range(0, len(input_paths), max_size) } kwargs["input_paths"] = "".join("${%s}" % s for s in split_vars.keys()) @@ -250,22 +240,23 @@ def echo(msg, stream="stderr", batch_id=None): # this information is needed for log tailing ds = ctx.obj.flow_datastore.get_task_datastore( - mode='w', - run_id=kwargs['run_id'], + mode="w", + run_id=kwargs["run_id"], step_name=step_name, - task_id=kwargs['task_id'], - attempt=int(retry_count) + task_id=kwargs["task_id"], + attempt=int(retry_count), ) - stdout_location = ds.get_log_location(TASK_LOG_SOURCE, 'stdout') - stderr_location = ds.get_log_location(TASK_LOG_SOURCE, 'stderr') + stdout_location = ds.get_log_location(TASK_LOG_SOURCE, "stdout") + stderr_location = ds.get_log_location(TASK_LOG_SOURCE, "stderr") def _sync_metadata(): - if ctx.obj.metadata.TYPE == 'local': + if ctx.obj.metadata.TYPE == "local": sync_local_metadata_from_datastore( - DATASTORE_LOCAL_DIR, - ctx.obj.flow_datastore.get_task_datastore(kwargs['run_id'], - step_name, - kwargs['task_id'])) + DATASTORE_LOCAL_DIR, + ctx.obj.flow_datastore.get_task_datastore( + kwargs["run_id"], step_name, kwargs["task_id"] + ), + ) batch = Batch(ctx.obj.metadata, ctx.obj.environment) try: @@ -303,4 +294,4 @@ def _sync_metadata(): traceback.print_exc() sys.exit(METAFLOW_EXIT_DISALLOW_RETRY) finally: - _sync_metadata() \ No newline at end of file + _sync_metadata() diff --git a/metaflow/plugins/aws/batch/batch_client.py b/metaflow/plugins/aws/batch/batch_client.py index 64e988d5c2e..dc2faf06c23 100644 --- a/metaflow/plugins/aws/batch/batch_client.py +++ b/metaflow/plugins/aws/batch/batch_client.py @@ -15,18 +15,20 @@ from metaflow.exception import MetaflowException from metaflow.metaflow_config import AWS_SANDBOX_ENABLED + class BatchClient(object): def __init__(self): from ..aws_client import get_aws_client - self._client = get_aws_client('batch') + + self._client = get_aws_client("batch") def active_job_queues(self): - paginator = self._client.get_paginator('describe_job_queues') + paginator = self._client.get_paginator("describe_job_queues") return ( - queue['jobQueueName'] + queue["jobQueueName"] for page in paginator.paginate() - for queue in page['jobQueues'] - if queue['state'] == 'ENABLED' and queue['status'] == 'VALID' + for queue in page["jobQueues"] + if queue["state"] == "ENABLED" and queue["status"] == "VALID" ) def unfinished_jobs(self): @@ -34,22 +36,23 @@ def unfinished_jobs(self): return ( job for queue in queues - for status in ['SUBMITTED', 'PENDING', 'RUNNABLE', 'STARTING', 'RUNNING'] - for page in self._client.get_paginator('list_jobs').paginate( + for status in ["SUBMITTED", "PENDING", "RUNNABLE", "STARTING", "RUNNING"] + for page in self._client.get_paginator("list_jobs").paginate( jobQueue=queue, jobStatus=status ) - for job in page['jobSummaryList'] + for job in page["jobSummaryList"] ) def describe_jobs(self, job_ids): - for jobIds in [job_ids[i:i+100] for i in range(0, len(job_ids), 100)]: - for jobs in self._client.describe_jobs(jobs=jobIds)['jobs']: + for jobIds in [job_ids[i : i + 100] for i in range(0, len(job_ids), 100)]: + for jobs in self._client.describe_jobs(jobs=jobIds)["jobs"]: yield jobs def describe_job_queue(self, job_queue): - paginator = self._client.get_paginator('describe_job_queues').paginate( - jobQueues=[job_queue], maxResults=1) - return paginator.paginate()['jobQueues'][0] + paginator = self._client.get_paginator("describe_job_queues").paginate( + jobQueues=[job_queue], maxResults=1 + ) + return paginator.paginate()["jobQueues"][0] def job(self): return BatchJob(self._client) @@ -63,7 +66,7 @@ def region(self): class BatchJobException(MetaflowException): - headline = 'AWS Batch job error' + headline = "AWS Batch job error" class BatchJob(object): @@ -75,34 +78,37 @@ def __init__(self, client): def execute(self): if self._image is None: raise BatchJobException( - 'Unable to launch AWS Batch job. No docker image specified.' + "Unable to launch AWS Batch job. No docker image specified." ) if self._iam_role is None: raise BatchJobException( - 'Unable to launch AWS Batch job. No IAM role specified.' + "Unable to launch AWS Batch job. No IAM role specified." + ) + if "jobDefinition" not in self.payload: + self.payload["jobDefinition"] = self._register_job_definition( + self._image, + self._iam_role, + self.payload["job_queue"], + self._execution_role, + self._shared_memory, + self._max_swap, + self._swappiness, ) - if 'jobDefinition' not in self.payload: - self.payload['jobDefinition'] = \ - self._register_job_definition(self._image, - self._iam_role, - self.payload['job_queue'], - self._execution_role, - self._shared_memory, - self._max_swap, - self._swappiness) response = self._client.submit_job(**self.payload) - job = RunningJob(response['jobId'], self._client) + job = RunningJob(response["jobId"], self._client) return job.update() - def _register_job_definition(self, - image, - job_role, - job_queue, - execution_role, - shared_memory, - max_swap, - swappiness, - host_volumes): + def _register_job_definition( + self, + image, + job_role, + job_queue, + execution_role, + shared_memory, + max_swap, + swappiness, + host_volumes, + ): # identify platform from any compute environment associated with the # queue if AWS_SANDBOX_ENABLED: @@ -112,145 +118,159 @@ def _register_job_definition(self, platform = "EC2" else: response = self._client.describe_job_queues(jobQueues=[job_queue]) - if len(response['jobQueues']) == 0: - raise BatchJobException( - 'AWS Batch Job Queue %s not found.' % job_queue) - compute_environment = response['jobQueues'][0] \ - ['computeEnvironmentOrder'][0] \ - ['computeEnvironment'] + if len(response["jobQueues"]) == 0: + raise BatchJobException("AWS Batch Job Queue %s not found." % job_queue) + compute_environment = response["jobQueues"][0]["computeEnvironmentOrder"][ + 0 + ]["computeEnvironment"] response = self._client.describe_compute_environments( - computeEnvironments=[compute_environment]) - platform = response['computeEnvironments'][0] \ - ['computeResources']['type'] + computeEnvironments=[compute_environment] + ) + platform = response["computeEnvironments"][0]["computeResources"]["type"] # compose job definition job_definition = { - 'type': 'container', - 'containerProperties': { - 'image': image, - 'jobRoleArn': job_role, - 'command': ['echo', 'hello world'], - 'resourceRequirements': [ - { - 'value': '1', - 'type': 'VCPU' - }, - { - 'value': '4096', - 'type': 'MEMORY' - } - ] + "type": "container", + "containerProperties": { + "image": image, + "jobRoleArn": job_role, + "command": ["echo", "hello world"], + "resourceRequirements": [ + {"value": "1", "type": "VCPU"}, + {"value": "4096", "type": "MEMORY"}, + ], }, # This propagates the AWS Batch resource tags to the underlying # ECS tasks. - 'propagateTags': True + "propagateTags": True, } - if platform == 'FARGATE' or platform == 'FARGATE_SPOT': + if platform == "FARGATE" or platform == "FARGATE_SPOT": if execution_role is None: raise BatchJobException( - 'No AWS Fargate task execution IAM role found. Please see ' - 'https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html ' - 'and set the role as METAFLOW_ECS_FARGATE_EXECUTION_ROLE ' - 'environment variable.') - job_definition['containerProperties']['executionRoleArn'] = \ - execution_role - job_definition['platformCapabilities'] = ['FARGATE'] - job_definition['containerProperties']['networkConfiguration'] = \ - {'assignPublicIp': 'ENABLED'} - - if platform == 'EC2' or platform == 'SPOT': - if 'linuxParameters' not in job_definition['containerProperties']: - job_definition['containerProperties']['linuxParameters'] = {} + "No AWS Fargate task execution IAM role found. Please see " + "https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html " + "and set the role as METAFLOW_ECS_FARGATE_EXECUTION_ROLE " + "environment variable." + ) + job_definition["containerProperties"]["executionRoleArn"] = execution_role + job_definition["platformCapabilities"] = ["FARGATE"] + job_definition["containerProperties"]["networkConfiguration"] = { + "assignPublicIp": "ENABLED" + } + + if platform == "EC2" or platform == "SPOT": + if "linuxParameters" not in job_definition["containerProperties"]: + job_definition["containerProperties"]["linuxParameters"] = {} if shared_memory is not None: - if not (isinstance(shared_memory, (int, unicode, basestring)) and - int(shared_memory) > 0): + if not ( + isinstance(shared_memory, (int, unicode, basestring)) + and int(shared_memory) > 0 + ): raise BatchJobException( - 'Invalid shared memory size value ({}); ' - 'it should be greater than 0'.format(shared_memory)) + "Invalid shared memory size value ({}); " + "it should be greater than 0".format(shared_memory) + ) else: - job_definition['containerProperties'] \ - ['linuxParameters']['sharedMemorySize'] = int(shared_memory) - if swappiness is not None: - if not (isinstance(swappiness, (int, unicode, basestring)) and - int(swappiness) >= 0 and int(swappiness) < 100): + job_definition["containerProperties"]["linuxParameters"][ + "sharedMemorySize" + ] = int(shared_memory) + if swappiness is not None: + if not ( + isinstance(swappiness, (int, unicode, basestring)) + and int(swappiness) >= 0 + and int(swappiness) < 100 + ): raise BatchJobException( - 'Invalid swappiness value ({}); ' - '(should be 0 or greater and less than 100)'.format(swappiness)) + "Invalid swappiness value ({}); " + "(should be 0 or greater and less than 100)".format(swappiness) + ) else: - job_definition['containerProperties'] \ - ['linuxParameters']['swappiness'] = int(swappiness) - if max_swap is not None: - if not (isinstance(max_swap, (int, unicode, basestring)) and - int(max_swap) >= 0): + job_definition["containerProperties"]["linuxParameters"][ + "swappiness" + ] = int(swappiness) + if max_swap is not None: + if not ( + isinstance(max_swap, (int, unicode, basestring)) + and int(max_swap) >= 0 + ): raise BatchJobException( - 'Invalid swappiness value ({}); ' - '(should be 0 or greater)'.format(max_swap)) + "Invalid swappiness value ({}); " + "(should be 0 or greater)".format(max_swap) + ) else: - job_definition['containerProperties'] \ - ['linuxParameters']['maxSwap'] = int(max_swap) + job_definition["containerProperties"]["linuxParameters"][ + "maxSwap" + ] = int(max_swap) if host_volumes: - job_definition['containerProperties']['volumes'] = [] - job_definition['containerProperties']['mountPoints'] = [] + job_definition["containerProperties"]["volumes"] = [] + job_definition["containerProperties"]["mountPoints"] = [] for host_path in host_volumes: - name = host_path.replace('/', '_').replace('.', '_') - job_definition['containerProperties']['volumes'].append( - {'name': name, 'host': {'sourcePath': host_path}} + name = host_path.replace("/", "_").replace(".", "_") + job_definition["containerProperties"]["volumes"].append( + {"name": name, "host": {"sourcePath": host_path}} ) - job_definition['containerProperties']['mountPoints'].append( + job_definition["containerProperties"]["mountPoints"].append( {"sourceVolume": name, "containerPath": host_path} ) # check if job definition already exists - def_name = 'metaflow_%s' % \ - hashlib.sha224(str(job_definition).encode('utf-8')).hexdigest() - payload = {'jobDefinitionName': def_name, 'status': 'ACTIVE'} + def_name = ( + "metaflow_%s" + % hashlib.sha224(str(job_definition).encode("utf-8")).hexdigest() + ) + payload = {"jobDefinitionName": def_name, "status": "ACTIVE"} response = self._client.describe_job_definitions(**payload) - if len(response['jobDefinitions']) > 0: - return response['jobDefinitions'][0]['jobDefinitionArn'] + if len(response["jobDefinitions"]) > 0: + return response["jobDefinitions"][0]["jobDefinitionArn"] # else create a job definition - job_definition['jobDefinitionName'] = def_name + job_definition["jobDefinitionName"] = def_name try: response = self._client.register_job_definition(**job_definition) except Exception as ex: - if type(ex).__name__ == 'ParamValidationError' and \ - (platform == 'FARGATE' or platform == 'FARGATE_SPOT'): + if type(ex).__name__ == "ParamValidationError" and ( + platform == "FARGATE" or platform == "FARGATE_SPOT" + ): raise BatchJobException( - '%s \nPlease ensure you have installed boto3>=1.16.29 if ' - 'you intend to launch AWS Batch jobs on AWS Fargate ' - 'compute platform.' % ex) + "%s \nPlease ensure you have installed boto3>=1.16.29 if " + "you intend to launch AWS Batch jobs on AWS Fargate " + "compute platform." % ex + ) else: raise ex - return response['jobDefinitionArn'] - - def job_def(self, - image, - iam_role, - job_queue, - execution_role, - shared_memory, - max_swap, - swappiness, - host_volumes): - self.payload['jobDefinition'] = \ - self._register_job_definition(image, - iam_role, - job_queue, - execution_role, - shared_memory, - max_swap, - swappiness, - host_volumes) + return response["jobDefinitionArn"] + + def job_def( + self, + image, + iam_role, + job_queue, + execution_role, + shared_memory, + max_swap, + swappiness, + host_volumes, + ): + self.payload["jobDefinition"] = self._register_job_definition( + image, + iam_role, + job_queue, + execution_role, + shared_memory, + max_swap, + swappiness, + host_volumes, + ) return self def job_name(self, job_name): - self.payload['jobName'] = job_name + self.payload["jobName"] = job_name return self def job_queue(self, job_queue): - self.payload['jobQueue'] = job_queue + self.payload["jobQueue"] = job_queue return self def image(self, image): @@ -278,77 +298,81 @@ def swappiness(self, swappiness): return self def command(self, command): - if 'command' not in self.payload['containerOverrides']: - self.payload['containerOverrides']['command'] = [] - self.payload['containerOverrides']['command'].extend(command) + if "command" not in self.payload["containerOverrides"]: + self.payload["containerOverrides"]["command"] = [] + self.payload["containerOverrides"]["command"].extend(command) return self def cpu(self, cpu): if not (isinstance(cpu, (int, unicode, basestring, float)) and float(cpu) > 0): raise BatchJobException( - 'Invalid CPU value ({}); it should be greater than 0'.format(cpu)) - if 'resourceRequirements' not in self.payload['containerOverrides']: - self.payload['containerOverrides']['resourceRequirements'] = [] - self.payload['containerOverrides']['resourceRequirements'].append( - {'value' : str(cpu), 'type': 'VCPU'} + "Invalid CPU value ({}); it should be greater than 0".format(cpu) + ) + if "resourceRequirements" not in self.payload["containerOverrides"]: + self.payload["containerOverrides"]["resourceRequirements"] = [] + self.payload["containerOverrides"]["resourceRequirements"].append( + {"value": str(cpu), "type": "VCPU"} ) return self def memory(self, mem): if not (isinstance(mem, (int, unicode, basestring)) and int(mem) > 0): raise BatchJobException( - 'Invalid memory value ({}); it should be greater than 0'.format(mem)) - if 'resourceRequirements' not in self.payload['containerOverrides']: - self.payload['containerOverrides']['resourceRequirements'] = [] - self.payload['containerOverrides']['resourceRequirements'].append( - {'value' : str(mem), 'type': 'MEMORY'} + "Invalid memory value ({}); it should be greater than 0".format(mem) + ) + if "resourceRequirements" not in self.payload["containerOverrides"]: + self.payload["containerOverrides"]["resourceRequirements"] = [] + self.payload["containerOverrides"]["resourceRequirements"].append( + {"value": str(mem), "type": "MEMORY"} ) return self def gpu(self, gpu): if not (isinstance(gpu, (int, unicode, basestring))): raise BatchJobException( - 'invalid gpu value: ({}) (should be 0 or greater)'.format(gpu)) + "invalid gpu value: ({}) (should be 0 or greater)".format(gpu) + ) if int(gpu) > 0: - if 'resourceRequirements' not in self.payload['containerOverrides']: - self.payload['containerOverrides']['resourceRequirements'] = [] - self.payload['containerOverrides']['resourceRequirements'].append( - {'type': 'GPU', 'value': str(gpu)} + if "resourceRequirements" not in self.payload["containerOverrides"]: + self.payload["containerOverrides"]["resourceRequirements"] = [] + self.payload["containerOverrides"]["resourceRequirements"].append( + {"type": "GPU", "value": str(gpu)} ) return self def environment_variable(self, name, value): - if 'environment' not in self.payload['containerOverrides']: - self.payload['containerOverrides']['environment'] = [] + if "environment" not in self.payload["containerOverrides"]: + self.payload["containerOverrides"]["environment"] = [] value = str(value) if value.startswith("$$.") or value.startswith("$."): # Context Object substitution for AWS Step Functions # https://docs.aws.amazon.com/step-functions/latest/dg/input-output-contextobject.html - self.payload['containerOverrides']['environment'].append( - {'name': name, 'value.$': value} + self.payload["containerOverrides"]["environment"].append( + {"name": name, "value.$": value} ) else: - self.payload['containerOverrides']['environment'].append( - {'name': name, 'value': value} + self.payload["containerOverrides"]["environment"].append( + {"name": name, "value": value} ) return self def timeout_in_secs(self, timeout_in_secs): - self.payload['timeout']['attemptDurationSeconds'] = timeout_in_secs + self.payload["timeout"]["attemptDurationSeconds"] = timeout_in_secs return self def tag(self, key, value): - self.payload['tags'][key] = str(value) + self.payload["tags"][key] = str(value) return self def parameter(self, key, value): - self.payload['parameters'][key] = str(value) + self.payload["parameters"][key] = str(value) return self def attempts(self, attempts): - self.payload['retryStrategy']['attempts'] = attempts + self.payload["retryStrategy"]["attempts"] = attempts return self + class Throttle(object): def __init__(self, delta_in_secs=1, num_tries=20): self.delta_in_secs = delta_in_secs @@ -372,14 +396,18 @@ def wrapped(*args, **kwargs): self._tries_left -= 1 if self._tries_left == 0: raise ex.ex - self._wait = (self.delta_in_secs*1.2)**(self.num_tries-self._tries_left) + \ - random.randint(0, 3*self.delta_in_secs) + self._wait = (self.delta_in_secs * 1.2) ** ( + self.num_tries - self._tries_left + ) + random.randint(0, 3 * self.delta_in_secs) + return wrapped + class TriableException(Exception): def __init__(self, ex): self.ex = ex + class RunningJob(object): NUM_RETRIES = 8 @@ -390,7 +418,7 @@ def __init__(self, id, client): self._data = {} def __repr__(self): - return '{}(\'{}\')'.format(self.__class__.__name__, self._id) + return "{}('{}')".format(self.__class__.__name__, self._id) def _apply(self, data): self._data = data @@ -400,7 +428,7 @@ def _update(self): try: data = self._client.describe_jobs(jobs=[self._id]) except self._client.exceptions.ClientError as err: - code = err.response['ResponseMetadata']['HTTPStatusCode'] + code = err.response["ResponseMetadata"]["HTTPStatusCode"] if code == 429 or code >= 500: raise TriableException(err) raise err @@ -411,8 +439,8 @@ def _update(self): # will ensure that we poll `batch.describe_jobs` until we get a # satisfactory response at least once through out the lifecycle of # the job. - if len(data['jobs']) == 1: - self._apply(data['jobs'][0]) + if len(data["jobs"]) == 1: + self._apply(data["jobs"][0]) def update(self): self._update() @@ -432,29 +460,29 @@ def info(self): @property def job_name(self): - return self.info['jobName'] + return self.info["jobName"] @property def job_queue(self): - return self.info['jobQueue'] + return self.info["jobQueue"] @property def status(self): if not self.is_done: self.update() - return self.info['status'] + return self.info["status"] @property def status_reason(self): - return self.info.get('statusReason') + return self.info.get("statusReason") @property def created_at(self): - return self.info['createdAt'] + return self.info["createdAt"] @property def stopped_at(self): - return self.info.get('stoppedAt', 0) + return self.info.get("stoppedAt", 0) @property def is_done(self): @@ -464,29 +492,30 @@ def is_done(self): @property def is_running(self): - return self.status == 'RUNNING' + return self.status == "RUNNING" @property def is_successful(self): - return self.status == 'SUCCEEDED' + return self.status == "SUCCEEDED" @property def is_crashed(self): # TODO: Check statusmessage to find if the job crashed instead of failing - return self.status == 'FAILED' + return self.status == "FAILED" @property def reason(self): - return self.info['container'].get('reason') + return self.info["container"].get("reason") @property def status_code(self): if not self.is_done: self.update() - return self.info['container'].get('exitCode') + return self.info["container"].get("exitCode") def kill(self): if not self.is_done: self._client.terminate_job( - jobId=self._id, reason='Metaflow initiated job termination.') + jobId=self._id, reason="Metaflow initiated job termination." + ) return self.update() diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index 2ffb77c1144..3e69bd50e9d 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -11,24 +11,30 @@ from metaflow.plugins.timeout_decorator import get_run_time_limit_for_task from metaflow.metadata import MetaDatum from metaflow.metadata.util import sync_local_metadata_to_datastore -from metaflow.metaflow_config import ECS_S3_ACCESS_IAM_ROLE, BATCH_JOB_QUEUE, \ - BATCH_CONTAINER_IMAGE, BATCH_CONTAINER_REGISTRY, \ - ECS_FARGATE_EXECUTION_ROLE, DATASTORE_LOCAL_DIR +from metaflow.metaflow_config import ( + ECS_S3_ACCESS_IAM_ROLE, + BATCH_JOB_QUEUE, + BATCH_CONTAINER_IMAGE, + BATCH_CONTAINER_REGISTRY, + ECS_FARGATE_EXECUTION_ROLE, + DATASTORE_LOCAL_DIR, +) from metaflow.sidecar import SidecarSubProcess from .batch import BatchException from ..aws_utils import get_docker_registry + class BatchDecorator(StepDecorator): """ Step decorator to specify that this step should execute on AWS Batch. - - This decorator indicates that your step should execute on AWS Batch. Note - that you can apply this decorator automatically to all steps using the - ```--with batch``` argument when calling run/resume. Step level decorators + + This decorator indicates that your step should execute on AWS Batch. Note + that you can apply this decorator automatically to all steps using the + ```--with batch``` argument when calling run/resume. Step level decorators within the code are overrides and will force a step to execute on AWS Batch regardless of the ```--with``` specification. - + To use, annotate your step as follows: ``` @batch @@ -45,30 +51,30 @@ def my_step(self): Number of GPUs required for this step. Defaults to 0. If @resources is also present, the maximum value from all decorators is used memory : int - Memory size (in MB) required for this step. Defaults to 4096. If + Memory size (in MB) required for this step. Defaults to 4096. If @resources is also present, the maximum value from all decorators is used image : string - Docker image to use when launching on AWS Batch. If not specified, a + Docker image to use when launching on AWS Batch. If not specified, a default docker image mapping to the current version of Python is used queue : string - AWS Batch Job Queue to submit the job to. Defaults to the one + AWS Batch Job Queue to submit the job to. Defaults to the one specified by the environment variable METAFLOW_BATCH_JOB_QUEUE iam_role : string AWS IAM role that AWS Batch container uses to access AWS cloud resources (Amazon S3, Amazon DynamoDb, etc). Defaults to the one specified by the environment variable METAFLOW_ECS_S3_ACCESS_IAM_ROLE execution_role : string - AWS IAM role that AWS Batch can use to trigger AWS Fargate tasks. - Defaults to the one determined by the environment variable + AWS IAM role that AWS Batch can use to trigger AWS Fargate tasks. + Defaults to the one determined by the environment variable METAFLOW_ECS_FARGATE_EXECUTION_ROLE https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html shared_memory : int The value for the size (in MiB) of the /dev/shm volume for this step. This parameter maps to the --shm-size option to docker run. max_swap : int - The total amount of swap memory (in MiB) a container can use for this - step. This parameter is translated to the --memory-swap option to - docker run where the value is the sum of the container memory plus the + The total amount of swap memory (in MiB) a container can use for this + step. This parameter is translated to the --memory-swap option to + docker run where the value is the sum of the container memory plus the max_swap value. swappiness : int This allows you to tune memory swappiness behavior for this step. @@ -76,19 +82,20 @@ def my_step(self): necessary. A swappiness value of 100 causes pages to be swapped very aggressively. Accepted values are whole numbers between 0 and 100. """ - name = 'batch' + + name = "batch" defaults = { - 'cpu': '1', - 'gpu': '0', - 'memory': '4096', - 'image': None, - 'queue': BATCH_JOB_QUEUE, - 'iam_role': ECS_S3_ACCESS_IAM_ROLE, - 'execution_role': ECS_FARGATE_EXECUTION_ROLE, - 'shared_memory': None, - 'max_swap': None, - 'swappiness': None, - 'host_volumes': None, + "cpu": "1", + "gpu": "0", + "memory": "4096", + "image": None, + "queue": BATCH_JOB_QUEUE, + "iam_role": ECS_S3_ACCESS_IAM_ROLE, + "execution_role": ECS_FARGATE_EXECUTION_ROLE, + "shared_memory": None, + "max_swap": None, + "swappiness": None, + "host_volumes": None, } package_url = None package_sha = None @@ -98,42 +105,37 @@ def __init__(self, attributes=None, statically_defined=False): super(BatchDecorator, self).__init__(attributes, statically_defined) # If no docker image is explicitly specified, impute a default image. - if not self.attributes['image']: + if not self.attributes["image"]: # If metaflow-config specifies a docker image, just use that. if BATCH_CONTAINER_IMAGE: - self.attributes['image'] = BATCH_CONTAINER_IMAGE - # If metaflow-config doesn't specify a docker image, assign a + self.attributes["image"] = BATCH_CONTAINER_IMAGE + # If metaflow-config doesn't specify a docker image, assign a # default docker image. else: # Metaflow-R has it's own default docker image (rocker family) if R.use_r(): - self.attributes['image'] = R.container_image() + self.attributes["image"] = R.container_image() # Default to vanilla Python image corresponding to major.minor # version of the Python interpreter launching the flow. else: - self.attributes['image'] = \ - 'python:%s.%s' % (platform.python_version_tuple()[0], - platform.python_version_tuple()[1]) + self.attributes["image"] = "python:%s.%s" % ( + platform.python_version_tuple()[0], + platform.python_version_tuple()[1], + ) # Assign docker registry URL for the image. - if not get_docker_registry(self.attributes['image']): + if not get_docker_registry(self.attributes["image"]): if BATCH_CONTAINER_REGISTRY: - self.attributes['image'] = \ - '%s/%s' % (BATCH_CONTAINER_REGISTRY.rstrip('/'), - self.attributes['image']) + self.attributes["image"] = "%s/%s" % ( + BATCH_CONTAINER_REGISTRY.rstrip("/"), + self.attributes["image"], + ) # Refer https://github.com/Netflix/metaflow/blob/master/docs/lifecycle.png # to understand where these functions are invoked in the lifecycle of a # Metaflow flow. - def step_init(self, - flow, - graph, - step, - decos, - environment, - flow_datastore, - logger): - if flow_datastore.TYPE != 's3': - raise BatchException('The *@batch* decorator requires --datastore=s3.') + def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger): + if flow_datastore.TYPE != "s3": + raise BatchException("The *@batch* decorator requires --datastore=s3.") # Set internal state. self.logger = logger @@ -147,161 +149,152 @@ def step_init(self, # TODO: Fix https://github.com/Netflix/metaflow/issues/467 my_val = self.attributes.get(k) if not (my_val is None and v is None): - self.attributes[k] = \ - str(max(int(my_val or 0), int(v or 0))) - + self.attributes[k] = str(max(int(my_val or 0), int(v or 0))) + # Set run time limit for the AWS Batch job. self.run_time_limit = get_run_time_limit_for_task(decos) if self.run_time_limit < 60: - raise BatchException('The timeout for step *{step}* should be at ' - 'least 60 seconds for execution on AWS Batch.'.format(step=step)) + raise BatchException( + "The timeout for step *{step}* should be at " + "least 60 seconds for execution on AWS Batch.".format(step=step) + ) - def runtime_init(self, - flow, - graph, - package, - run_id): + def runtime_init(self, flow, graph, package, run_id): # Set some more internal state. self.flow = flow self.graph = graph self.package = package self.run_id = run_id - def runtime_task_created(self, - task_datastore, - task_id, - split_index, - input_paths, - is_cloned, - ubf_context): + def runtime_task_created( + self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context + ): if not is_cloned: self._save_package_once(self.flow_datastore, self.package) - def runtime_step_cli(self, - cli_args, - retry_count, - max_user_code_retries, - ubf_context): + def runtime_step_cli( + self, cli_args, retry_count, max_user_code_retries, ubf_context + ): if retry_count <= max_user_code_retries: # after all attempts to run the user code have failed, we don't need - # to execute on AWS Batch anymore. We can execute possible fallback + # to execute on AWS Batch anymore. We can execute possible fallback # code locally. - cli_args.commands = ['batch', 'step'] + cli_args.commands = ["batch", "step"] cli_args.command_args.append(self.package_sha) cli_args.command_args.append(self.package_url) cli_args.command_options.update(self.attributes) - cli_args.command_options['run-time-limit'] = self.run_time_limit + cli_args.command_options["run-time-limit"] = self.run_time_limit if not R.use_r(): cli_args.entrypoint[0] = sys.executable - def task_pre_step(self, - step_name, - task_datastore, - metadata, - run_id, - task_id, - flow, - graph, - retry_count, - max_retries, - ubf_context, - inputs): + def task_pre_step( + self, + step_name, + task_datastore, + metadata, + run_id, + task_id, + flow, + graph, + retry_count, + max_retries, + ubf_context, + inputs, + ): self.metadata = metadata self.task_datastore = task_datastore - # task_pre_step may run locally if fallback is activated for @catch + # task_pre_step may run locally if fallback is activated for @catch # decorator. In that scenario, we skip collecting AWS Batch execution # metadata. A rudimentary way to detect non-local execution is to # check for the existence of AWS_BATCH_JOB_ID environment variable. - if 'AWS_BATCH_JOB_ID' in os.environ: + if "AWS_BATCH_JOB_ID" in os.environ: meta = {} - meta['aws-batch-job-id'] = os.environ['AWS_BATCH_JOB_ID'] - meta['aws-batch-job-attempt'] = os.environ['AWS_BATCH_JOB_ATTEMPT'] - meta['aws-batch-ce-name'] = os.environ['AWS_BATCH_CE_NAME'] - meta['aws-batch-jq-name'] = os.environ['AWS_BATCH_JQ_NAME'] - meta['aws-batch-execution-env'] = os.environ['AWS_EXECUTION_ENV'] - + meta["aws-batch-job-id"] = os.environ["AWS_BATCH_JOB_ID"] + meta["aws-batch-job-attempt"] = os.environ["AWS_BATCH_JOB_ATTEMPT"] + meta["aws-batch-ce-name"] = os.environ["AWS_BATCH_CE_NAME"] + meta["aws-batch-jq-name"] = os.environ["AWS_BATCH_JQ_NAME"] + meta["aws-batch-execution-env"] = os.environ["AWS_EXECUTION_ENV"] # Capture AWS Logs metadata. This is best effort only since # only V4 of the metadata uri for the ECS container hosts this - # information and it is quite likely that not all consumers of + # information and it is quite likely that not all consumers of # Metaflow would be running the container agent compatible with # version V4. # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint.html try: - logs_meta = requests.get( - url=os.environ['ECS_CONTAINER_METADATA_URI_V4']) \ - .json() \ - .get('LogOptions', {}) - meta['aws-batch-awslogs-group'] = logs_meta.get('awslogs-group') - meta['aws-batch-awslogs-region'] = logs_meta.get('awslogs-region') - meta['aws-batch-awslogs-stream'] = logs_meta.get('awslogs-stream') + logs_meta = ( + requests.get(url=os.environ["ECS_CONTAINER_METADATA_URI_V4"]) + .json() + .get("LogOptions", {}) + ) + meta["aws-batch-awslogs-group"] = logs_meta.get("awslogs-group") + meta["aws-batch-awslogs-region"] = logs_meta.get("awslogs-region") + meta["aws-batch-awslogs-stream"] = logs_meta.get("awslogs-stream") except: pass - entries = [MetaDatum( - field=k, value=v, type=k, tags=["attempt_id:{0}".format(retry_count)]) - for k, v in meta.items()] + entries = [ + MetaDatum( + field=k, + value=v, + type=k, + tags=["attempt_id:{0}".format(retry_count)], + ) + for k, v in meta.items() + ] # Register book-keeping metadata for debugging. metadata.register_metadata(run_id, step_name, task_id, entries) - - self._save_logs_sidecar = SidecarSubProcess('save_logs_periodically') - def task_post_step(self, - step_name, - flow, - graph, - retry_count, - max_user_code_retries): - # task_post_step may run locally if fallback is activated for @catch + self._save_logs_sidecar = SidecarSubProcess("save_logs_periodically") + + def task_post_step( + self, step_name, flow, graph, retry_count, max_user_code_retries + ): + # task_post_step may run locally if fallback is activated for @catch # decorator. - if 'AWS_BATCH_JOB_ID' in os.environ: + if "AWS_BATCH_JOB_ID" in os.environ: # If `local` metadata is configured, we would need to copy task # execution metadata from the AWS Batch container to user's # local file system after the user code has finished execution. # This happens via datastore as a communication bridge. - if self.metadata.TYPE == 'local': - # Note that the datastore is *always* Amazon S3 (see + if self.metadata.TYPE == "local": + # Note that the datastore is *always* Amazon S3 (see # runtime_task_created function). - sync_local_metadata_to_datastore(DATASTORE_LOCAL_DIR, - self.task_datastore) + sync_local_metadata_to_datastore( + DATASTORE_LOCAL_DIR, self.task_datastore + ) - def task_exception(self, - exception, - step_name, - flow, - graph, - retry_count, - max_user_code_retries): - # task_exception may run locally if fallback is activated for @catch + def task_exception( + self, exception, step_name, flow, graph, retry_count, max_user_code_retries + ): + # task_exception may run locally if fallback is activated for @catch # decorator. - if 'AWS_BATCH_JOB_ID' in os.environ: + if "AWS_BATCH_JOB_ID" in os.environ: # If `local` metadata is configured, we would need to copy task # execution metadata from the AWS Batch container to user's # local file system after the user code has finished execution. # This happens via datastore as a communication bridge. - if self.metadata.TYPE == 'local': - # Note that the datastore is *always* Amazon S3 (see + if self.metadata.TYPE == "local": + # Note that the datastore is *always* Amazon S3 (see # runtime_task_created function). - sync_local_metadata_to_datastore(DATASTORE_LOCAL_DIR, - self.task_datastore) + sync_local_metadata_to_datastore( + DATASTORE_LOCAL_DIR, self.task_datastore + ) - def task_finished(self, - step_name, - flow, - graph, - is_task_ok, - retry_count, - max_retries): - try: - self._save_logs_sidecar.kill() - except: - # Best effort kill - pass + def task_finished( + self, step_name, flow, graph, is_task_ok, retry_count, max_retries + ): + try: + self._save_logs_sidecar.kill() + except: + # Best effort kill + pass @classmethod def _save_package_once(cls, flow_datastore, package): if cls.package_url is None: cls.package_url, cls.package_sha = flow_datastore.save_data( - [package.blob], len_hint=1)[0] \ No newline at end of file + [package.blob], len_hint=1 + )[0] diff --git a/metaflow/plugins/aws/eks/kubernetes.py b/metaflow/plugins/aws/eks/kubernetes.py index 764b362d4c5..eb61c006fae 100644 --- a/metaflow/plugins/aws/eks/kubernetes.py +++ b/metaflow/plugins/aws/eks/kubernetes.py @@ -22,7 +22,7 @@ export_mflog_env_vars, bash_capture_logs, update_delay, - BASH_SAVE_LOGS + BASH_SAVE_LOGS, ) from metaflow.mflog.mflog import refine, set_should_persist @@ -44,41 +44,36 @@ class KubernetesKilledException(MetaflowException): headline = "Kubernetes Batch job killed" -def generate_rfc1123_name(flow_name, - run_id, - step_name, - task_id, - attempt -): +def generate_rfc1123_name(flow_name, run_id, step_name, task_id, attempt): """ Generate RFC 1123 compatible name. Specifically, the format is: [*[]] - - The generated name consists from a human-readable prefix, derived from + + The generated name consists from a human-readable prefix, derived from flow/step/task/attempt, and a hash suffux. """ long_name = "-".join( - [ - flow_name, - run_id, - step_name, - task_id, - attempt, - ] - ) - hash = hashlib.sha256(long_name.encode('utf-8')).hexdigest() - - if long_name.startswith('_'): + [ + flow_name, + run_id, + step_name, + task_id, + attempt, + ] + ) + hash = hashlib.sha256(long_name.encode("utf-8")).hexdigest() + + if long_name.startswith("_"): # RFC 1123 names can't start with hyphen so slap an extra prefix on it - sanitized_long_name = 'u' + long_name.replace('_', '-').lower() + sanitized_long_name = "u" + long_name.replace("_", "-").lower() else: - sanitized_long_name = long_name.replace('_', '-').lower() + sanitized_long_name = long_name.replace("_", "-").lower() # the name has to be under 63 chars total - return sanitized_long_name[:57] + '-' + hash[:5] + return sanitized_long_name[:57] + "-" + hash[:5] -LABEL_VALUE_REGEX = re.compile(r'^[a-zA-Z0-9]([a-zA-Z0-9\-\_\.]{0,61}[a-zA-Z0-9])?$') +LABEL_VALUE_REGEX = re.compile(r"^[a-zA-Z0-9]([a-zA-Z0-9\-\_\.]{0,61}[a-zA-Z0-9])?$") def sanitize_label_value(val): @@ -90,14 +85,14 @@ def sanitize_label_value(val): # position, this function will likely return distinct values, so you can # still filter on those. For example, "alice$" and "alice&" will be # sanitized into different values "alice_b3f201" and "alice_2a6f13". - if val == '' or LABEL_VALUE_REGEX.match(val): + if val == "" or LABEL_VALUE_REGEX.match(val): return val - hash = hashlib.sha256(val.encode('utf-8')).hexdigest() + hash = hashlib.sha256(val.encode("utf-8")).hexdigest() # Replace invalid chars with dots, and if the first char is # non-alphahanumeric, replace it with 'u' to make it valid - sanitized_val = re.sub('^[^A-Z0-9a-z]', 'u', re.sub(r"[^A-Za-z0-9.\-_]", "_", val)) - return sanitized_val[:57] + '-' + hash[:5] + sanitized_val = re.sub("^[^A-Z0-9a-z]", "u", re.sub(r"[^A-Za-z0-9.\-_]", "_", val)) + return sanitized_val[:57] + "-" + hash[:5] class Kubernetes(object): @@ -141,8 +136,7 @@ def _command( init_expr = " && ".join(init_cmds) step_expr = bash_capture_logs( " && ".join( - self._environment.bootstrap_commands(self._step_name) - + step_cmds + self._environment.bootstrap_commands(self._step_name) + step_cmds ) ) @@ -238,16 +232,12 @@ def create_job( .environment_variable("METAFLOW_CODE_URL", code_package_url) .environment_variable("METAFLOW_CODE_DS", code_package_ds) .environment_variable("METAFLOW_USER", user) - .environment_variable( - "METAFLOW_SERVICE_URL", BATCH_METADATA_SERVICE_URL - ) + .environment_variable("METAFLOW_SERVICE_URL", BATCH_METADATA_SERVICE_URL) .environment_variable( "METAFLOW_SERVICE_HEADERS", json.dumps(BATCH_METADATA_SERVICE_HEADERS), ) - .environment_variable( - "METAFLOW_DATASTORE_SYSROOT_S3", DATASTORE_SYSROOT_S3 - ) + .environment_variable("METAFLOW_DATASTORE_SYSROOT_S3", DATASTORE_SYSROOT_S3) .environment_variable("METAFLOW_DATATOOLS_S3ROOT", DATATOOLS_S3ROOT) .environment_variable("METAFLOW_DEFAULT_DATASTORE", "s3") .environment_variable("METAFLOW_DEFAULT_METADATA", DEFAULT_METADATA) @@ -282,14 +272,13 @@ def create_job( for sys_tag in self._metadata.sticky_sys_tags: job.label( "metaflow/%s" % sys_tag[: sys_tag.index(":")], - sanitize_label_value(sys_tag[sys_tag.index(":") + 1 :]) + sanitize_label_value(sys_tag[sys_tag.index(":") + 1 :]), ) # TODO: Add annotations based on https://kubernetes.io/blog/2021/04/20/annotating-k8s-for-humans/ return job.create() def wait(self, stdout_location, stderr_location, echo=None): - def wait_for_launch(job): status = job.status echo( @@ -380,14 +369,11 @@ def _print_available(tail, stream, should_persist=False): ) if exit_code: if int(exit_code) == 139: - raise KubernetesException( - "Task failed with a segmentation fault." - ) + raise KubernetesException("Task failed with a segmentation fault.") else: msg = "%s (exit code %s)" % (msg, exit_code) raise KubernetesException( - "%s. This could be a transient error. " - "Use @retry to retry." % msg + "%s. This could be a transient error. " "Use @retry to retry." % msg ) exit_code, _ = self._job.reason diff --git a/metaflow/plugins/aws/eks/kubernetes_cli.py b/metaflow/plugins/aws/eks/kubernetes_cli.py index 32315986b47..3a8c5b224d5 100644 --- a/metaflow/plugins/aws/eks/kubernetes_cli.py +++ b/metaflow/plugins/aws/eks/kubernetes_cli.py @@ -69,12 +69,8 @@ def kubernetes(): ) @click.option("--cpu", help="CPU requirement for Kubernetes job on Amazon EKS.") @click.option("--gpu", help="GPU requirement for Kubernetes job on Amazon EKS.") -@click.option( - "--disk", help="Disk requirement for Kubernetes job on Amazon EKS." -) -@click.option( - "--memory", help="Memory requirement for Kubernetes job on Amazon EKS." -) +@click.option("--disk", help="Disk requirement for Kubernetes job on Amazon EKS.") +@click.option("--memory", help="Memory requirement for Kubernetes job on Amazon EKS.") @click.option("--run-id", help="Passed to the top-level 'step'.") @click.option("--task-id", help="Passed to the top-level 'step'.") @click.option("--input-paths", help="Passed to the top-level 'step'.") @@ -84,12 +80,8 @@ def kubernetes(): @click.option( "--tag", multiple=True, default=None, help="Passed to the top-level 'step'." ) -@click.option( - "--namespace", default=None, help="Passed to the top-level 'step'." -) -@click.option( - "--retry-count", default=0, help="Passed to the top-level 'step'." -) +@click.option("--namespace", default=None, help="Passed to the top-level 'step'.") +@click.option("--retry-count", default=0, help="Passed to the top-level 'step'.") @click.option( "--max-user-code-retries", default=0, help="Passed to the top-level 'step'." ) @@ -141,8 +133,7 @@ def echo(msg, stream="stderr", job_id=None): if input_paths: max_size = 30 * 1024 split_vars = { - "METAFLOW_INPUT_PATHS_%d" - % (i // max_size): input_paths[i : i + max_size] + "METAFLOW_INPUT_PATHS_%d" % (i // max_size): input_paths[i : i + max_size] for i in range(0, len(input_paths), max_size) } kwargs["input_paths"] = "".join("${%s}" % s for s in split_vars.keys()) @@ -158,8 +149,7 @@ def echo(msg, stream="stderr", job_id=None): ) if retry_count: ctx.obj.echo_always( - "Sleeping %d minutes before the next retry" - % minutes_between_retries + "Sleeping %d minutes before the next retry" % minutes_between_retries ) time.sleep(minutes_between_retries * 60) @@ -172,22 +162,23 @@ def echo(msg, stream="stderr", job_id=None): # this information is needed for log tailing ds = ctx.obj.flow_datastore.get_task_datastore( - mode='w', - run_id=kwargs['run_id'], + mode="w", + run_id=kwargs["run_id"], step_name=step_name, - task_id=kwargs['task_id'], - attempt=int(retry_count) + task_id=kwargs["task_id"], + attempt=int(retry_count), ) - stdout_location = ds.get_log_location(TASK_LOG_SOURCE, 'stdout') - stderr_location = ds.get_log_location(TASK_LOG_SOURCE, 'stderr') + stdout_location = ds.get_log_location(TASK_LOG_SOURCE, "stdout") + stderr_location = ds.get_log_location(TASK_LOG_SOURCE, "stderr") def _sync_metadata(): - if ctx.obj.metadata.TYPE == 'local': + if ctx.obj.metadata.TYPE == "local": sync_local_metadata_from_datastore( - DATASTORE_LOCAL_DIR, - ctx.obj.flow_datastore.get_task_datastore(kwargs['run_id'], - step_name, - kwargs['task_id'])) + DATASTORE_LOCAL_DIR, + ctx.obj.flow_datastore.get_task_datastore( + kwargs["run_id"], step_name, kwargs["task_id"] + ), + ) try: kubernetes = Kubernetes( @@ -231,4 +222,4 @@ def _sync_metadata(): traceback.print_exc() sys.exit(METAFLOW_EXIT_DISALLOW_RETRY) finally: - _sync_metadata() \ No newline at end of file + _sync_metadata() diff --git a/metaflow/plugins/aws/eks/kubernetes_client.py b/metaflow/plugins/aws/eks/kubernetes_client.py index b0c143328c0..1e3214a335a 100644 --- a/metaflow/plugins/aws/eks/kubernetes_client.py +++ b/metaflow/plugins/aws/eks/kubernetes_client.py @@ -30,7 +30,7 @@ def wrapper(*args, **kwargs): deadline = time.time() + deadline_seconds retry_number = 0 - + while True: try: result = function(*args, **kwargs) @@ -38,17 +38,20 @@ def wrapper(*args, **kwargs): except client.rest.ApiException as e: if e.status == 500: current_t = time.time() - backoff_delay = min(math.pow(2, retry_number) + random.random(), max_backoff) + backoff_delay = min( + math.pow(2, retry_number) + random.random(), max_backoff + ) if current_t + backoff_delay < deadline: time.sleep(backoff_delay) retry_number += 1 - continue # retry again + continue # retry again else: raise else: raise return wrapper + return decorator diff --git a/metaflow/plugins/aws/eks/kubernetes_decorator.py b/metaflow/plugins/aws/eks/kubernetes_decorator.py index 46e937d674c..9f28ff60e98 100644 --- a/metaflow/plugins/aws/eks/kubernetes_decorator.py +++ b/metaflow/plugins/aws/eks/kubernetes_decorator.py @@ -79,9 +79,7 @@ def my_step(self): run_time_limit = None def __init__(self, attributes=None, statically_defined=False): - super(KubernetesDecorator, self).__init__( - attributes, statically_defined - ) + super(KubernetesDecorator, self).__init__(attributes, statically_defined) # TODO: Unify the logic with AWS Batch # If no docker image is explicitly specified, impute a default image. @@ -109,16 +107,13 @@ def __init__(self, attributes=None, statically_defined=False): # Refer https://github.com/Netflix/metaflow/blob/master/docs/lifecycle.png # to understand where these functions are invoked in the lifecycle of a # Metaflow flow. - def step_init( - self, flow, graph, step, decos, environment, flow_datastore, logger - ): + def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger): # Executing Kubernetes jobs requires a non-local datastore at the # moment. # TODO: To support MiniKube we need to enable local datastore execution. if flow_datastore.TYPE != "s3": raise KubernetesException( - "The *@kubernetes* decorator requires --datastore=s3 " - "at the moment." + "The *@kubernetes* decorator requires --datastore=s3 " "at the moment." ) # Set internal state. @@ -133,9 +128,7 @@ def step_init( # TODO: Fix https://github.com/Netflix/metaflow/issues/467 my_val = self.attributes.get(k) if not (my_val is None and v is None): - self.attributes[k] = str( - max(int(my_val or 0), int(v or 0)) - ) + self.attributes[k] = str(max(int(my_val or 0), int(v or 0))) # Set run time limit for the Kubernetes job. self.run_time_limit = get_run_time_limit_for_task(decos) @@ -153,13 +146,9 @@ def runtime_init(self, flow, graph, package, run_id): self.package = package self.run_id = run_id - def runtime_task_created(self, - task_datastore, - task_id, - split_index, - input_paths, - is_cloned, - ubf_context): + def runtime_task_created( + self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context + ): # To execute the Kubernetes job, the job container needs to have # access to the code package. We store the package in the datastore # which the pod is able to download as part of it's entrypoint. @@ -176,29 +165,31 @@ def runtime_step_cli( cli_args.commands = ["kubernetes", "step"] cli_args.command_args.append(self.package_sha) cli_args.command_args.append(self.package_url) - + # --namespace is used to specify Metaflow namespace (different # concept from k8s namespace). - for k,v in self.attributes.items(): - if k == 'namespace': - cli_args.command_options['k8s_namespace'] = v + for k, v in self.attributes.items(): + if k == "namespace": + cli_args.command_options["k8s_namespace"] = v else: cli_args.command_options[k] = v cli_args.command_options["run-time-limit"] = self.run_time_limit cli_args.entrypoint[0] = sys.executable - def task_pre_step(self, - step_name, - task_datastore, - metadata, - run_id, - task_id, - flow, - graph, - retry_count, - max_retries, - ubf_context, - inputs): + def task_pre_step( + self, + step_name, + task_datastore, + metadata, + run_id, + task_id, + flow, + graph, + retry_count, + max_retries, + ubf_context, + inputs, + ): self.metadata = metadata self.task_datastore = task_datastore @@ -212,80 +203,67 @@ def task_pre_step(self, meta = {} # TODO: Get kubernetes job id and job name meta["kubernetes-pod-id"] = os.environ["METAFLOW_KUBERNETES_POD_ID"] - meta["kubernetes-pod-name"] = os.environ[ - "METAFLOW_KUBERNETES_POD_NAME" - ] + meta["kubernetes-pod-name"] = os.environ["METAFLOW_KUBERNETES_POD_NAME"] meta["kubernetes-pod-namespace"] = os.environ[ "METAFLOW_KUBERNETES_POD_NAMESPACE" ] # meta['kubernetes-job-attempt'] = ? entries = [ - MetaDatum(field=k, value=v, type=k, tags=[]) - for k, v in meta.items() + MetaDatum(field=k, value=v, type=k, tags=[]) for k, v in meta.items() ] # Register book-keeping metadata for debugging. metadata.register_metadata(run_id, step_name, task_id, entries) # Start MFLog sidecar to collect task logs. - self._save_logs_sidecar = SidecarSubProcess( - "save_logs_periodically" - ) + self._save_logs_sidecar = SidecarSubProcess("save_logs_periodically") - def task_post_step(self, - step_name, - flow, - graph, - retry_count, - max_user_code_retries): - # task_post_step may run locally if fallback is activated for @catch + def task_post_step( + self, step_name, flow, graph, retry_count, max_user_code_retries + ): + # task_post_step may run locally if fallback is activated for @catch # decorator. - if 'METAFLOW_KUBERNETES_WORKLOAD' in os.environ: + if "METAFLOW_KUBERNETES_WORKLOAD" in os.environ: # If `local` metadata is configured, we would need to copy task # execution metadata from the AWS Batch container to user's # local file system after the user code has finished execution. # This happens via datastore as a communication bridge. - if self.metadata.TYPE == 'local': - # Note that the datastore is *always* Amazon S3 (see + if self.metadata.TYPE == "local": + # Note that the datastore is *always* Amazon S3 (see # runtime_task_created function). - sync_local_metadata_to_datastore(DATASTORE_LOCAL_DIR, - self.task_datastore) + sync_local_metadata_to_datastore( + DATASTORE_LOCAL_DIR, self.task_datastore + ) - def task_exception(self, - exception, - step_name, - flow, - graph, - retry_count, - max_user_code_retries): - # task_exception may run locally if fallback is activated for @catch + def task_exception( + self, exception, step_name, flow, graph, retry_count, max_user_code_retries + ): + # task_exception may run locally if fallback is activated for @catch # decorator. - if 'METAFLOW_KUBERNETES_WORKLOAD' in os.environ: + if "METAFLOW_KUBERNETES_WORKLOAD" in os.environ: # If `local` metadata is configured, we would need to copy task # execution metadata from the AWS Batch container to user's # local file system after the user code has finished execution. # This happens via datastore as a communication bridge. - if self.metadata.TYPE == 'local': - # Note that the datastore is *always* Amazon S3 (see + if self.metadata.TYPE == "local": + # Note that the datastore is *always* Amazon S3 (see # runtime_task_created function). - sync_local_metadata_to_datastore(DATASTORE_LOCAL_DIR, - self.task_datastore) + sync_local_metadata_to_datastore( + DATASTORE_LOCAL_DIR, self.task_datastore + ) - def task_finished(self, - step_name, - flow, - graph, - is_task_ok, - retry_count, - max_retries): - try: - self._save_logs_sidecar.kill() - except: - # Best effort kill - pass + def task_finished( + self, step_name, flow, graph, is_task_ok, retry_count, max_retries + ): + try: + self._save_logs_sidecar.kill() + except: + # Best effort kill + pass @classmethod def _save_package_once(cls, flow_datastore, package): if cls.package_url is None: cls.package_url, cls.package_sha = flow_datastore.save_data( - [package.blob], len_hint=1)[0] \ No newline at end of file + [package.blob], len_hint=1 + )[0] diff --git a/metaflow/plugins/aws/step_functions/dynamo_db_client.py b/metaflow/plugins/aws/step_functions/dynamo_db_client.py index 483fbdd25f6..52386b26b94 100644 --- a/metaflow/plugins/aws/step_functions/dynamo_db_client.py +++ b/metaflow/plugins/aws/step_functions/dynamo_db_client.py @@ -4,77 +4,59 @@ class DynamoDbClient(object): - def __init__(self): from ..aws_client import get_aws_client + self._client = get_aws_client( - 'dynamodb', - params={'region_name': self._get_instance_region()}) + "dynamodb", params={"region_name": self._get_instance_region()} + ) self.name = SFN_DYNAMO_DB_TABLE - def save_foreach_cardinality(self, - foreach_split_task_id, - foreach_cardinality, - ttl): + def save_foreach_cardinality(self, foreach_split_task_id, foreach_cardinality, ttl): return self._client.put_item( - TableName = self.name, - Item = { - 'pathspec': { - 'S': foreach_split_task_id + TableName=self.name, + Item={ + "pathspec": {"S": foreach_split_task_id}, + "for_each_cardinality": { + "NS": list(map(str, range(foreach_cardinality))) }, - 'for_each_cardinality': { - 'NS': list(map(str, range(foreach_cardinality))) - }, - 'ttl': { - 'N': str(ttl) - } - } + "ttl": {"N": str(ttl)}, + }, ) - def save_parent_task_id_for_foreach_join(self, - foreach_split_task_id, - foreach_join_parent_task_id): + def save_parent_task_id_for_foreach_join( + self, foreach_split_task_id, foreach_join_parent_task_id + ): return self._client.update_item( - TableName = self.name, - Key = { - 'pathspec': { - 'S': foreach_split_task_id - } - }, - UpdateExpression = 'ADD parent_task_ids_for_foreach_join :val', - ExpressionAttributeValues = { - ':val': { - 'SS': [foreach_join_parent_task_id] - } - } + TableName=self.name, + Key={"pathspec": {"S": foreach_split_task_id}}, + UpdateExpression="ADD parent_task_ids_for_foreach_join :val", + ExpressionAttributeValues={":val": {"SS": [foreach_join_parent_task_id]}}, ) - def get_parent_task_ids_for_foreach_join(self, - foreach_split_task_id): + def get_parent_task_ids_for_foreach_join(self, foreach_split_task_id): response = self._client.get_item( - TableName = self.name, - Key = { - 'pathspec': { - 'S': foreach_split_task_id - } - }, - ProjectionExpression = 'parent_task_ids_for_foreach_join', - ConsistentRead = True - ) - return response['Item']['parent_task_ids_for_foreach_join']['SS'] + TableName=self.name, + Key={"pathspec": {"S": foreach_split_task_id}}, + ProjectionExpression="parent_task_ids_for_foreach_join", + ConsistentRead=True, + ) + return response["Item"]["parent_task_ids_for_foreach_join"]["SS"] def _get_instance_region(self): - region = os.environ.get('AWS_REGION') + region = os.environ.get("AWS_REGION") # region is available as an env variable in AWS Fargate but not in EC2 if region is not None: return region - metadata_url = "http://169.254.169.254/latest/meta-data/placement/availability-zone/" - r = requests.get( - url = metadata_url + metadata_url = ( + "http://169.254.169.254/latest/meta-data/placement/availability-zone/" ) + r = requests.get(url=metadata_url) if r.status_code != 200: - raise RuntimeError("Failed to query instance metadata. Url [%s]" % metadata_url + - " Error code [%s]" % str(r.status_code)) + raise RuntimeError( + "Failed to query instance metadata. Url [%s]" % metadata_url + + " Error code [%s]" % str(r.status_code) + ) - return r.text[:-1] \ No newline at end of file + return r.text[:-1] diff --git a/metaflow/plugins/aws/step_functions/event_bridge_client.py b/metaflow/plugins/aws/step_functions/event_bridge_client.py index 789c4ef9714..5620b1de039 100644 --- a/metaflow/plugins/aws/step_functions/event_bridge_client.py +++ b/metaflow/plugins/aws/step_functions/event_bridge_client.py @@ -4,11 +4,12 @@ from metaflow.util import to_bytes, to_unicode -class EventBridgeClient(object): +class EventBridgeClient(object): def __init__(self, name): from ..aws_client import get_aws_client - self._client = get_aws_client('events') + + self._client = get_aws_client("events") self.name = format(name) def cron(self, cron): @@ -33,9 +34,7 @@ def schedule(self): def _disable(self): try: - self._client.disable_rule( - Name=self.name - ) + self._client.disable_rule(Name=self.name) except self._client.exceptions.ResourceNotFoundException: pass @@ -43,33 +42,34 @@ def _set(self): # Generate a new rule or update existing rule. self._client.put_rule( Name=self.name, - ScheduleExpression='cron(%s)' % self.cron, - Description='Metaflow generated rule for %s' % self.name, - State='ENABLED' + ScheduleExpression="cron(%s)" % self.cron, + Description="Metaflow generated rule for %s" % self.name, + State="ENABLED", ) # Assign AWS Step Functions ARN to the rule as a target. self._client.put_targets( Rule=self.name, Targets=[ { - 'Id':self.name, - 'Arn':self.state_machine_arn, + "Id": self.name, + "Arn": self.state_machine_arn, # Set input parameters to empty. - 'Input':json.dumps({'Parameters':json.dumps({})}), - 'RoleArn':self.role_arn + "Input": json.dumps({"Parameters": json.dumps({})}), + "RoleArn": self.role_arn, } - ] + ], ) + def format(name): # AWS Event Bridge has a limit of 64 chars for rule names. # We truncate the rule name if the computed name is greater # than 64 chars and append a hashed suffix to ensure uniqueness. if len(name) > 64: - name_hash = to_unicode( - base64.b32encode( - sha1(to_bytes(name)).digest()))[:16].lower() + name_hash = to_unicode(base64.b32encode(sha1(to_bytes(name)).digest()))[ + :16 + ].lower() # construct an 64 character long rule name - return '%s-%s' % (name[:47], name_hash) + return "%s-%s" % (name[:47], name_hash) else: - return name \ No newline at end of file + return name diff --git a/metaflow/plugins/aws/step_functions/production_token.py b/metaflow/plugins/aws/step_functions/production_token.py index 9d84cdb1286..d8f75a019a8 100644 --- a/metaflow/plugins/aws/step_functions/production_token.py +++ b/metaflow/plugins/aws/step_functions/production_token.py @@ -7,13 +7,15 @@ from metaflow.util import to_bytes + def _token_generator(token_prefix): for i in range(10000): - prefix = '%s-%d-' % (token_prefix, i) + prefix = "%s-%d-" % (token_prefix, i) # we need to use a consistent hash here, which is why # random.seed(prefix) or random.seed(hash(prefix)) won't work random.seed(zlib.adler32(to_bytes(prefix))) - yield prefix + ''.join(random.sample(string.ascii_lowercase, 4)) + yield prefix + "".join(random.sample(string.ascii_lowercase, 4)) + def _makedirs(path): # this is for python2 compatibility. @@ -26,6 +28,7 @@ def _makedirs(path): else: raise + def _load_config(path): if os.path.exists(path): with open(path) as f: @@ -33,30 +36,33 @@ def _load_config(path): else: return {} + def _path(token_prefix): - home = os.environ.get('METAFLOW_HOME', '~/.metaflowconfig') - return os.path.expanduser('%s/%s' % (home, token_prefix)) + home = os.environ.get("METAFLOW_HOME", "~/.metaflowconfig") + return os.path.expanduser("%s/%s" % (home, token_prefix)) + def new_token(token_prefix, prev_token=None): if prev_token is None: for token in _token_generator(token_prefix): return token else: - it = dropwhile(lambda x: x != prev_token, - _token_generator(token_prefix)) + it = dropwhile(lambda x: x != prev_token, _token_generator(token_prefix)) for _ in it: return next(it) else: return None + def load_token(token_prefix): config = _load_config(_path(token_prefix)) - return config.get('production_token') + return config.get("production_token") + def store_token(token_prefix, token): path = _path(token_prefix) config = _load_config(path) - config['production_token'] = token + config["production_token"] = token _makedirs(os.path.dirname(path)) - with open(path, 'w') as f: - json.dump(config, f) \ No newline at end of file + with open(path, "w") as f: + json.dump(config, f) diff --git a/metaflow/plugins/aws/step_functions/schedule_decorator.py b/metaflow/plugins/aws/step_functions/schedule_decorator.py index d3e789620ec..a5f95f55a5b 100644 --- a/metaflow/plugins/aws/step_functions/schedule_decorator.py +++ b/metaflow/plugins/aws/step_functions/schedule_decorator.py @@ -2,30 +2,21 @@ class ScheduleDecorator(FlowDecorator): - name = 'schedule' - defaults = {'cron': None, - 'weekly': False, - 'daily': True, - 'hourly': False} + name = "schedule" + defaults = {"cron": None, "weekly": False, "daily": True, "hourly": False} - def flow_init(self, - flow, - graph, - environment, - flow_datastore, - metadata, - logger, - echo, - options): + def flow_init( + self, flow, graph, environment, flow_datastore, metadata, logger, echo, options + ): # Currently supports quartz cron expressions in UTC as defined in # https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions - if self.attributes['cron']: - self.schedule = self.attributes['cron'] - elif self.attributes['weekly']: - self.schedule = '0 0 ? * SUN *' - elif self.attributes['hourly']: - self.schedule = '0 * * * ? *' - elif self.attributes['daily']: - self.schedule = '0 0 * * ? *' + if self.attributes["cron"]: + self.schedule = self.attributes["cron"] + elif self.attributes["weekly"]: + self.schedule = "0 0 ? * SUN *" + elif self.attributes["hourly"]: + self.schedule = "0 * * * ? *" + elif self.attributes["daily"]: + self.schedule = "0 0 * * ? *" else: - self.schedule = None \ No newline at end of file + self.schedule = None diff --git a/metaflow/plugins/aws/step_functions/set_batch_environment.py b/metaflow/plugins/aws/step_functions/set_batch_environment.py index e238e1a81e7..0663cb6acfc 100644 --- a/metaflow/plugins/aws/step_functions/set_batch_environment.py +++ b/metaflow/plugins/aws/step_functions/set_batch_environment.py @@ -4,30 +4,34 @@ from .dynamo_db_client import DynamoDbClient + def export_parameters(output_file): - input = json.loads(os.environ.get('METAFLOW_PARAMETERS', '{}')) - params = json.loads(os.environ.get('METAFLOW_DEFAULT_PARAMETERS', '{}')) + input = json.loads(os.environ.get("METAFLOW_PARAMETERS", "{}")) + params = json.loads(os.environ.get("METAFLOW_DEFAULT_PARAMETERS", "{}")) params.update(input) - with open(output_file, 'w') as f: + with open(output_file, "w") as f: for k in params: # Replace `-` with `_` is parameter names since `-` isn't an # allowed character for environment variables. cli.py will # correctly translate the replaced `-`s. - f.write('export METAFLOW_INIT_%s=%s\n' % - (k.upper().replace('-', '_'), json.dumps(params[k]))) + f.write( + "export METAFLOW_INIT_%s=%s\n" + % (k.upper().replace("-", "_"), json.dumps(params[k])) + ) os.chmod(output_file, 509) + def export_parent_task_ids(output_file): - input = os.environ['METAFLOW_SPLIT_PARENT_TASK_ID'] + input = os.environ["METAFLOW_SPLIT_PARENT_TASK_ID"] task_ids = DynamoDbClient().get_parent_task_ids_for_foreach_join(input) - with open(output_file, 'w') as f: - f.write('export METAFLOW_PARENT_TASK_IDS=%s' % ','.join(task_ids)) + with open(output_file, "w") as f: + f.write("export METAFLOW_PARENT_TASK_IDS=%s" % ",".join(task_ids)) os.chmod(output_file, 509) # TODO: Maybe use click someday instead of conditional. -if __name__ == '__main__': - if sys.argv[1] == 'parameters': +if __name__ == "__main__": + if sys.argv[1] == "parameters": export_parameters(sys.argv[2]) - elif sys.argv[1] == 'parent_tasks': - export_parent_task_ids(sys.argv[2]) \ No newline at end of file + elif sys.argv[1] == "parent_tasks": + export_parent_task_ids(sys.argv[2]) diff --git a/metaflow/plugins/aws/step_functions/step_functions.py b/metaflow/plugins/aws/step_functions/step_functions.py index 8e306e0c3d0..f71b560fdbc 100644 --- a/metaflow/plugins/aws/step_functions/step_functions.py +++ b/metaflow/plugins/aws/step_functions/step_functions.py @@ -13,8 +13,12 @@ from metaflow.parameters import deploy_time_eval from metaflow.decorators import flow_decorators from metaflow.util import compress_list, dict_to_cli_options, to_pascalcase -from metaflow.metaflow_config import SFN_IAM_ROLE, \ - EVENTS_SFN_ACCESS_IAM_ROLE, SFN_DYNAMO_DB_TABLE, SFN_EXECUTION_LOG_GROUP_ARN +from metaflow.metaflow_config import ( + SFN_IAM_ROLE, + EVENTS_SFN_ACCESS_IAM_ROLE, + SFN_DYNAMO_DB_TABLE, + SFN_EXECUTION_LOG_GROUP_ARN, +) from metaflow import R from .step_functions_client import StepFunctionsClient @@ -23,31 +27,34 @@ class StepFunctionsException(MetaflowException): - headline = 'AWS Step Functions error' + headline = "AWS Step Functions error" + class StepFunctionsSchedulingException(MetaflowException): - headline = 'AWS Step Functions scheduling error' + headline = "AWS Step Functions scheduling error" -class StepFunctions(object): - def __init__(self, - name, - graph, - flow, - code_package_sha, - code_package_url, - production_token, - metadata, - flow_datastore, - environment, - event_logger, - monitor, - tags=None, - namespace=None, - username=None, - max_workers=None, - workflow_timeout=None, - is_project=False): +class StepFunctions(object): + def __init__( + self, + name, + graph, + flow, + code_package_sha, + code_package_url, + production_token, + metadata, + flow_datastore, + environment, + event_logger, + monitor, + tags=None, + namespace=None, + username=None, + max_workers=None, + workflow_timeout=None, + is_project=False, + ): self.name = name self.graph = graph self.flow = flow @@ -64,7 +71,7 @@ def __init__(self, self.username = username self.max_workers = max_workers self.workflow_timeout = workflow_timeout - + self._client = StepFunctionsClient() self._workflow = self._compile() self._cron = self._cron() @@ -76,65 +83,74 @@ def to_json(self): def trigger_explanation(self): if self._cron: # Sometime in the future, we should vendor (or write) a utility - # that can translate cron specifications into a human readable + # that can translate cron specifications into a human readable # format and push to the user for a better UX, someday. - return 'This workflow triggers automatically '\ - 'via a cron schedule *%s* defined in AWS EventBridge.' \ + return ( + "This workflow triggers automatically " + "via a cron schedule *%s* defined in AWS EventBridge." % self.event_bridge_rule + ) else: - return 'No triggers defined. '\ - 'You need to launch this workflow manually.' + return "No triggers defined. " "You need to launch this workflow manually." def deploy(self, log_execution_history): if SFN_IAM_ROLE is None: - raise StepFunctionsException("No IAM role found for AWS Step " - "Functions. You can create one " - "following the instructions listed at " - "*https://admin-docs.metaflow.org/meta" - "flow-on-aws/deployment-guide/manual-d" - "eployment#scheduling* and " - "re-configure Metaflow using " - "*metaflow configure aws* on your " - "terminal.") + raise StepFunctionsException( + "No IAM role found for AWS Step " + "Functions. You can create one " + "following the instructions listed at " + "*https://admin-docs.metaflow.org/meta" + "flow-on-aws/deployment-guide/manual-d" + "eployment#scheduling* and " + "re-configure Metaflow using " + "*metaflow configure aws* on your " + "terminal." + ) if log_execution_history: if SFN_EXECUTION_LOG_GROUP_ARN is None: - raise StepFunctionsException("No AWS CloudWatch Logs log " - "group ARN found for emitting " - "state machine execution logs for " - "your workflow. You can set it in " - "your environment by using the " - "METAFLOW_SFN_EXECUTION_LOG_GROUP_ARN " - "environment variable.") + raise StepFunctionsException( + "No AWS CloudWatch Logs log " + "group ARN found for emitting " + "state machine execution logs for " + "your workflow. You can set it in " + "your environment by using the " + "METAFLOW_SFN_EXECUTION_LOG_GROUP_ARN " + "environment variable." + ) try: self._state_machine_arn = self._client.push( - name = self.name, - definition = self.to_json(), - role_arn = SFN_IAM_ROLE, - log_execution_history = log_execution_history - ) + name=self.name, + definition=self.to_json(), + role_arn=SFN_IAM_ROLE, + log_execution_history=log_execution_history, + ) except Exception as e: raise StepFunctionsException(repr(e)) def schedule(self): # Scheduling is currently enabled via AWS Event Bridge. if EVENTS_SFN_ACCESS_IAM_ROLE is None: - raise StepFunctionsSchedulingException("No IAM role found for AWS " - "Events Bridge. You can " - "create one following the " - "instructions listed at " - "*https://admin-docs.metaflo" - "w.org/metaflow-on-aws/deplo" - "yment-guide/manual-deployme" - "nt#scheduling* and " - "re-configure Metaflow " - "using *metaflow configure " - "aws* on your terminal.") + raise StepFunctionsSchedulingException( + "No IAM role found for AWS " + "Events Bridge. You can " + "create one following the " + "instructions listed at " + "*https://admin-docs.metaflo" + "w.org/metaflow-on-aws/deplo" + "yment-guide/manual-deployme" + "nt#scheduling* and " + "re-configure Metaflow " + "using *metaflow configure " + "aws* on your terminal." + ) try: - self.event_bridge_rule = EventBridgeClient(self.name) \ - .cron(self._cron) \ - .role_arn(EVENTS_SFN_ACCESS_IAM_ROLE) \ - .state_machine_arn(self._state_machine_arn) \ - .schedule() + self.event_bridge_rule = ( + EventBridgeClient(self.name) + .cron(self._cron) + .role_arn(EVENTS_SFN_ACCESS_IAM_ROLE) + .state_machine_arn(self._state_machine_arn) + .schedule() + ) except Exception as e: raise StepFunctionsSchedulingException(repr(e)) @@ -145,21 +161,25 @@ def trigger(cls, name, parameters): except Exception as e: raise StepFunctionsException(repr(e)) if state_machine is None: - raise StepFunctionsException("The workflow *%s* doesn't exist " - "on AWS Step Functions. Please " - "deploy your flow first." % name) + raise StepFunctionsException( + "The workflow *%s* doesn't exist " + "on AWS Step Functions. Please " + "deploy your flow first." % name + ) # Dump parameters into `Parameters` input field. - input = json.dumps({"Parameters" : json.dumps(parameters)}) + input = json.dumps({"Parameters": json.dumps(parameters)}) # AWS Step Functions limits input to be 32KiB, but AWS Batch # has it's own limitation of 30KiB for job specification length. # Reserving 10KiB for rest of the job sprecification leaves 20KiB # for us, which should be enough for most use cases for now. if len(input) > 20480: - raise StepFunctionsException("Length of parameter names and " - "values shouldn't exceed 20480 as " - "imposed by AWS Step Functions.") + raise StepFunctionsException( + "Length of parameter names and " + "values shouldn't exceed 20480 as " + "imposed by AWS Step Functions." + ) try: - state_machine_arn = state_machine.get('stateMachineArn') + state_machine_arn = state_machine.get("stateMachineArn") return StepFunctionsClient().trigger(state_machine_arn, input) except Exception as e: raise StepFunctionsException(repr(e)) @@ -171,12 +191,12 @@ def list(cls, name, states): except Exception as e: raise StepFunctionsException(repr(e)) if state_machine is None: - raise StepFunctionsException("The workflow *%s* doesn't exist " - "on AWS Step Functions." % name) + raise StepFunctionsException( + "The workflow *%s* doesn't exist " "on AWS Step Functions." % name + ) try: - state_machine_arn = state_machine.get('stateMachineArn') - return StepFunctionsClient() \ - .list_executions(state_machine_arn, states) + state_machine_arn = state_machine.get("stateMachineArn") + return StepFunctionsClient().list_executions(state_machine_arn, states) except Exception as e: raise StepFunctionsException(repr(e)) @@ -185,18 +205,21 @@ def get_existing_deployment(cls, name): workflow = StepFunctionsClient().get(name) if workflow is not None: try: - start = json.loads(workflow['definition'])['States']['start'] - parameters = start['Parameters']['Parameters'] - return parameters.get('metaflow.owner'), \ - parameters.get('metaflow.production_token') + start = json.loads(workflow["definition"])["States"]["start"] + parameters = start["Parameters"]["Parameters"] + return parameters.get("metaflow.owner"), parameters.get( + "metaflow.production_token" + ) except KeyError as e: - raise StepFunctionsException("An existing non-metaflow " - "workflow with the same name as " - "*%s* already exists in AWS Step " - "Functions. Please modify the " - "name of this flow or delete your " - "existing workflow on AWS Step " - "Functions." % name) + raise StepFunctionsException( + "An existing non-metaflow " + "workflow with the same name as " + "*%s* already exists in AWS Step " + "Functions. Please modify the " + "name of this flow or delete your " + "existing workflow on AWS Step " + "Functions." % name + ) return None def _compile(self): @@ -204,95 +227,97 @@ def _compile(self): # Visit every node of the flow and recursively build the state machine. def _visit(node, workflow, exit_node=None): # Assign an AWS Batch job to the AWS Step Functions state - # and pass the intermediate state by exposing `JobId` and - # `Parameters` to the child job(s) as outputs. `Index` and + # and pass the intermediate state by exposing `JobId` and + # `Parameters` to the child job(s) as outputs. `Index` and # `SplitParentTaskId` are populated optionally, when available. # We can't modify the names of keys in AWS Step Functions aside # from a blessed few which are set as `Parameters` for the Map - # state. That's why even though `JobId` refers to the parent task + # state. That's why even though `JobId` refers to the parent task # id, we can't call it as such. Similar situation for `Parameters`. - state = State(node.name) \ - .batch(self._batch(node)) \ - .output_path('$.[\'JobId\', ' - '\'Parameters\', ' - '\'Index\', ' - '\'SplitParentTaskId\']') + state = ( + State(node.name) + .batch(self._batch(node)) + .output_path( + "$.['JobId', " "'Parameters', " "'Index', " "'SplitParentTaskId']" + ) + ) # End the (sub)workflow if we have reached the end of the flow or # the parent step of matching_join of the sub workflow. - if node.type == 'end' or exit_node in node.out_funcs: + if node.type == "end" or exit_node in node.out_funcs: workflow.add_state(state.end()) - # Continue linear assignment within the (sub)workflow if the node + # Continue linear assignment within the (sub)workflow if the node # doesn't branch or fork. - elif node.type in ('linear', 'join'): + elif node.type in ("linear", "join"): workflow.add_state(state.next(node.out_funcs[0])) _visit(self.graph[node.out_funcs[0]], workflow, exit_node) # Create a `Parallel` state and assign sub workflows if the node # branches out. - elif node.type == 'split-and': - branch_name = hashlib.sha224('&'.join(node.out_funcs) \ - .encode('utf-8')) \ - .hexdigest() + elif node.type == "split-and": + branch_name = hashlib.sha224( + "&".join(node.out_funcs).encode("utf-8") + ).hexdigest() workflow.add_state(state.next(branch_name)) - branch = Parallel(branch_name) \ - .next(node.matching_join) + branch = Parallel(branch_name).next(node.matching_join) # Generate as many sub workflows as branches and recurse. for n in node.out_funcs: branch.branch( _visit( - self.graph[n], - Workflow(n).start_at(n), - node.matching_join)) + self.graph[n], Workflow(n).start_at(n), node.matching_join + ) + ) workflow.add_state(branch) # Continue the traversal from the matching_join. _visit(self.graph[node.matching_join], workflow, exit_node) # Create a `Map` state and assign sub workflow if the node forks. - elif node.type == 'foreach': + elif node.type == "foreach": # Fetch runtime cardinality via an AWS DynamoDb Get call before # configuring the node - cardinality_state_name = '#%s' % node.out_funcs[0] + cardinality_state_name = "#%s" % node.out_funcs[0] workflow.add_state(state.next(cardinality_state_name)) - cardinality_state = State(cardinality_state_name) \ - .dynamo_db(SFN_DYNAMO_DB_TABLE, - '$.JobId', - 'for_each_cardinality') \ - .result_path('$.Result') - iterator_name = '*%s' % node.out_funcs[0] + cardinality_state = ( + State(cardinality_state_name) + .dynamo_db(SFN_DYNAMO_DB_TABLE, "$.JobId", "for_each_cardinality") + .result_path("$.Result") + ) + iterator_name = "*%s" % node.out_funcs[0] workflow.add_state(cardinality_state.next(iterator_name)) workflow.add_state( - Map(iterator_name) \ - .items_path('$.Result.Item.for_each_cardinality.NS') \ - .parameter('JobId.$', '$.JobId') \ - .parameter('SplitParentTaskId.$', '$.JobId') \ - .parameter('Parameters.$', '$.Parameters') \ - .parameter('Index.$', '$$.Map.Item.Value') \ - .next(node.matching_join) \ - .iterator( - _visit( - self.graph[node.out_funcs[0]], - Workflow(node.out_funcs[0]) \ - .start_at(node.out_funcs[0]), - node.matching_join)) \ - .max_concurrency(self.max_workers) \ - .output_path('$.[0]')) + Map(iterator_name) + .items_path("$.Result.Item.for_each_cardinality.NS") + .parameter("JobId.$", "$.JobId") + .parameter("SplitParentTaskId.$", "$.JobId") + .parameter("Parameters.$", "$.Parameters") + .parameter("Index.$", "$$.Map.Item.Value") + .next(node.matching_join) + .iterator( + _visit( + self.graph[node.out_funcs[0]], + Workflow(node.out_funcs[0]).start_at(node.out_funcs[0]), + node.matching_join, + ) + ) + .max_concurrency(self.max_workers) + .output_path("$.[0]") + ) # Continue the traversal from the matching_join. _visit(self.graph[node.matching_join], workflow, exit_node) # We shouldn't ideally ever get here. else: - raise StepFunctionsException("Node type *%s* for step *%s* " - "is not currently supported by " - "AWS Step Functions." - % (node.type, node.name)) + raise StepFunctionsException( + "Node type *%s* for step *%s* " + "is not currently supported by " + "AWS Step Functions." % (node.type, node.name) + ) return workflow - workflow = Workflow(self.name) \ - .start_at('start') + workflow = Workflow(self.name).start_at("start") if self.workflow_timeout: workflow.timeout_seconds(self.workflow_timeout) - return _visit(self.graph['start'], workflow) + return _visit(self.graph["start"], workflow) def _cron(self): - schedule = self.flow._flow_decorators.get('schedule') + schedule = self.flow._flow_decorators.get("schedule") if schedule: return schedule.schedule return None @@ -305,24 +330,26 @@ def _process_parameters(self): # Throw an exception if the parameter is specified twice. norm = param.name.lower() if norm in seen: - raise MetaflowException("Parameter *%s* is specified twice. " - "Note that parameter names are " - "case-insensitive." % param.name) + raise MetaflowException( + "Parameter *%s* is specified twice. " + "Note that parameter names are " + "case-insensitive." % param.name + ) seen.add(norm) - is_required = param.kwargs.get('required', False) + is_required = param.kwargs.get("required", False) # Throw an exception if a schedule is set for a flow with required # parameters with no defaults. We currently don't have any notion # of data triggers in AWS Event Bridge. - if 'default' not in param.kwargs and is_required and has_schedule: - raise MetaflowException("The parameter *%s* does not have a " - "default and is required. Scheduling " - "such parameters via AWS Event Bridge " - "is not currently supported." - % param.name) - value = deploy_time_eval(param.kwargs.get('default')) - parameters.append(dict(name=param.name, - value=value)) + if "default" not in param.kwargs and is_required and has_schedule: + raise MetaflowException( + "The parameter *%s* does not have a " + "default and is required. Scheduling " + "such parameters via AWS Event Bridge " + "is not currently supported." % param.name + ) + value = deploy_time_eval(param.kwargs.get("default")) + parameters.append(dict(name=param.name, value=value)) return parameters def _batch(self, node): @@ -330,147 +357,150 @@ def _batch(self, node): # metaflow.user is only used for setting the AWS Job Name. # Since job executions are no longer tied to a specific user # identity, we will just set their user to `SFN`. We still do need - # access to the owner of the workflow for production tokens, which + # access to the owner of the workflow for production tokens, which # we can stash in metaflow.owner. - 'metaflow.user': 'SFN', - 'metaflow.owner': self.username, - 'metaflow.flow_name': self.flow.name, - 'metaflow.step_name': node.name, - 'metaflow.run_id.$': '$$.Execution.Name', - # Unfortunately we can't set the task id here since AWS Step - # Functions lacks any notion of run-scoped task identifiers. We + "metaflow.user": "SFN", + "metaflow.owner": self.username, + "metaflow.flow_name": self.flow.name, + "metaflow.step_name": node.name, + "metaflow.run_id.$": "$$.Execution.Name", + # Unfortunately we can't set the task id here since AWS Step + # Functions lacks any notion of run-scoped task identifiers. We # instead co-opt the AWS Batch job id as the task id. This also # means that the AWS Batch job name will have missing fields since # the job id is determined at job execution, but since the job id is # part of the job description payload, we don't lose much except for # a few ugly looking black fields in the AWS Batch UI. - - # Also, unfortunately we can't set the retry count since - # `$$.State.RetryCount` resolves to an int dynamically and - # AWS Batch job specification only accepts strings. We handle + # Also, unfortunately we can't set the retry count since + # `$$.State.RetryCount` resolves to an int dynamically and + # AWS Batch job specification only accepts strings. We handle # retries/catch within AWS Batch to get around this limitation. - 'metaflow.version': self.environment.get_environment_info()[ - 'metaflow_version' + "metaflow.version": self.environment.get_environment_info()[ + "metaflow_version" ], # We rely on step names and task ids of parent steps to construct - # input paths for a task. Since the only information we can pass - # between states (via `InputPath` and `ResultPath`) in AWS Step - # Functions is the job description, we run the risk of exceeding - # 32K state size limit rather quickly if we don't filter the job + # input paths for a task. Since the only information we can pass + # between states (via `InputPath` and `ResultPath`) in AWS Step + # Functions is the job description, we run the risk of exceeding + # 32K state size limit rather quickly if we don't filter the job # description to a minimal set of fields. Unfortunately, the partial - # `JsonPath` implementation within AWS Step Functions makes this + # `JsonPath` implementation within AWS Step Functions makes this # work a little non-trivial; it doesn't like dots in keys, so we - # have to add the field again. - # This pattern is repeated in a lot of other places, where we use - # AWS Batch parameters to store AWS Step Functions state - # information, since this field is the only field in the AWS Batch + # have to add the field again. + # This pattern is repeated in a lot of other places, where we use + # AWS Batch parameters to store AWS Step Functions state + # information, since this field is the only field in the AWS Batch # specification that allows us to set key-values. - 'step_name': node.name + "step_name": node.name, } # Store production token within the `start` step, so that subsequent # `step-functions create` calls can perform a rudimentary authorization # check. - if node.name == 'start': - attrs['metaflow.production_token'] = self.production_token + if node.name == "start": + attrs["metaflow.production_token"] = self.production_token # Add env vars from the optional @environment decorator. - env_deco = [deco for deco in node.decorators - if deco.name == 'environment'] + env_deco = [deco for deco in node.decorators if deco.name == "environment"] env = {} if env_deco: - env = env_deco[0].attributes['vars'] + env = env_deco[0].attributes["vars"] - if node.name == 'start': + if node.name == "start": # Initialize parameters for the flow in the `start` step. parameters = self._process_parameters() if parameters: # Get user-defined parameters from State Machine Input. # Since AWS Step Functions doesn't allow for optional inputs - # currently, we have to unfortunately place an artificial + # currently, we have to unfortunately place an artificial # constraint that every parameterized workflow needs to include - # `Parameters` as a key in the input to the workflow. - # `step-functions trigger` already takes care of this + # `Parameters` as a key in the input to the workflow. + # `step-functions trigger` already takes care of this # requirement, but within the UI, the users will be required to # specify an input with key as `Parameters` and value as a - # stringified json of the actual parameters - + # stringified json of the actual parameters - # {"Parameters": "{\"alpha\": \"beta\"}"} - env['METAFLOW_PARAMETERS'] = '$.Parameters' + env["METAFLOW_PARAMETERS"] = "$.Parameters" default_parameters = {} for parameter in parameters: - if parameter['value'] is not None: - default_parameters[parameter['name']] = \ - parameter['value'] + if parameter["value"] is not None: + default_parameters[parameter["name"]] = parameter["value"] # Dump the default values specified in the flow. - env['METAFLOW_DEFAULT_PARAMETERS'] = \ - json.dumps(default_parameters) + env["METAFLOW_DEFAULT_PARAMETERS"] = json.dumps(default_parameters) # `start` step has no upstream input dependencies aside from # parameters. input_paths = None else: # We need to rely on the `InputPath` of the AWS Step Functions # specification to grab task ids and the step names of the parent - # to properly construct input_paths at runtime. Thanks to the - # JsonPath-foo embedded in the parent states, we have this - # information easily available. + # to properly construct input_paths at runtime. Thanks to the + # JsonPath-foo embedded in the parent states, we have this + # information easily available. # Handle foreach join. - if node.type == 'join' and \ - self.graph[node.split_parents[-1]].type == 'foreach': - input_paths = \ - 'sfn-${METAFLOW_RUN_ID}/%s/:' \ - '${METAFLOW_PARENT_TASK_IDS}' % node.in_funcs[0] + if ( + node.type == "join" + and self.graph[node.split_parents[-1]].type == "foreach" + ): + input_paths = ( + "sfn-${METAFLOW_RUN_ID}/%s/:" + "${METAFLOW_PARENT_TASK_IDS}" % node.in_funcs[0] + ) # Unfortunately, AWS Batch only allows strings as value types # in it's specification and we don't have any way to concatenate - # the task ids array from the parent steps within AWS Step - # Functions and pass it down to AWS Batch. We instead have to + # the task ids array from the parent steps within AWS Step + # Functions and pass it down to AWS Batch. We instead have to # rely on publishing the state to DynamoDb and fetching it back # in within the AWS Batch entry point to set # `METAFLOW_PARENT_TASK_IDS`. The state is scoped to the parent # foreach task `METAFLOW_SPLIT_PARENT_TASK_ID`. We decided on - # AWS DynamoDb and not AWS Lambdas, because deploying and - # debugging Lambdas would be a nightmare as far as OSS support + # AWS DynamoDb and not AWS Lambdas, because deploying and + # debugging Lambdas would be a nightmare as far as OSS support # is concerned. - env['METAFLOW_SPLIT_PARENT_TASK_ID'] = \ - '$.Parameters.split_parent_task_id_%s' % \ - node.split_parents[-1] + env["METAFLOW_SPLIT_PARENT_TASK_ID"] = ( + "$.Parameters.split_parent_task_id_%s" % node.split_parents[-1] + ) else: # Set appropriate environment variables for runtime replacement. if len(node.in_funcs) == 1: - input_paths = \ - 'sfn-${METAFLOW_RUN_ID}/%s/${METAFLOW_PARENT_TASK_ID}' \ - % node.in_funcs[0] - env['METAFLOW_PARENT_TASK_ID'] = '$.JobId' + input_paths = ( + "sfn-${METAFLOW_RUN_ID}/%s/${METAFLOW_PARENT_TASK_ID}" + % node.in_funcs[0] + ) + env["METAFLOW_PARENT_TASK_ID"] = "$.JobId" else: # Generate the input paths in a quasi-compressed format. - # See util.decompress_list for why this is written the way + # See util.decompress_list for why this is written the way # it is. - input_paths = 'sfn-${METAFLOW_RUN_ID}:' + ','.join( - '/${METAFLOW_PARENT_%s_STEP}/' - '${METAFLOW_PARENT_%s_TASK_ID}' % (idx, idx) - for idx, _ in enumerate(node.in_funcs)) + input_paths = "sfn-${METAFLOW_RUN_ID}:" + ",".join( + "/${METAFLOW_PARENT_%s_STEP}/" + "${METAFLOW_PARENT_%s_TASK_ID}" % (idx, idx) + for idx, _ in enumerate(node.in_funcs) + ) for idx, _ in enumerate(node.in_funcs): - env['METAFLOW_PARENT_%s_TASK_ID' % idx] = \ - '$.[%s].JobId' % idx - env['METAFLOW_PARENT_%s_STEP' % idx] = \ - '$.[%s].Parameters.step_name' % idx - env['METAFLOW_INPUT_PATHS'] = input_paths + env["METAFLOW_PARENT_%s_TASK_ID" % idx] = "$.[%s].JobId" % idx + env["METAFLOW_PARENT_%s_STEP" % idx] = ( + "$.[%s].Parameters.step_name" % idx + ) + env["METAFLOW_INPUT_PATHS"] = input_paths if node.is_inside_foreach: - # Set the task id of the parent job of the foreach split in - # our favorite dumping ground, the AWS Batch attrs. For - # subsequent descendent tasks, this attrs blob becomes the + # Set the task id of the parent job of the foreach split in + # our favorite dumping ground, the AWS Batch attrs. For + # subsequent descendent tasks, this attrs blob becomes the # input to those descendent tasks. We set and propagate the # task ids pointing to split_parents through every state. - if any(self.graph[n].type == 'foreach' for n in node.in_funcs): - attrs['split_parent_task_id_%s.$' % \ - node.split_parents[-1]] = '$.SplitParentTaskId' + if any(self.graph[n].type == "foreach" for n in node.in_funcs): + attrs[ + "split_parent_task_id_%s.$" % node.split_parents[-1] + ] = "$.SplitParentTaskId" for parent in node.split_parents[:-1]: - if self.graph[parent].type == 'foreach': - attrs['split_parent_task_id_%s.$' % parent] = \ - '$.Parameters.split_parent_task_id_%s' % parent - elif node.type == 'join': - if self.graph[node.split_parents[-1]].type == 'foreach': + if self.graph[parent].type == "foreach": + attrs["split_parent_task_id_%s.$" % parent] = ( + "$.Parameters.split_parent_task_id_%s" % parent + ) + elif node.type == "join": + if self.graph[node.split_parents[-1]].type == "foreach": # A foreach join only gets one set of input from the # parent tasks. We filter the Map state to only output # `$.[0]`, since we don't need any of the other outputs, @@ -481,66 +511,68 @@ def _batch(self, node): # instead of referencing `Parameters` fields by index # (like in `split-and`), we can just reference them # directly. - attrs['split_parent_task_id_%s.$' % \ - node.split_parents[-1]] = \ - '$.Parameters.split_parent_task_id_%s' % \ - node.split_parents[-1] + attrs["split_parent_task_id_%s.$" % node.split_parents[-1]] = ( + "$.Parameters.split_parent_task_id_%s" + % node.split_parents[-1] + ) for parent in node.split_parents[:-1]: - if self.graph[parent].type == 'foreach': - attrs['split_parent_task_id_%s.$' % parent] = \ - '$.Parameters.split_parent_task_id_%s' % \ - parent + if self.graph[parent].type == "foreach": + attrs["split_parent_task_id_%s.$" % parent] = ( + "$.Parameters.split_parent_task_id_%s" % parent + ) else: for parent in node.split_parents: - if self.graph[parent].type == 'foreach': - attrs['split_parent_task_id_%s.$' % parent] = \ - '$.[0].Parameters.split_parent_task_id_%s' % \ - parent + if self.graph[parent].type == "foreach": + attrs["split_parent_task_id_%s.$" % parent] = ( + "$.[0].Parameters.split_parent_task_id_%s" % parent + ) else: for parent in node.split_parents: - if self.graph[parent].type == 'foreach': - attrs['split_parent_task_id_%s.$' % parent] = \ - '$.Parameters.split_parent_task_id_%s' % parent + if self.graph[parent].type == "foreach": + attrs["split_parent_task_id_%s.$" % parent] = ( + "$.Parameters.split_parent_task_id_%s" % parent + ) - # Set `METAFLOW_SPLIT_PARENT_TASK_ID_FOR_FOREACH_JOIN` if the - # next transition is to a foreach join, so that the + # Set `METAFLOW_SPLIT_PARENT_TASK_ID_FOR_FOREACH_JOIN` if the + # next transition is to a foreach join, so that the # stepfunctions decorator can write the mapping for input path # to DynamoDb. - if any(self.graph[n].type == 'join' and \ - self.graph[self.graph[n].split_parents[-1]].type == \ - 'foreach' - for n in node.out_funcs): - env['METAFLOW_SPLIT_PARENT_TASK_ID_FOR_FOREACH_JOIN'] = \ - attrs['split_parent_task_id_%s.$' % \ - self.graph[node.out_funcs[0]].split_parents[-1]] + if any( + self.graph[n].type == "join" + and self.graph[self.graph[n].split_parents[-1]].type == "foreach" + for n in node.out_funcs + ): + env["METAFLOW_SPLIT_PARENT_TASK_ID_FOR_FOREACH_JOIN"] = attrs[ + "split_parent_task_id_%s.$" + % self.graph[node.out_funcs[0]].split_parents[-1] + ] # Set ttl for the values we set in AWS DynamoDB. - if node.type == 'foreach': + if node.type == "foreach": if self.workflow_timeout: - env['METAFLOW_SFN_WORKFLOW_TIMEOUT'] = \ - self.workflow_timeout + env["METAFLOW_SFN_WORKFLOW_TIMEOUT"] = self.workflow_timeout # Handle split index for for-each. - if any(self.graph[n].type == 'foreach' for n in node.in_funcs): - env['METAFLOW_SPLIT_INDEX'] = '$.Index' - - env['METAFLOW_CODE_URL'] = self.code_package_url - env['METAFLOW_FLOW_NAME'] = attrs['metaflow.flow_name'] - env['METAFLOW_STEP_NAME'] = attrs['metaflow.step_name'] - env['METAFLOW_RUN_ID'] = attrs['metaflow.run_id.$'] - env['METAFLOW_PRODUCTION_TOKEN'] = self.production_token - env['SFN_STATE_MACHINE'] = self.name - env['METAFLOW_OWNER'] = attrs['metaflow.owner'] + if any(self.graph[n].type == "foreach" for n in node.in_funcs): + env["METAFLOW_SPLIT_INDEX"] = "$.Index" + + env["METAFLOW_CODE_URL"] = self.code_package_url + env["METAFLOW_FLOW_NAME"] = attrs["metaflow.flow_name"] + env["METAFLOW_STEP_NAME"] = attrs["metaflow.step_name"] + env["METAFLOW_RUN_ID"] = attrs["metaflow.run_id.$"] + env["METAFLOW_PRODUCTION_TOKEN"] = self.production_token + env["SFN_STATE_MACHINE"] = self.name + env["METAFLOW_OWNER"] = attrs["metaflow.owner"] # Can't set `METAFLOW_TASK_ID` due to lack of run-scoped identifiers. # We will instead rely on `AWS_BATCH_JOB_ID` as the task identifier. # Can't set `METAFLOW_RETRY_COUNT` either due to integer casting issue. - metadata_env = self.metadata.get_runtime_environment('step-functions') + metadata_env = self.metadata.get_runtime_environment("step-functions") env.update(metadata_env) metaflow_version = self.environment.get_environment_info() - metaflow_version['flow_name'] = self.graph.name - metaflow_version['production_token'] = self.production_token - env['METAFLOW_VERSION'] = json.dumps(metaflow_version) + metaflow_version["flow_name"] = self.graph.name + metaflow_version["production_token"] = self.production_token + env["METAFLOW_VERSION"] = json.dumps(metaflow_version) # Set AWS DynamoDb Table Name for state tracking for for-eaches. # There are three instances when metaflow runtime directly interacts @@ -550,73 +582,83 @@ def _batch(self, node): # Functions. # 2. To set the input paths from the parent steps of a foreach join. # 3. To read the input paths in a foreach join. - if node.type == 'foreach' or \ - (node.is_inside_foreach and \ - any(self.graph[n].type == 'join' and \ - self.graph[self.graph[n].split_parents[-1]].type == \ - 'foreach' for n in node.out_funcs)) or \ - (node.type == 'join' and \ - self.graph[node.split_parents[-1]].type == 'foreach'): + if ( + node.type == "foreach" + or ( + node.is_inside_foreach + and any( + self.graph[n].type == "join" + and self.graph[self.graph[n].split_parents[-1]].type == "foreach" + for n in node.out_funcs + ) + ) + or ( + node.type == "join" + and self.graph[node.split_parents[-1]].type == "foreach" + ) + ): if SFN_DYNAMO_DB_TABLE is None: - raise StepFunctionsException("An AWS DynamoDB table is needed " - "to support foreach in your flow. " - "You can create one following the " - "instructions listed at *https://a" - "dmin-docs.metaflow.org/metaflow-o" - "n-aws/deployment-guide/manual-dep" - "loyment#scheduling* and " - "re-configure Metaflow using " - "*metaflow configure aws* on your " - "terminal.") - env['METAFLOW_SFN_DYNAMO_DB_TABLE'] = SFN_DYNAMO_DB_TABLE + raise StepFunctionsException( + "An AWS DynamoDB table is needed " + "to support foreach in your flow. " + "You can create one following the " + "instructions listed at *https://a" + "dmin-docs.metaflow.org/metaflow-o" + "n-aws/deployment-guide/manual-dep" + "loyment#scheduling* and " + "re-configure Metaflow using " + "*metaflow configure aws* on your " + "terminal." + ) + env["METAFLOW_SFN_DYNAMO_DB_TABLE"] = SFN_DYNAMO_DB_TABLE # Resolve AWS Batch resource requirements. - batch_deco = [deco for deco in node.decorators - if deco.name == 'batch'][0] + batch_deco = [deco for deco in node.decorators if deco.name == "batch"][0] resources = batch_deco.attributes # Resolve retry strategy. - user_code_retries, total_retries= self._get_retries(node) + user_code_retries, total_retries = self._get_retries(node) task_spec = { - 'flow_name': attrs['metaflow.flow_name'], - 'step_name': attrs['metaflow.step_name'], - 'run_id': 'sfn-$METAFLOW_RUN_ID', - # Use AWS Batch job identifier as the globally unique + "flow_name": attrs["metaflow.flow_name"], + "step_name": attrs["metaflow.step_name"], + "run_id": "sfn-$METAFLOW_RUN_ID", + # Use AWS Batch job identifier as the globally unique # task identifier. - 'task_id': '$AWS_BATCH_JOB_ID', + "task_id": "$AWS_BATCH_JOB_ID", # Since retries are handled by AWS Batch, we can rely on # AWS_BATCH_JOB_ATTEMPT as the job counter. - 'retry_count': '$((AWS_BATCH_JOB_ATTEMPT-1))' + "retry_count": "$((AWS_BATCH_JOB_ATTEMPT-1))", } - return Batch(self.metadata, self.environment) \ - .create_job( - step_name=node.name, - step_cli=self._step_cli(node, - input_paths, - self.code_package_url, - user_code_retries), - task_spec=task_spec, - code_package_sha=self.code_package_sha, - code_package_url=self.code_package_url, - code_package_ds=self.flow_datastore.TYPE, - image=resources['image'], - queue=resources['queue'], - iam_role=resources['iam_role'], - execution_role=resources['execution_role'], - cpu=resources['cpu'], - gpu=resources['gpu'], - memory=resources['memory'], - run_time_limit=batch_deco.run_time_limit, - shared_memory=resources['shared_memory'], - max_swap=resources['max_swap'], - swappiness=resources['swappiness'], - env=env, - attrs=attrs, - host_volumes=resources['host_volumes'], - ) \ - .attempts(total_retries + 1) + return ( + Batch(self.metadata, self.environment) + .create_job( + step_name=node.name, + step_cli=self._step_cli( + node, input_paths, self.code_package_url, user_code_retries + ), + task_spec=task_spec, + code_package_sha=self.code_package_sha, + code_package_url=self.code_package_url, + code_package_ds=self.flow_datastore.TYPE, + image=resources["image"], + queue=resources["queue"], + iam_role=resources["iam_role"], + execution_role=resources["execution_role"], + cpu=resources["cpu"], + gpu=resources["gpu"], + memory=resources["memory"], + run_time_limit=batch_deco.run_time_limit, + shared_memory=resources["shared_memory"], + max_swap=resources["max_swap"], + swappiness=resources["swappiness"], + env=env, + attrs=attrs, + host_volumes=resources["host_volumes"], + ) + .attempts(total_retries + 1) + ) def _get_retries(self, node): max_user_code_retries = 0 @@ -625,18 +667,12 @@ def _get_retries(self, node): # the max of them. for deco in node.decorators: user_code_retries, error_retries = deco.step_task_retry_count() - max_user_code_retries = max(max_user_code_retries, - user_code_retries) + max_user_code_retries = max(max_user_code_retries, user_code_retries) max_error_retries = max(max_error_retries, error_retries) - return max_user_code_retries,\ - max_user_code_retries + max_error_retries + return max_user_code_retries, max_user_code_retries + max_error_retries - def _step_cli(self, - node, - paths, - code_package_url, - user_code_retries): + def _step_cli(self, node, paths, code_package_url, user_code_retries): cmds = [] script_name = os.path.basename(sys.argv[0]) @@ -648,7 +684,7 @@ def _step_cli(self, entrypoint = [executable, script_name] # Use AWS Batch job identifier as the globally unique task identifier. - task_id = '${AWS_BATCH_JOB_ID}' + task_id = "${AWS_BATCH_JOB_ID}" # FlowDecorators can define their own top-level options. They are # responsible for adding their own top-level options and values through @@ -658,90 +694,101 @@ def _step_cli(self, top_opts_dict.update(deco.get_top_level_options()) top_opts = list(dict_to_cli_options(top_opts_dict)) - if node.name == 'start': + if node.name == "start": # We need a separate unique ID for the special _parameters task - task_id_params = '%s-params' % task_id + task_id_params = "%s-params" % task_id # Export user-defined parameters into runtime environment - param_file = ''.join(random.choice(string.ascii_lowercase) - for _ in range(10)) - export_params = \ - 'python -m ' \ - 'metaflow.plugins.aws.step_functions.set_batch_environment ' \ - 'parameters %s && . `pwd`/%s' % (param_file, param_file) - params = entrypoint + top_opts +\ - ['--quiet', - '--metadata=%s' % self.metadata.TYPE, - '--environment=%s' % self.environment.TYPE, - '--datastore=s3', - '--event-logger=%s' % self.event_logger.logger_type, - '--monitor=%s' % self.monitor.monitor_type, - '--no-pylint', - 'init', - '--run-id sfn-$METAFLOW_RUN_ID', - '--task-id %s' % task_id_params] + param_file = "".join( + random.choice(string.ascii_lowercase) for _ in range(10) + ) + export_params = ( + "python -m " + "metaflow.plugins.aws.step_functions.set_batch_environment " + "parameters %s && . `pwd`/%s" % (param_file, param_file) + ) + params = ( + entrypoint + + top_opts + + [ + "--quiet", + "--metadata=%s" % self.metadata.TYPE, + "--environment=%s" % self.environment.TYPE, + "--datastore=s3", + "--event-logger=%s" % self.event_logger.logger_type, + "--monitor=%s" % self.monitor.monitor_type, + "--no-pylint", + "init", + "--run-id sfn-$METAFLOW_RUN_ID", + "--task-id %s" % task_id_params, + ] + ) # Assign tags to run objects. if self.tags: - params.extend('--tag %s' % tag for tag in self.tags) + params.extend("--tag %s" % tag for tag in self.tags) - # If the start step gets retried, we must be careful not to - # regenerate multiple parameters tasks. Hence we check first if + # If the start step gets retried, we must be careful not to + # regenerate multiple parameters tasks. Hence we check first if # _parameters exists already. - exists = entrypoint +\ - ['dump', - '--max-value-size=0', - 'sfn-${METAFLOW_RUN_ID}/_parameters/%s' % (task_id_params)] - cmd = 'if ! %s >/dev/null 2>/dev/null; then %s && %s; fi'\ - % (' '.join(exists), export_params, ' '.join(params)) + exists = entrypoint + [ + "dump", + "--max-value-size=0", + "sfn-${METAFLOW_RUN_ID}/_parameters/%s" % (task_id_params), + ] + cmd = "if ! %s >/dev/null 2>/dev/null; then %s && %s; fi" % ( + " ".join(exists), + export_params, + " ".join(params), + ) cmds.append(cmd) - paths = 'sfn-${METAFLOW_RUN_ID}/_parameters/%s' % (task_id_params) - - if node.type == 'join' and\ - self.graph[node.split_parents[-1]].type == 'foreach': - parent_tasks_file = ''.join(random.choice(string.ascii_lowercase) - for _ in range(10)) - export_parent_tasks = \ - 'python -m ' \ - 'metaflow.plugins.aws.step_functions.set_batch_environment ' \ - 'parent_tasks %s && . `pwd`/%s' \ - % (parent_tasks_file, parent_tasks_file) + paths = "sfn-${METAFLOW_RUN_ID}/_parameters/%s" % (task_id_params) + + if node.type == "join" and self.graph[node.split_parents[-1]].type == "foreach": + parent_tasks_file = "".join( + random.choice(string.ascii_lowercase) for _ in range(10) + ) + export_parent_tasks = ( + "python -m " + "metaflow.plugins.aws.step_functions.set_batch_environment " + "parent_tasks %s && . `pwd`/%s" % (parent_tasks_file, parent_tasks_file) + ) cmds.append(export_parent_tasks) top_level = top_opts + [ - '--quiet', - '--metadata=%s' % self.metadata.TYPE, - '--environment=%s' % self.environment.TYPE, - '--datastore=%s' % self.flow_datastore.TYPE, - '--datastore-root=%s' % self.flow_datastore.datastore_root, - '--event-logger=%s' % self.event_logger.logger_type, - '--monitor=%s' % self.monitor.monitor_type, - '--no-pylint', - '--with=step_functions_internal' + "--quiet", + "--metadata=%s" % self.metadata.TYPE, + "--environment=%s" % self.environment.TYPE, + "--datastore=%s" % self.flow_datastore.TYPE, + "--datastore-root=%s" % self.flow_datastore.datastore_root, + "--event-logger=%s" % self.event_logger.logger_type, + "--monitor=%s" % self.monitor.monitor_type, + "--no-pylint", + "--with=step_functions_internal", ] step = [ - 'step', + "step", node.name, - '--run-id sfn-$METAFLOW_RUN_ID', - '--task-id %s' % task_id, + "--run-id sfn-$METAFLOW_RUN_ID", + "--task-id %s" % task_id, # Since retries are handled by AWS Batch, we can rely on # AWS_BATCH_JOB_ATTEMPT as the job counter. - '--retry-count $((AWS_BATCH_JOB_ATTEMPT-1))', - '--max-user-code-retries %d' % user_code_retries, - '--input-paths %s' % paths, + "--retry-count $((AWS_BATCH_JOB_ATTEMPT-1))", + "--max-user-code-retries %d" % user_code_retries, + "--input-paths %s" % paths, # Set decorator to batch to execute `task_*` hooks for batch # decorator. - '--with=batch' + "--with=batch", ] - if any(self.graph[n].type == 'foreach' for n in node.in_funcs): + if any(self.graph[n].type == "foreach" for n in node.in_funcs): # We set the `METAFLOW_SPLIT_INDEX` through JSONPath-foo # to pass the state from the parent DynamoDb state for for-each. - step.append('--split-index $METAFLOW_SPLIT_INDEX') + step.append("--split-index $METAFLOW_SPLIT_INDEX") if self.tags: - step.extend('--tag %s' % tag for tag in self.tags) + step.extend("--tag %s" % tag for tag in self.tags) if self.namespace is not None: - step.append('--namespace=%s' % self.namespace) - cmds.append(' '.join(entrypoint + top_level + step)) - return ' && '.join(cmds) + step.append("--namespace=%s" % self.namespace) + cmds.append(" ".join(entrypoint + top_level + step)) + return " && ".join(cmds) class Workflow(object): @@ -751,15 +798,15 @@ def __init__(self, name): self.payload = tree() def start_at(self, start_at): - self.payload['StartAt'] = start_at + self.payload["StartAt"] = start_at return self def add_state(self, state): - self.payload['States'][state.name] = state.payload + self.payload["States"][state.name] = state.payload return self def timeout_seconds(self, timeout_seconds): - self.payload['TimeoutSeconds'] = timeout_seconds + self.payload["TimeoutSeconds"] = timeout_seconds return self def to_json(self, pretty=False): @@ -767,132 +814,130 @@ def to_json(self, pretty=False): class State(object): - def __init__(self, name): self.name = name tree = lambda: defaultdict(tree) self.payload = tree() - self.payload['Type'] = 'Task' + self.payload["Type"] = "Task" def resource(self, resource): - self.payload['Resource'] = resource + self.payload["Resource"] = resource return self def next(self, state): - self.payload['Next'] = state + self.payload["Next"] = state return self - + def end(self): - self.payload['End'] = True + self.payload["End"] = True return self def parameter(self, name, value): - self.payload['Parameters'][name] = value + self.payload["Parameters"][name] = value return self def output_path(self, output_path): - self.payload['OutputPath'] = output_path + self.payload["OutputPath"] = output_path return self def result_path(self, result_path): - self.payload['ResultPath'] = result_path + self.payload["ResultPath"] = result_path return self def _partition(self): # This is needed to support AWS Gov Cloud and AWS CN regions - return SFN_IAM_ROLE.split(':')[1] + return SFN_IAM_ROLE.split(":")[1] def batch(self, job): - self.resource('arn:%s:states:::batch:submitJob.sync' - % self._partition()) \ - .parameter('JobDefinition', job.payload['jobDefinition']) \ - .parameter('JobName', job.payload['jobName']) \ - .parameter('JobQueue', job.payload['jobQueue']) \ - .parameter('Parameters', job.payload['parameters']) \ - .parameter('ContainerOverrides', - to_pascalcase(job.payload['containerOverrides'])) \ - .parameter('RetryStrategy', - to_pascalcase(job.payload['retryStrategy'])) \ - .parameter('Timeout', - to_pascalcase(job.payload['timeout'])) + self.resource( + "arn:%s:states:::batch:submitJob.sync" % self._partition() + ).parameter("JobDefinition", job.payload["jobDefinition"]).parameter( + "JobName", job.payload["jobName"] + ).parameter( + "JobQueue", job.payload["jobQueue"] + ).parameter( + "Parameters", job.payload["parameters"] + ).parameter( + "ContainerOverrides", to_pascalcase(job.payload["containerOverrides"]) + ).parameter( + "RetryStrategy", to_pascalcase(job.payload["retryStrategy"]) + ).parameter( + "Timeout", to_pascalcase(job.payload["timeout"]) + ) # tags may not be present in all scenarios - if 'tags' in job.payload: - self.parameter('Tags', job.payload['tags']) + if "tags" in job.payload: + self.parameter("Tags", job.payload["tags"]) return self def dynamo_db(self, table_name, primary_key, values): - self.resource('arn:%s:states:::dynamodb:getItem' % self._partition()) \ - .parameter('TableName', table_name) \ - .parameter('Key', { - "pathspec": { - "S.$": primary_key - } - }) \ - .parameter('ConsistentRead', True) \ - .parameter('ProjectionExpression', values) - return self + self.resource("arn:%s:states:::dynamodb:getItem" % self._partition()).parameter( + "TableName", table_name + ).parameter("Key", {"pathspec": {"S.$": primary_key}}).parameter( + "ConsistentRead", True + ).parameter( + "ProjectionExpression", values + ) + return self class Parallel(object): - def __init__(self, name): self.name = name tree = lambda: defaultdict(tree) self.payload = tree() - self.payload['Type'] = 'Parallel' + self.payload["Type"] = "Parallel" def branch(self, workflow): - if 'Branches' not in self.payload: - self.payload['Branches'] = [] - self.payload['Branches'].append(workflow.payload) + if "Branches" not in self.payload: + self.payload["Branches"] = [] + self.payload["Branches"].append(workflow.payload) return self def next(self, state): - self.payload['Next'] = state + self.payload["Next"] = state return self def output_path(self, output_path): - self.payload['OutputPath'] = output_path + self.payload["OutputPath"] = output_path return self def result_path(self, result_path): - self.payload['ResultPath'] = result_path - return self + self.payload["ResultPath"] = result_path + return self class Map(object): - def __init__(self, name): self.name = name tree = lambda: defaultdict(tree) self.payload = tree() - self.payload['Type'] = 'Map' - self.payload['MaxConcurrency'] = 0 + self.payload["Type"] = "Map" + self.payload["MaxConcurrency"] = 0 def iterator(self, workflow): - self.payload['Iterator'] = workflow.payload + self.payload["Iterator"] = workflow.payload return self def next(self, state): - self.payload['Next'] = state + self.payload["Next"] = state return self def items_path(self, items_path): - self.payload['ItemsPath'] = items_path + self.payload["ItemsPath"] = items_path return self def parameter(self, name, value): - self.payload['Parameters'][name] = value + self.payload["Parameters"][name] = value return self def max_concurrency(self, max_concurrency): - self.payload['MaxConcurrency'] = max_concurrency + self.payload["MaxConcurrency"] = max_concurrency return self def output_path(self, output_path): - self.payload['OutputPath'] = output_path + self.payload["OutputPath"] = output_path return self def result_path(self, result_path): - self.payload['ResultPath'] = result_path - return self \ No newline at end of file + self.payload["ResultPath"] = result_path + return self diff --git a/metaflow/plugins/aws/step_functions/step_functions_cli.py b/metaflow/plugins/aws/step_functions/step_functions_cli.py index 3192cc593d2..afb3bc001ee 100644 --- a/metaflow/plugins/aws/step_functions/step_functions_cli.py +++ b/metaflow/plugins/aws/step_functions/step_functions_cli.py @@ -15,258 +15,303 @@ from .step_functions import StepFunctions from .production_token import load_token, store_token, new_token -VALID_NAME = re.compile('[^a-zA-Z0-9_\-\.]') +VALID_NAME = re.compile("[^a-zA-Z0-9_\-\.]") + class IncorrectProductionToken(MetaflowException): headline = "Incorrect production token" + class IncorrectMetadataServiceVersion(MetaflowException): headline = "Incorrect version for metaflow service" + class StepFunctionsStateMachineNameTooLong(MetaflowException): headline = "AWS Step Functions state machine name too long" + @click.group() def cli(): pass + @cli.group(help="Commands related to AWS Step Functions.") -@click.option('--name', - default=None, - type=str, - help="State Machine name. The flow name is used instead " - "if this option is not specified") +@click.option( + "--name", + default=None, + type=str, + help="State Machine name. The flow name is used instead " + "if this option is not specified", +) @click.pass_obj -def step_functions(obj, - name=None): +def step_functions(obj, name=None): obj.check(obj.graph, obj.flow, obj.environment, pylint=obj.pylint) - obj.state_machine_name, obj.token_prefix, obj.is_project = \ - resolve_state_machine_name(obj, name) - -@step_functions.command(help="Deploy a new version of this workflow to " - "AWS Step Functions.") -@click.option('--authorize', - default=None, - help="Authorize using this production token. You need this " - "when you are re-deploying an existing flow for the first " - "time. The token is cached in METAFLOW_HOME, so you only " - "need to specify this once.") -@click.option('--generate-new-token', - is_flag=True, - help="Generate a new production token for this flow. " - "This will move the production flow to a new " - "namespace.") -@click.option('--new-token', - 'given_token', - default=None, - help="Use the given production token for this flow. " - "This will move the production flow to the given " - "namespace.") -@click.option('--tag', - 'tags', - multiple=True, - default=None, - help="Annotate all objects produced by AWS Step Functions runs " - "with the given tag. You can specify this option multiple " - "times to attach multiple tags.") -@click.option('--namespace', - 'user_namespace', - default=None, - help="Change the namespace from the default (production token) " - "to the given tag. See run --help for more information.") -@click.option('--only-json', - is_flag=True, - default=False, - help="Only print out JSON sent to AWS Step Functions. Do not " - "deploy anything.") -@click.option('--max-workers', - default=100, - show_default=True, - help="Maximum number of parallel processes.") -@click.option('--workflow-timeout', - default=None, - type=int, - help="Workflow timeout in seconds.") -@click.option('--log-execution-history', - is_flag=True, - help="Log AWS Step Functions execution history to AWS CloudWatch " - "Logs log group.") + ( + obj.state_machine_name, + obj.token_prefix, + obj.is_project, + ) = resolve_state_machine_name(obj, name) + + +@step_functions.command( + help="Deploy a new version of this workflow to " "AWS Step Functions." +) +@click.option( + "--authorize", + default=None, + help="Authorize using this production token. You need this " + "when you are re-deploying an existing flow for the first " + "time. The token is cached in METAFLOW_HOME, so you only " + "need to specify this once.", +) +@click.option( + "--generate-new-token", + is_flag=True, + help="Generate a new production token for this flow. " + "This will move the production flow to a new " + "namespace.", +) +@click.option( + "--new-token", + "given_token", + default=None, + help="Use the given production token for this flow. " + "This will move the production flow to the given " + "namespace.", +) +@click.option( + "--tag", + "tags", + multiple=True, + default=None, + help="Annotate all objects produced by AWS Step Functions runs " + "with the given tag. You can specify this option multiple " + "times to attach multiple tags.", +) +@click.option( + "--namespace", + "user_namespace", + default=None, + help="Change the namespace from the default (production token) " + "to the given tag. See run --help for more information.", +) +@click.option( + "--only-json", + is_flag=True, + default=False, + help="Only print out JSON sent to AWS Step Functions. Do not " "deploy anything.", +) +@click.option( + "--max-workers", + default=100, + show_default=True, + help="Maximum number of parallel processes.", +) +@click.option( + "--workflow-timeout", default=None, type=int, help="Workflow timeout in seconds." +) +@click.option( + "--log-execution-history", + is_flag=True, + help="Log AWS Step Functions execution history to AWS CloudWatch " + "Logs log group.", +) @click.pass_obj -def create(obj, - tags=None, - user_namespace=None, - only_json=False, - authorize=None, - generate_new_token=False, - given_token=None, - max_workers=None, - workflow_timeout=None, - log_execution_history=False): - obj.echo("Deploying *%s* to AWS Step Functions..." % obj.state_machine_name, - bold=True) +def create( + obj, + tags=None, + user_namespace=None, + only_json=False, + authorize=None, + generate_new_token=False, + given_token=None, + max_workers=None, + workflow_timeout=None, + log_execution_history=False, +): + obj.echo( + "Deploying *%s* to AWS Step Functions..." % obj.state_machine_name, bold=True + ) check_metadata_service_version(obj) - token = resolve_token(obj.state_machine_name, - obj.token_prefix, - obj, - authorize, - given_token, - generate_new_token, - obj.is_project) - - flow = make_flow(obj, - token, - obj.state_machine_name, - tags, - user_namespace, - max_workers, - workflow_timeout, - obj.is_project) + token = resolve_token( + obj.state_machine_name, + obj.token_prefix, + obj, + authorize, + given_token, + generate_new_token, + obj.is_project, + ) + + flow = make_flow( + obj, + token, + obj.state_machine_name, + tags, + user_namespace, + max_workers, + workflow_timeout, + obj.is_project, + ) if only_json: obj.echo_always(flow.to_json(), err=False, no_bold=True) else: flow.deploy(log_execution_history) - obj.echo("State Machine *{state_machine}* " - "for flow *{name}* pushed to " - "AWS Step Functions successfully.\n" - .format(state_machine=obj.state_machine_name, - name=current.flow_name), - bold=True) + obj.echo( + "State Machine *{state_machine}* " + "for flow *{name}* pushed to " + "AWS Step Functions successfully.\n".format( + state_machine=obj.state_machine_name, name=current.flow_name + ), + bold=True, + ) if obj._is_state_machine_name_hashed: - obj.echo("Note that the flow was deployed with a truncated name " - "due to a length limit on AWS Step Functions. The " - "original long name is stored in task metadata.\n") + obj.echo( + "Note that the flow was deployed with a truncated name " + "due to a length limit on AWS Step Functions. The " + "original long name is stored in task metadata.\n" + ) flow.schedule() obj.echo("What will trigger execution of the workflow:", bold=True) obj.echo(flow.trigger_explanation(), indent=True) - + def check_metadata_service_version(obj): metadata = obj.metadata version = metadata.version() - if version == 'local': + if version == "local": return - elif version is not None and LooseVersion(version) >= LooseVersion('2.0.2'): + elif version is not None and LooseVersion(version) >= LooseVersion("2.0.2"): # Metaflow metadata service needs to be at least at version 2.0.2 return else: obj.echo("") - obj.echo("You are running a version of the metaflow service " - "that currently doesn't support AWS Step Functions. ") - obj.echo("For more information on how to upgrade your " - "service to a compatible version (>= 2.0.2), visit:") - obj.echo(" https://admin-docs.metaflow.org/metaflow-on-aws/operation" - "s-guide/metaflow-service-migration-guide", fg='green') - obj.echo("Once you have upgraded your metadata service, please " - "re-execute your command.") - raise IncorrectMetadataServiceVersion("Try again with a more recent " - "version of metaflow service " - "(>=2.0.2).") + obj.echo( + "You are running a version of the metaflow service " + "that currently doesn't support AWS Step Functions. " + ) + obj.echo( + "For more information on how to upgrade your " + "service to a compatible version (>= 2.0.2), visit:" + ) + obj.echo( + " https://admin-docs.metaflow.org/metaflow-on-aws/operation" + "s-guide/metaflow-service-migration-guide", + fg="green", + ) + obj.echo( + "Once you have upgraded your metadata service, please " + "re-execute your command." + ) + raise IncorrectMetadataServiceVersion( + "Try again with a more recent " "version of metaflow service " "(>=2.0.2)." + ) + def resolve_state_machine_name(obj, name): def attach_prefix(name): - if SFN_STATE_MACHINE_PREFIX is not None: - return SFN_STATE_MACHINE_PREFIX + '_' + name - return name - project = current.get('project_name') + if SFN_STATE_MACHINE_PREFIX is not None: + return SFN_STATE_MACHINE_PREFIX + "_" + name + return name + + project = current.get("project_name") obj._is_state_machine_name_hashed = False if project: if name: - raise MetaflowException("--name is not supported for @projects. " - "Use --branch instead.") + raise MetaflowException( + "--name is not supported for @projects. " "Use --branch instead." + ) state_machine_name = attach_prefix(current.project_flow_name) - project_branch = to_bytes('.'.join((project, current.branch_name))) - token_prefix = 'mfprj-%s' % to_unicode( - base64.b32encode( - sha1(project_branch).digest()))[:16] + project_branch = to_bytes(".".join((project, current.branch_name))) + token_prefix = ( + "mfprj-%s" + % to_unicode(base64.b32encode(sha1(project_branch).digest()))[:16] + ) is_project = True # AWS Step Functions has a limit of 80 chars for state machine names. # We truncate the state machine name if the computed name is greater # than 60 chars and append a hashed suffix to ensure uniqueness. if len(state_machine_name) > 60: name_hash = to_unicode( - base64.b32encode( - sha1(to_bytes(state_machine_name)) \ - .digest()))[:16].lower() - state_machine_name =\ - '%s-%s' % (state_machine_name[:60], name_hash) + base64.b32encode(sha1(to_bytes(state_machine_name)).digest()) + )[:16].lower() + state_machine_name = "%s-%s" % (state_machine_name[:60], name_hash) obj._is_state_machine_name_hashed = True else: if name and VALID_NAME.search(name): - raise MetaflowException( - "Name '%s' contains invalid characters." % name) + raise MetaflowException("Name '%s' contains invalid characters." % name) state_machine_name = attach_prefix(name if name else current.flow_name) token_prefix = state_machine_name is_project = False if len(state_machine_name) > 80: - msg = "The full name of the workflow:\n*%s*\nis longer than 80 "\ - "characters.\n\n"\ - "To deploy this workflow to AWS Step Functions, please "\ - "assign a shorter name\nusing the option\n"\ - "*step-functions --name create*." % state_machine_name + msg = ( + "The full name of the workflow:\n*%s*\nis longer than 80 " + "characters.\n\n" + "To deploy this workflow to AWS Step Functions, please " + "assign a shorter name\nusing the option\n" + "*step-functions --name create*." % state_machine_name + ) raise StepFunctionsStateMachineNameTooLong(msg) return state_machine_name, token_prefix.lower(), is_project -def make_flow(obj, - token, - name, - tags, - namespace, - max_workers, - workflow_timeout, - is_project): - if obj.flow_datastore.TYPE != 's3': + +def make_flow( + obj, token, name, tags, namespace, max_workers, workflow_timeout, is_project +): + if obj.flow_datastore.TYPE != "s3": raise MetaflowException("AWS Step Functions requires --datastore=s3.") # Attach AWS Batch decorator to the flow decorators._attach_decorators(obj.flow, [BatchDecorator.name]) - decorators._init_step_decorators(obj.flow, - obj.graph, - obj.environment, - obj.flow_datastore, - obj.logger) + decorators._init_step_decorators( + obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger + ) obj.package = MetaflowPackage( - obj.flow, obj.environment, obj.echo, obj.package_suffixes) + obj.flow, obj.environment, obj.echo, obj.package_suffixes + ) package_url, package_sha = obj.flow_datastore.save_data( - [obj.package.blob], len_hint=1)[0] - - return StepFunctions(name, - obj.graph, - obj.flow, - package_sha, - package_url, - token, - obj.metadata, - obj.flow_datastore, - obj.environment, - obj.event_logger, - obj.monitor, - tags=tags, - namespace=namespace, - max_workers=max_workers, - username=get_username(), - workflow_timeout=workflow_timeout, - is_project=is_project) - -def resolve_token(name, - token_prefix, - obj, - authorize, - given_token, - generate_new_token, - is_project): + [obj.package.blob], len_hint=1 + )[0] + + return StepFunctions( + name, + obj.graph, + obj.flow, + package_sha, + package_url, + token, + obj.metadata, + obj.flow_datastore, + obj.environment, + obj.event_logger, + obj.monitor, + tags=tags, + namespace=namespace, + max_workers=max_workers, + username=get_username(), + workflow_timeout=workflow_timeout, + is_project=is_project, + ) + + +def resolve_token( + name, token_prefix, obj, authorize, given_token, generate_new_token, is_project +): # 1) retrieve the previous deployment, if one exists workflow = StepFunctions.get_existing_deployment(name) if workflow is None: - obj.echo("It seems this is the first time you are deploying *%s* to " - "AWS Step Functions." % name) + obj.echo( + "It seems this is the first time you are deploying *%s* to " + "AWS Step Functions." % name + ) prev_token = None else: prev_user, prev_token = workflow @@ -275,155 +320,194 @@ def resolve_token(name, if prev_token is not None: if authorize is None: authorize = load_token(token_prefix) - elif authorize.startswith('production:'): + elif authorize.startswith("production:"): authorize = authorize[11:] # we allow the user who deployed the previous version to re-deploy, # even if they don't have the token if prev_user != get_username() and authorize != prev_token: - obj.echo("There is an existing version of *%s* on AWS Step " - "Functions which was deployed by the user " - "*%s*." % (name, prev_user)) - obj.echo("To deploy a new version of this flow, you need to use " - "the same production token that they used. ") - obj.echo("Please reach out to them to get the token. Once you " - "have it, call this command:") - obj.echo(" step-functions create --authorize MY_TOKEN", - fg='green') - obj.echo('See "Organizing Results" at docs.metaflow.org for more ' - "information about production tokens.") - raise IncorrectProductionToken("Try again with the correct " - "production token.") + obj.echo( + "There is an existing version of *%s* on AWS Step " + "Functions which was deployed by the user " + "*%s*." % (name, prev_user) + ) + obj.echo( + "To deploy a new version of this flow, you need to use " + "the same production token that they used. " + ) + obj.echo( + "Please reach out to them to get the token. Once you " + "have it, call this command:" + ) + obj.echo(" step-functions create --authorize MY_TOKEN", fg="green") + obj.echo( + 'See "Organizing Results" at docs.metaflow.org for more ' + "information about production tokens." + ) + raise IncorrectProductionToken( + "Try again with the correct " "production token." + ) # 3) do we need a new token or should we use the existing token? if given_token: if is_project: # we rely on a known prefix for @project tokens, so we can't # allow the user to specify a custom token with an arbitrary prefix - raise MetaflowException("--new-token is not supported for " - "@projects. Use --generate-new-token to " - "create a new token.") - if given_token.startswith('production:'): + raise MetaflowException( + "--new-token is not supported for " + "@projects. Use --generate-new-token to " + "create a new token." + ) + if given_token.startswith("production:"): given_token = given_token[11:] token = given_token - obj.echo('') + obj.echo("") obj.echo("Using the given token, *%s*." % token) elif prev_token is None or generate_new_token: token = new_token(token_prefix, prev_token) if token is None: if prev_token is None: - raise MetaflowInternalError("We could not generate a new " - "token. This is unexpected. ") + raise MetaflowInternalError( + "We could not generate a new " "token. This is unexpected. " + ) else: - raise MetaflowException("--generate-new-token option is not " - "supported after using --new-token. " - "Use --new-token to make a new " - "namespace.") - obj.echo('') + raise MetaflowException( + "--generate-new-token option is not " + "supported after using --new-token. " + "Use --new-token to make a new " + "namespace." + ) + obj.echo("") obj.echo("A new production token generated.") else: token = prev_token - obj.echo('') + obj.echo("") obj.echo("The namespace of this production flow is") - obj.echo(' production:%s' % token, fg='green') - obj.echo("To analyze results of this production flow " - "add this line in your notebooks:") - obj.echo(' namespace(\"production:%s\")' % token, fg='green') - obj.echo("If you want to authorize other people to deploy new versions " - "of this flow to AWS Step Functions, they need to call") - obj.echo(" step-functions create --authorize %s" % token, fg='green') - obj.echo("when deploying this flow to AWS Step Functions for the first " - "time.") - obj.echo('See "Organizing Results" at https://docs.metaflow.org/ for more ' - "information about production tokens.") - obj.echo('') + obj.echo(" production:%s" % token, fg="green") + obj.echo( + "To analyze results of this production flow " "add this line in your notebooks:" + ) + obj.echo(' namespace("production:%s")' % token, fg="green") + obj.echo( + "If you want to authorize other people to deploy new versions " + "of this flow to AWS Step Functions, they need to call" + ) + obj.echo(" step-functions create --authorize %s" % token, fg="green") + obj.echo("when deploying this flow to AWS Step Functions for the first " "time.") + obj.echo( + 'See "Organizing Results" at https://docs.metaflow.org/ for more ' + "information about production tokens." + ) + obj.echo("") store_token(token_prefix, token) return token @parameters.add_custom_parameters(deploy_mode=False) @step_functions.command(help="Trigger the workflow on AWS Step Functions.") -@click.option('--run-id-file', - default=None, - show_default=True, - type=str, - help="Write the ID of this run to the file specified.") +@click.option( + "--run-id-file", + default=None, + show_default=True, + type=str, + help="Write the ID of this run to the file specified.", +) @click.pass_obj def trigger(obj, run_id_file=None, **kwargs): def _convert_value(param): # Swap `-` with `_` in parameter name to match click's behavior - val = kwargs.get(param.name.replace('-', '_').lower()) - return json.dumps(val) if param.kwargs.get('type') == JSONType else \ - val() if callable(val) else val - - params = {param.name: _convert_value(param) - for _, param in obj.flow._get_parameters() - if kwargs.get(param.name.replace('-', '_').lower()) is not None} + val = kwargs.get(param.name.replace("-", "_").lower()) + return ( + json.dumps(val) + if param.kwargs.get("type") == JSONType + else val() + if callable(val) + else val + ) + + params = { + param.name: _convert_value(param) + for _, param in obj.flow._get_parameters() + if kwargs.get(param.name.replace("-", "_").lower()) is not None + } response = StepFunctions.trigger(obj.state_machine_name, params) - id = response['executionArn'].split(':')[-1] - run_id = 'sfn-' + id + id = response["executionArn"].split(":")[-1] + run_id = "sfn-" + id if run_id_file: - with open(run_id_file, 'w') as f: + with open(run_id_file, "w") as f: f.write(str(run_id)) - obj.echo("Workflow *{name}* triggered on AWS Step Functions " - "(run-id *{run_id}*)." - .format(name=obj.state_machine_name, run_id=run_id), bold=True) - - -@step_functions.command( - help="List all runs of the workflow on AWS Step Functions.") -@click.option("--running", default=False, is_flag=True, - help="List all runs of the workflow in RUNNING state on " - "AWS Step Functions.") -@click.option("--succeeded", default=False, is_flag=True, - help="List all runs of the workflow in SUCCEEDED state on " - "AWS Step Functions.") -@click.option("--failed", default=False, is_flag=True, - help="List all runs of the workflow in FAILED state on " - "AWS Step Functions.") -@click.option("--timed-out", default=False, is_flag=True, - help="List all runs of the workflow in TIMED_OUT state on " - "AWS Step Functions.") -@click.option("--aborted", default=False, is_flag=True, - help="List all runs of the workflow in ABORTED state on " - "AWS Step Functions.") + obj.echo( + "Workflow *{name}* triggered on AWS Step Functions " + "(run-id *{run_id}*).".format(name=obj.state_machine_name, run_id=run_id), + bold=True, + ) + + +@step_functions.command(help="List all runs of the workflow on AWS Step Functions.") +@click.option( + "--running", + default=False, + is_flag=True, + help="List all runs of the workflow in RUNNING state on " "AWS Step Functions.", +) +@click.option( + "--succeeded", + default=False, + is_flag=True, + help="List all runs of the workflow in SUCCEEDED state on " "AWS Step Functions.", +) +@click.option( + "--failed", + default=False, + is_flag=True, + help="List all runs of the workflow in FAILED state on " "AWS Step Functions.", +) +@click.option( + "--timed-out", + default=False, + is_flag=True, + help="List all runs of the workflow in TIMED_OUT state on " "AWS Step Functions.", +) +@click.option( + "--aborted", + default=False, + is_flag=True, + help="List all runs of the workflow in ABORTED state on " "AWS Step Functions.", +) @click.pass_obj -def list_runs(obj, - running=False, - succeeded=False, - failed=False, - timed_out=False, - aborted=False): +def list_runs( + obj, running=False, succeeded=False, failed=False, timed_out=False, aborted=False +): states = [] if running: - states.append('RUNNING') + states.append("RUNNING") if succeeded: - states.append('SUCCEEDED') + states.append("SUCCEEDED") if failed: - states.append('FAILED') + states.append("FAILED") if timed_out: - states.append('TIMED_OUT') + states.append("TIMED_OUT") if aborted: - states.append('ABORTED') + states.append("ABORTED") executions = StepFunctions.list(obj.state_machine_name, states) found = False for execution in executions: found = True - if execution.get('stopDate'): + if execution.get("stopDate"): obj.echo( "*sfn-{id}* " "startedAt:'{startDate}' " "stoppedAt:'{stopDate}' " "*{status}*".format( - id=execution['name'], - status=execution['status'], - startDate=execution['startDate'].replace(microsecond=0), - stopDate=execution['stopDate'].replace(microsecond=0), + id=execution["name"], + status=execution["status"], + startDate=execution["startDate"].replace(microsecond=0), + stopDate=execution["stopDate"].replace(microsecond=0), ) ) else: @@ -431,24 +515,28 @@ def list_runs(obj, "*sfn-{id}* " "startedAt:'{startDate}' " "*{status}*".format( - id=execution['name'], - status=execution['status'], - startDate=execution['startDate'].replace(microsecond=0) + id=execution["name"], + status=execution["status"], + startDate=execution["startDate"].replace(microsecond=0), ) ) if not found: if len(states) > 0: - status = '' + status = "" for idx, state in enumerate(states): if idx == 0: pass elif idx == len(states) - 1: - status += ' and ' + status += " and " else: - status += ', ' - status += '*%s*' % state - obj.echo('No %s executions for *%s* found on AWS Step Functions.' - % (status, obj.state_machine_name)) + status += ", " + status += "*%s*" % state + obj.echo( + "No %s executions for *%s* found on AWS Step Functions." + % (status, obj.state_machine_name) + ) else: - obj.echo('No executions for *%s* found on AWS Step Functions.' \ - % (obj.state_machine_name)) + obj.echo( + "No executions for *%s* found on AWS Step Functions." + % (obj.state_machine_name) + ) diff --git a/metaflow/plugins/aws/step_functions/step_functions_client.py b/metaflow/plugins/aws/step_functions/step_functions_client.py index 02b9492ef4a..c42bb0a8907 100644 --- a/metaflow/plugins/aws/step_functions/step_functions_client.py +++ b/metaflow/plugins/aws/step_functions/step_functions_client.py @@ -1,45 +1,49 @@ -from metaflow.metaflow_config import \ - AWS_SANDBOX_ENABLED, AWS_SANDBOX_REGION, SFN_EXECUTION_LOG_GROUP_ARN +from metaflow.metaflow_config import ( + AWS_SANDBOX_ENABLED, + AWS_SANDBOX_REGION, + SFN_EXECUTION_LOG_GROUP_ARN, +) class StepFunctionsClient(object): - def __init__(self): from ..aws_client import get_aws_client - self._client = get_aws_client('stepfunctions') + + self._client = get_aws_client("stepfunctions") def search(self, name): - paginator = self._client.get_paginator('list_state_machines') - return next(( - state_machine - for page in paginator.paginate() - for state_machine in page['stateMachines'] - if state_machine['name'] == name - ), None) + paginator = self._client.get_paginator("list_state_machines") + return next( + ( + state_machine + for page in paginator.paginate() + for state_machine in page["stateMachines"] + if state_machine["name"] == name + ), + None, + ) - def push(self, - name, - definition, - role_arn, - log_execution_history): + def push(self, name, definition, role_arn, log_execution_history): try: response = self._client.create_state_machine( - name = name, - definition = definition, - roleArn = role_arn, - loggingConfiguration = \ - self._default_logging_configuration(log_execution_history) + name=name, + definition=definition, + roleArn=role_arn, + loggingConfiguration=self._default_logging_configuration( + log_execution_history + ), ) - state_machine_arn = response['stateMachineArn'] + state_machine_arn = response["stateMachineArn"] except self._client.exceptions.StateMachineAlreadyExists as e: # State Machine already exists, update it instead of creating it. - state_machine_arn = e.response['Error']['Message'].split("'")[1] + state_machine_arn = e.response["Error"]["Message"].split("'")[1] self._client.update_state_machine( - stateMachineArn = state_machine_arn, - definition = definition, - roleArn = role_arn, - loggingConfiguration = \ - self._default_logging_configuration(log_execution_history) + stateMachineArn=state_machine_arn, + definition=definition, + roleArn=role_arn, + loggingConfiguration=self._default_logging_configuration( + log_execution_history + ), ) return state_machine_arn @@ -49,15 +53,14 @@ def get(self, name): return None try: return self._client.describe_state_machine( - stateMachineArn = state_machine_arn, + stateMachineArn=state_machine_arn, ) except self._client.exceptions.StateMachineDoesNotExist: return None def trigger(self, state_machine_arn, input): return self._client.start_execution( - stateMachineArn = state_machine_arn, - input = input + stateMachineArn=state_machine_arn, input=input ) def list_executions(self, state_machine_arn, states): @@ -65,41 +68,38 @@ def list_executions(self, state_machine_arn, states): return ( execution for state in states - for page in self._client.get_paginator('list_executions') - .paginate( - stateMachineArn=state_machine_arn, - statusFilter=state - ) - for execution in page['executions'] + for page in self._client.get_paginator("list_executions").paginate( + stateMachineArn=state_machine_arn, statusFilter=state + ) + for execution in page["executions"] ) return ( execution - for page in self._client.get_paginator('list_executions') - .paginate(stateMachineArn=state_machine_arn) - for execution in page['executions'] + for page in self._client.get_paginator("list_executions").paginate( + stateMachineArn=state_machine_arn + ) + for execution in page["executions"] ) def terminate_execution(self, state_machine_arn, execution_arn): - #TODO + # TODO pass def _default_logging_configuration(self, log_execution_history): if log_execution_history: return { - 'level': 'ALL', - 'includeExecutionData': True, - 'destinations': [ + "level": "ALL", + "includeExecutionData": True, + "destinations": [ { - 'cloudWatchLogsLogGroup': { - 'logGroupArn': SFN_EXECUTION_LOG_GROUP_ARN + "cloudWatchLogsLogGroup": { + "logGroupArn": SFN_EXECUTION_LOG_GROUP_ARN } } - ] + ], } else: - return { - 'level': 'OFF' - } + return {"level": "OFF"} def get_state_machine_arn(self, name): if AWS_SANDBOX_ENABLED: @@ -107,13 +107,13 @@ def get_state_machine_arn(self, name): # but we can construct the statemachine arn since we have # explicit access to the region. from ..aws_client import get_aws_client - account_id = get_aws_client('sts').get_caller_identity().get('Account') + + account_id = get_aws_client("sts").get_caller_identity().get("Account") region = AWS_SANDBOX_REGION # Sandboxes are in aws partition - return 'arn:aws:states:%s:%s:stateMachine:%s' \ - % (region, account_id, name) + return "arn:aws:states:%s:%s:stateMachine:%s" % (region, account_id, name) else: state_machine = self.search(name) if state_machine: - return state_machine['stateMachineArn'] - return None \ No newline at end of file + return state_machine["stateMachineArn"] + return None diff --git a/metaflow/plugins/aws/step_functions/step_functions_decorator.py b/metaflow/plugins/aws/step_functions/step_functions_decorator.py index c26a3ebca07..bffb313d381 100644 --- a/metaflow/plugins/aws/step_functions/step_functions_decorator.py +++ b/metaflow/plugins/aws/step_functions/step_functions_decorator.py @@ -7,89 +7,92 @@ from .dynamo_db_client import DynamoDbClient + class StepFunctionsInternalDecorator(StepDecorator): - name = 'step_functions_internal' + name = "step_functions_internal" - def task_pre_step(self, - step_name, - task_datastore, - metadata, - run_id, - task_id, - flow, - graph, - retry_count, - max_user_code_retries, - ubf_context, - inputs): + def task_pre_step( + self, + step_name, + task_datastore, + metadata, + run_id, + task_id, + flow, + graph, + retry_count, + max_user_code_retries, + ubf_context, + inputs, + ): meta = {} - meta['aws-step-functions-execution'] = os.environ['METAFLOW_RUN_ID'] - meta['aws-step-functions-state-machine'] =\ - os.environ['SFN_STATE_MACHINE'] - entries = [MetaDatum( - field=k, value=v, type=k, tags=["attempt_id:{0}".format(retry_count)]) - for k, v in meta.items()] + meta["aws-step-functions-execution"] = os.environ["METAFLOW_RUN_ID"] + meta["aws-step-functions-state-machine"] = os.environ["SFN_STATE_MACHINE"] + entries = [ + MetaDatum( + field=k, value=v, type=k, tags=["attempt_id:{0}".format(retry_count)] + ) + for k, v in meta.items() + ] # Register book-keeping metadata for debugging. metadata.register_metadata(run_id, step_name, task_id, entries) - def task_finished(self, - step_name, - flow, - graph, - is_task_ok, - retry_count, - max_user_code_retries): + def task_finished( + self, step_name, flow, graph, is_task_ok, retry_count, max_user_code_retries + ): if not is_task_ok: # The task finished with an exception - execution won't # continue so no need to do anything here. return # For foreaches, we need to dump the cardinality of the fanout - # into AWS DynamoDb so that AWS Step Functions can properly configure + # into AWS DynamoDb so that AWS Step Functions can properly configure # the Map job, in the absence of any better message passing feature # between the states. - if graph[step_name].type == 'foreach': + if graph[step_name].type == "foreach": # Since we can't generate the full path spec within AWS Step # Function DynamoDb Get task, we will just key by task id for now. # Also, we can afford to set the ttl only once for the key in AWS # DynamoDB here. AWS Step Functions can execute for up to a year # and execution history is available for 90 days after the # execution. - self._save_foreach_cardinality(os.environ['AWS_BATCH_JOB_ID'], - flow._foreach_num_splits, self._ttl()) + self._save_foreach_cardinality( + os.environ["AWS_BATCH_JOB_ID"], flow._foreach_num_splits, self._ttl() + ) # The parent task ids need to be available in a foreach join so that - # we can construct the input path. Unfortunately, while AWS Step + # we can construct the input path. Unfortunately, while AWS Step # Function provides access to an array of parent task ids, we can't # make use of them since AWS Batch job spec only accepts strings. We # instead write the task ids from the parent task to DynamoDb and read # it back in the foreach join - elif graph[step_name].is_inside_foreach and \ - any(graph[n].type == 'join' and \ - graph[graph[n].split_parents[-1]].type == 'foreach' - for n in graph[step_name].out_funcs): + elif graph[step_name].is_inside_foreach and any( + graph[n].type == "join" + and graph[graph[n].split_parents[-1]].type == "foreach" + for n in graph[step_name].out_funcs + ): self._save_parent_task_id_for_foreach_join( - os.environ['METAFLOW_SPLIT_PARENT_TASK_ID_FOR_FOREACH_JOIN'], - os.environ['AWS_BATCH_JOB_ID']) + os.environ["METAFLOW_SPLIT_PARENT_TASK_ID_FOR_FOREACH_JOIN"], + os.environ["AWS_BATCH_JOB_ID"], + ) - def _save_foreach_cardinality(self, - foreach_split_task_id, - for_each_cardinality, - ttl): - DynamoDbClient().save_foreach_cardinality(foreach_split_task_id, - for_each_cardinality, - ttl) + def _save_foreach_cardinality( + self, foreach_split_task_id, for_each_cardinality, ttl + ): + DynamoDbClient().save_foreach_cardinality( + foreach_split_task_id, for_each_cardinality, ttl + ) - def _save_parent_task_id_for_foreach_join(self, - foreach_split_task_id, - foreach_join_parent_task_id): + def _save_parent_task_id_for_foreach_join( + self, foreach_split_task_id, foreach_join_parent_task_id + ): DynamoDbClient().save_parent_task_id_for_foreach_join( - foreach_split_task_id, - foreach_join_parent_task_id) + foreach_split_task_id, foreach_join_parent_task_id + ) def _ttl(self): # Default is 1 year. delta = 366 * 24 * 60 * 60 - delta = int(os.environ.get('METAFLOW_SFN_WORKFLOW_TIMEOUT', delta)) + delta = int(os.environ.get("METAFLOW_SFN_WORKFLOW_TIMEOUT", delta)) # Add 90 days since AWS Step Functions maintains execution history for # that long. return delta + (90 * 24 * 60 * 60) + int(time.time()) diff --git a/metaflow/plugins/catch_decorator.py b/metaflow/plugins/catch_decorator.py index 4950f3876a9..7597fabc28e 100644 --- a/metaflow/plugins/catch_decorator.py +++ b/metaflow/plugins/catch_decorator.py @@ -1,7 +1,6 @@ import traceback -from metaflow.exception import MetaflowException,\ - MetaflowExceptionWrapper +from metaflow.exception import MetaflowException, MetaflowExceptionWrapper from metaflow.decorators import StepDecorator from metaflow.unbounded_foreach import UBF_CONTROL @@ -9,12 +8,14 @@ class FailureHandledByCatch(MetaflowException): - headline = 'Task execution failed but @catch handled it' + headline = "Task execution failed but @catch handled it" def __init__(self, retry_count): - msg = 'Task execution kept failing over %d attempts. '\ - 'Your code did not raise an exception. Something '\ - 'in the execution environment caused the failure.' % retry_count + msg = ( + "Task execution kept failing over %d attempts. " + "Your code did not raise an exception. Something " + "in the execution environment caused the failure." % retry_count + ) super(FailureHandledByCatch, self).__init__(msg) @@ -45,43 +46,40 @@ def myStep(self): Determines whether or not the exception is printed to stdout when caught. Defaults to True """ - name = 'catch' - defaults = {'var': None, - 'print_exception': True} + + name = "catch" + defaults = {"var": None, "print_exception": True} def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger): # handling _foreach_var and _foreach_num_splits requires some # deeper thinking, so let's not support that use case for now self.logger = logger - if graph[step].type == 'foreach': - raise MetaflowException('@catch is defined for the step *%s* ' - 'but @catch is not supported in foreach ' - 'split steps.' % step) + if graph[step].type == "foreach": + raise MetaflowException( + "@catch is defined for the step *%s* " + "but @catch is not supported in foreach " + "split steps." % step + ) def _print_exception(self, step, flow): - self.logger(head='@catch caught an exception from %s' % flow, - timestamp=False) + self.logger(head="@catch caught an exception from %s" % flow, timestamp=False) for line in traceback.format_exc().splitlines(): - self.logger('> %s' % line, timestamp=False) + self.logger("> %s" % line, timestamp=False) def _set_var(self, flow, val): - var = self.attributes.get('var') + var = self.attributes.get("var") if var: setattr(flow, var, val) - def task_exception(self, - exception, - step, - flow, - graph, - retry_count, - max_user_code_retries): + def task_exception( + self, exception, step, flow, graph, retry_count, max_user_code_retries + ): # Only "catch" exceptions after all retries are exhausted if retry_count < max_user_code_retries: return False - if self.attributes['print_exception']: + if self.attributes["print_exception"]: self._print_exception(step, flow) # pretend that self.next() was called as usual @@ -92,25 +90,18 @@ def task_exception(self, self._set_var(flow, picklable) return True - def task_post_step(self, - step_name, - flow, - graph, - retry_count, - max_user_code_retries): + def task_post_step( + self, step_name, flow, graph, retry_count, max_user_code_retries + ): # there was no exception, set the exception var (if any) to None self._set_var(flow, None) def step_task_retry_count(self): return 0, NUM_FALLBACK_RETRIES - def task_decorate(self, - step_func, - func, - graph, - retry_count, - max_user_code_retries, - ubf_context): + def task_decorate( + self, step_func, func, graph, retry_count, max_user_code_retries, ubf_context + ): # if the user code has failed max_user_code_retries times, @catch # runs a piece of fallback code instead. This way we can continue diff --git a/metaflow/plugins/conda/__init__.py b/metaflow/plugins/conda/__init__.py index 2535ea25ad9..ec0321b4e8e 100644 --- a/metaflow/plugins/conda/__init__.py +++ b/metaflow/plugins/conda/__init__.py @@ -3,7 +3,7 @@ import json import fcntl -CONDA_MAGIC_FILE = 'conda.dependencies' +CONDA_MAGIC_FILE = "conda.dependencies" def get_conda_manifest_path(ds_root, flow_name): @@ -26,7 +26,7 @@ def write_to_conda_manifest(ds_root, flow_name, key, value): except OSError as x: if x.errno != errno.EEXIST: raise - with os.fdopen(os.open(path, os.O_RDWR | os.O_CREAT), 'r+') as f: + with os.fdopen(os.open(path, os.O_RDWR | os.O_CREAT), "r+") as f: try: fcntl.flock(f, fcntl.LOCK_EX) data = {} @@ -41,4 +41,4 @@ def write_to_conda_manifest(ds_root, flow_name, key, value): if e.errno != errno.EAGAIN: raise finally: - fcntl.flock(f, fcntl.LOCK_UN) \ No newline at end of file + fcntl.flock(f, fcntl.LOCK_UN) diff --git a/metaflow/plugins/conda/batch_bootstrap.py b/metaflow/plugins/conda/batch_bootstrap.py index d62b5d3605b..12484ba5f6b 100644 --- a/metaflow/plugins/conda/batch_bootstrap.py +++ b/metaflow/plugins/conda/batch_bootstrap.py @@ -20,34 +20,45 @@ def bootstrap_environment(flow_name, env_id): packages = download_conda_packages(flow_name, env_id) install_conda_environment(env_id, packages) + def setup_conda_manifest(flow_name): manifest_folder = os.path.join(os.getcwd(), DATASTORE_LOCAL_DIR, flow_name) if not os.path.exists(manifest_folder): os.makedirs(manifest_folder) - shutil.move(os.path.join(os.getcwd(), CONDA_MAGIC_FILE), - os.path.join(manifest_folder, CONDA_MAGIC_FILE)) + shutil.move( + os.path.join(os.getcwd(), CONDA_MAGIC_FILE), + os.path.join(manifest_folder, CONDA_MAGIC_FILE), + ) + def download_conda_packages(flow_name, env_id): - pkgs_folder = os.path.join(os.getcwd(), 'pkgs') + pkgs_folder = os.path.join(os.getcwd(), "pkgs") if not os.path.exists(pkgs_folder): os.makedirs(pkgs_folder) manifest_folder = os.path.join(os.getcwd(), DATASTORE_LOCAL_DIR, flow_name) with open(os.path.join(manifest_folder, CONDA_MAGIC_FILE)) as f: env = json.load(f)[env_id] with S3() as s3: - for pkg in s3.get_many(env['cache_urls']): - shutil.move(pkg.path, os.path.join(pkgs_folder, os.path.basename(pkg.key))) - return env['order'] + for pkg in s3.get_many(env["cache_urls"]): + shutil.move( + pkg.path, os.path.join(pkgs_folder, os.path.basename(pkg.key)) + ) + return env["order"] + def install_conda_environment(env_id, packages): args = [ - 'if ! type conda >/dev/null 2>&1; \ + "if ! type conda >/dev/null 2>&1; \ then wget --no-check-certificate https://repo.anaconda.com/pkgs/misc/conda-execs/conda-latest-linux-64.exe -O conda >/dev/null 2>&1; \ chmod +x conda; \ - export PATH=$PATH:{0}; fi'.format(os.getcwd()), - 'cd {0}'.format(os.path.join(os.getcwd(), 'pkgs')), - 'conda create --yes --no-default-packages -p {0} --no-deps {1} >/dev/null 2>&1'.format(os.path.join(os.getcwd(), env_id), ' '.join(packages)), - 'cd {0}'.format(os.getcwd()) + export PATH=$PATH:{0}; fi".format( + os.getcwd() + ), + "cd {0}".format(os.path.join(os.getcwd(), "pkgs")), + "conda create --yes --no-default-packages -p {0} --no-deps {1} >/dev/null 2>&1".format( + os.path.join(os.getcwd(), env_id), " ".join(packages) + ), + "cd {0}".format(os.getcwd()), ] if ENV_ESCAPE_PY is not None: cwd = os.getcwd() @@ -56,7 +67,8 @@ def install_conda_environment(env_id, packages): else: pass # print("Could not find a environment escape interpreter") - os.system(' && '.join(args)) + os.system(" && ".join(args)) + -if __name__ == '__main__': - bootstrap_environment(sys.argv[1], sys.argv[2]) \ No newline at end of file +if __name__ == "__main__": + bootstrap_environment(sys.argv[1], sys.argv[2]) diff --git a/metaflow/plugins/conda/conda.py b/metaflow/plugins/conda/conda.py index 3734b640aa3..153fe7ad28d 100644 --- a/metaflow/plugins/conda/conda.py +++ b/metaflow/plugins/conda/conda.py @@ -9,54 +9,60 @@ from metaflow.metaflow_environment import InvalidEnvironmentException from metaflow.util import which + class CondaException(MetaflowException): - headline = 'Conda ran into an error while setting up environment.' + headline = "Conda ran into an error while setting up environment." def __init__(self, error): if isinstance(error, (list,)): - error = '\n'.join(error) - msg = '{error}'.format(error=error) + error = "\n".join(error) + msg = "{error}".format(error=error) super(CondaException, self).__init__(msg) -class CondaStepException(CondaException): +class CondaStepException(CondaException): def __init__(self, exception, step): - msg = 'Step: {step}, Error: {error}'.format(step=step, error=exception.message) + msg = "Step: {step}, Error: {error}".format(step=step, error=exception.message) super(CondaStepException, self).__init__(msg) -class Conda(object): +class Conda(object): def __init__(self): - self._bin = which('conda') + self._bin = which("conda") if self._bin is None: - raise InvalidEnvironmentException('No conda installation found. ' - 'Install conda first.') - if LooseVersion(self._info()['conda_version']) < LooseVersion('4.6.0'): - raise InvalidEnvironmentException('Conda version 4.6.0 or newer ' - 'is required. Visit ' - 'https://docs.conda.io/en/latest/miniconda.html ' - 'for installation instructions.') - if 'conda-forge' not in self.config()['channels']: - raise InvalidEnvironmentException('Conda channel \'conda-forge\' ' - 'is required. Specify it with CONDA_CHANNELS ' - 'environment variable.') - - def create(self, - step_name, - env_id, - deps, - architecture=None, - explicit=False, - disable_safety_checks=False): + raise InvalidEnvironmentException( + "No conda installation found. " "Install conda first." + ) + if LooseVersion(self._info()["conda_version"]) < LooseVersion("4.6.0"): + raise InvalidEnvironmentException( + "Conda version 4.6.0 or newer " + "is required. Visit " + "https://docs.conda.io/en/latest/miniconda.html " + "for installation instructions." + ) + if "conda-forge" not in self.config()["channels"]: + raise InvalidEnvironmentException( + "Conda channel 'conda-forge' " + "is required. Specify it with CONDA_CHANNELS " + "environment variable." + ) + + def create( + self, + step_name, + env_id, + deps, + architecture=None, + explicit=False, + disable_safety_checks=False, + ): # Create the conda environment try: with CondaLock(self._env_lock_file(env_id)): self._remove(env_id) - self._create(env_id, - deps, - explicit, - architecture, - disable_safety_checks) + self._create( + env_id, deps, explicit, architecture, disable_safety_checks + ) return self._deps(env_id) except CondaException as e: raise CondaStepException(e, step_name) @@ -71,71 +77,71 @@ def remove(self, step_name, env_id): def python(self, env_id): # Get Python interpreter for the conda environment - return os.path.join(self._env_path(env_id), 'bin/python') + return os.path.join(self._env_path(env_id), "bin/python") def environments(self, flow): # List all conda environments associated with the flow - envs = self._info()['envs'] + envs = self._info()["envs"] ret = {} for env in envs: - if '/envs/' in env: + if "/envs/" in env: name = os.path.basename(env) - if name.startswith('metaflow_%s' % flow): + if name.startswith("metaflow_%s" % flow): ret[name] = env return ret def config(self): # Show conda installation configuration - return json.loads(self._call_conda(['config', '--show'])) + return json.loads(self._call_conda(["config", "--show"])) def package_info(self, env_id): # Show conda environment package configuration # Not every parameter is exposed via conda cli hence this ignominy - metadata = os.path.join(self._env_path(env_id), 'conda-meta') + metadata = os.path.join(self._env_path(env_id), "conda-meta") for path, dirs, files in os.walk(metadata): for file in files: - if file.endswith('.json'): + if file.endswith(".json"): with open(os.path.join(path, file)) as f: yield json.loads(f.read()) def _info(self): - return json.loads(self._call_conda(['info'])) - - def _create(self, - env_id, - deps, - explicit=False, - architecture=None, - disable_safety_checks=False): - cmd = ['create', '--yes', '--no-default-packages', - '--name', env_id, '--quiet'] + return json.loads(self._call_conda(["info"])) + + def _create( + self, + env_id, + deps, + explicit=False, + architecture=None, + disable_safety_checks=False, + ): + cmd = ["create", "--yes", "--no-default-packages", "--name", env_id, "--quiet"] if explicit: - cmd.append('--no-deps') + cmd.append("--no-deps") cmd.extend(deps) - self._call_conda(cmd, - architecture=architecture, - disable_safety_checks=disable_safety_checks) + self._call_conda( + cmd, architecture=architecture, disable_safety_checks=disable_safety_checks + ) def _remove(self, env_id): - self._call_conda(['env', 'remove', '--name', - env_id, '--yes', '--quiet']) + self._call_conda(["env", "remove", "--name", env_id, "--yes", "--quiet"]) def _install(self, env_id, deps, explicit=False): - cmd = ['install', '--yes', '--name', env_id, '--quiet'] + cmd = ["install", "--yes", "--name", env_id, "--quiet"] if explicit: - cmd.append('--no-deps') + cmd.append("--no-deps") cmd.extend(deps) self._call_conda(cmd) def _install_order(self, env_id): - cmd = ['list', '--name', env_id, '--explicit'] - response = self._call_conda(cmd).decode('utf-8') + cmd = ["list", "--name", env_id, "--explicit"] + response = self._call_conda(cmd).decode("utf-8") emit = False result = [] for line in response.splitlines(): if emit: - result.append(line.split('/')[-1]) - if not emit and line == '@EXPLICIT': + result.append(line.split("/")[-1]) + if not emit and line == "@EXPLICIT": emit = True return result @@ -143,52 +149,54 @@ def _deps(self, env_id): exact_deps = [] urls = [] for package in self.package_info(env_id): - exact_deps.append('%s=%s=%s' % (package['name'], package['version'], package['build'])) - urls.append(package['url']) + exact_deps.append( + "%s=%s=%s" % (package["name"], package["version"], package["build"]) + ) + urls.append(package["url"]) order = self._install_order(env_id) return (exact_deps, urls, order) def _env_path(self, env_id): - envs = self._info()['envs'] + envs = self._info()["envs"] for env in envs: - if '/envs/' in env: + if "/envs/" in env: name = os.path.basename(env) if name == env_id: return env return None def _env_lock_file(self, env_id): - return os.path.join(self._info()['envs_dirs'][0], 'mf_env-creation.lock') + return os.path.join(self._info()["envs_dirs"][0], "mf_env-creation.lock") def _call_conda(self, args, architecture=None, disable_safety_checks=False): try: env = { - 'CONDA_JSON': 'True', - 'CONDA_SUBDIR': (architecture if architecture else ''), - 'CONDA_USE_ONLY_TAR_BZ2': 'True' + "CONDA_JSON": "True", + "CONDA_SUBDIR": (architecture if architecture else ""), + "CONDA_USE_ONLY_TAR_BZ2": "True", } if disable_safety_checks: - env['CONDA_SAFETY_CHECKS'] = 'disabled' + env["CONDA_SAFETY_CHECKS"] = "disabled" return subprocess.check_output( - [self._bin] + args, - stderr = subprocess.PIPE, - env = dict(os.environ, **env)).strip() + [self._bin] + args, stderr=subprocess.PIPE, env=dict(os.environ, **env) + ).strip() except subprocess.CalledProcessError as e: try: output = json.loads(e.output) - err = [output['error']] - for error in output.get('errors', []): - err.append(error['error']) + err = [output["error"]] + for error in output.get("errors", []): + err.append(error["error"]) raise CondaException(err) except (TypeError, ValueError) as ve: pass raise CondaException( - 'command \'{cmd}\' returned error ({code}): {output}, stderr={stderr}' - .format(cmd=e.cmd, code=e.returncode, output=e.output, stderr=e.stderr)) + "command '{cmd}' returned error ({code}): {output}, stderr={stderr}".format( + cmd=e.cmd, code=e.returncode, output=e.output, stderr=e.stderr + ) + ) class CondaLock(object): - def __init__(self, lock, timeout=3600, delay=10): self.lock = lock self.locked = False @@ -204,19 +212,18 @@ def _acquire(self): raise while True: try: - self.fd = os.open(self.lock, os.O_CREAT | - os.O_EXCL | os.O_RDWR) + self.fd = os.open(self.lock, os.O_CREAT | os.O_EXCL | os.O_RDWR) self.locked = True break except OSError as e: if e.errno != errno.EEXIST: raise if self.timeout is None: - raise CondaException( - 'Could not acquire lock {}'.format(self.lock)) + raise CondaException("Could not acquire lock {}".format(self.lock)) if (time.time() - start) >= self.timeout: raise CondaException( - 'Timeout occurred while acquiring lock {}'.format(self.lock)) + "Timeout occurred while acquiring lock {}".format(self.lock) + ) time.sleep(self.delay) def _release(self): diff --git a/metaflow/plugins/conda/conda_environment.py b/metaflow/plugins/conda/conda_environment.py index 37f0f47661c..7ca49e16385 100644 --- a/metaflow/plugins/conda/conda_environment.py +++ b/metaflow/plugins/conda/conda_environment.py @@ -14,7 +14,7 @@ class CondaEnvironment(MetaflowEnvironment): - TYPE = 'conda' + TYPE = "conda" _filecache = None def __init__(self, flow): @@ -31,13 +31,15 @@ def __init__(self, flow): # the default 'default environment' self.base_env = MetaflowEnvironment(self.flow) else: - self.base_env = [e for e in ENVIRONMENTS + [MetaflowEnvironment] - if e.TYPE == DEFAULT_ENVIRONMENT][0](self.flow) + self.base_env = [ + e + for e in ENVIRONMENTS + [MetaflowEnvironment] + if e.TYPE == DEFAULT_ENVIRONMENT + ][0](self.flow) def init_environment(self, echo): # Print a message for now - echo("Bootstrapping conda environment..." + - "(this could take a few minutes)") + echo("Bootstrapping conda environment..." + "(this could take a few minutes)") self.base_env.init_environment(echo) def validate_environment(self, echo): @@ -45,11 +47,11 @@ def validate_environment(self, echo): def decospecs(self): # Apply conda decorator and base environment's decorators to all steps - return ('conda',) + self.base_env.decospecs() + return ("conda",) + self.base_env.decospecs() def _get_conda_decorator(self, step_name): step = next(step for step in self.flow if step.name == step_name) - decorator = next(deco for deco in step.decorators if deco.name == 'conda') + decorator = next(deco for deco in step.decorators if deco.name == "conda") # Guaranteed to have a conda decorator because of self.decospecs() return decorator @@ -62,7 +64,7 @@ def _get_env_id(self, step_name): def _get_executable(self, step_name): env_id = self._get_env_id(step_name) if env_id is not None: - return (os.path.join(env_id, "bin/python -s")) + return os.path.join(env_id, "bin/python -s") return None def set_local_root(self, ds_root): @@ -73,11 +75,11 @@ def bootstrap_commands(self, step_name): env_id = self._get_env_id(step_name) if env_id is not None: return [ - "echo \'Bootstrapping environment...\'", - "python -m metaflow.plugins.conda.batch_bootstrap \"%s\" %s" % \ - (self.flow.name, env_id), - "echo \'Environment bootstrapped.\'", - ] + "echo 'Bootstrapping environment...'", + 'python -m metaflow.plugins.conda.batch_bootstrap "%s" %s' + % (self.flow.name, env_id), + "echo 'Environment bootstrapped.'", + ] return [] def add_to_package(self): @@ -91,7 +93,7 @@ def add_to_package(self): def pylint_config(self): config = self.base_env.pylint_config() # Disable (import-error) in pylint - config.append('--disable=F0401') + config.append("--disable=F0401") return config def executable(self, step_name): @@ -105,22 +107,26 @@ def executable(self, step_name): def get_client_info(cls, flow_name, metadata): if cls._filecache is None: from metaflow.client.filecache import FileCache + cls._filecache = FileCache() - info = metadata.get('code-package') - env_id = metadata.get('conda_env_id') + info = metadata.get("code-package") + env_id = metadata.get("conda_env_id") if info is None or env_id is None: - return {'type': 'conda'} + return {"type": "conda"} info = json.loads(info) - _, blobdata = cls._filecache.get_data(info['ds_type'], flow_name, info['location'], info['sha']) - with tarfile.open(fileobj=BytesIO(blobdata), mode='r:gz') as tar: + _, blobdata = cls._filecache.get_data( + info["ds_type"], flow_name, info["location"], info["sha"] + ) + with tarfile.open(fileobj=BytesIO(blobdata), mode="r:gz") as tar: conda_file = tar.extractfile(CONDA_MAGIC_FILE) if conda_file is None: - return {'type': 'conda'} - info = json.loads(conda_file.read().decode('utf-8')) + return {"type": "conda"} + info = json.loads(conda_file.read().decode("utf-8")) new_info = { - 'type': 'conda', - 'explicit': info[env_id]['explicit'], - 'deps': info[env_id]['deps']} + "type": "conda", + "explicit": info[env_id]["explicit"], + "deps": info[env_id]["deps"], + } return new_info def get_package_commands(self, code_package_url): diff --git a/metaflow/plugins/conda/conda_flow_decorator.py b/metaflow/plugins/conda/conda_flow_decorator.py index 18b0d07cae9..75340c9718d 100644 --- a/metaflow/plugins/conda/conda_flow_decorator.py +++ b/metaflow/plugins/conda/conda_flow_decorator.py @@ -32,20 +32,14 @@ class MyFlow(FlowSpec): InvalidEnvironmentException Raised if --environment=conda is not specified """ - name = 'conda_base' - defaults = {'libraries': {}, - 'python': None, - 'disabled': None} - def flow_init(self, - flow, - graph, - environment, - flow_datastore, - metadata, - logger, - echo, - options): - if environment.TYPE != 'conda': - raise InvalidEnvironmentException('The *@conda* decorator requires ' - '--environment=conda') \ No newline at end of file + name = "conda_base" + defaults = {"libraries": {}, "python": None, "disabled": None} + + def flow_init( + self, flow, graph, environment, flow_datastore, metadata, logger, echo, options + ): + if environment.TYPE != "conda": + raise InvalidEnvironmentException( + "The *@conda* decorator requires " "--environment=conda" + ) diff --git a/metaflow/plugins/conda/conda_step_decorator.py b/metaflow/plugins/conda/conda_step_decorator.py index 10a9b9894f2..fd5b71b59c3 100644 --- a/metaflow/plugins/conda/conda_step_decorator.py +++ b/metaflow/plugins/conda/conda_step_decorator.py @@ -6,6 +6,7 @@ import requests import shutil import tempfile + try: from urlparse import urlparse except: @@ -31,6 +32,7 @@ unicode = str basestring = str + class CondaStepDecorator(StepDecorator): """ Conda decorator that sets the Conda environment for your step @@ -55,118 +57,142 @@ def MyStep(self): disabled : bool If set to True, disables Conda. Defaults to False """ - name = 'conda' - defaults = {'libraries': {}, - 'python': None, - 'disabled': None} + + name = "conda" + defaults = {"libraries": {}, "python": None, "disabled": None} conda = None environments = None def _get_base_attributes(self): - if 'conda_base' in self.flow._flow_decorators: - return self.flow._flow_decorators['conda_base'].attributes + if "conda_base" in self.flow._flow_decorators: + return self.flow._flow_decorators["conda_base"].attributes return self.defaults def _python_version(self): - return next(x for x in [ - self.attributes['python'], - self.base_attributes['python'], - platform.python_version()] if x is not None) + return next( + x + for x in [ + self.attributes["python"], + self.base_attributes["python"], + platform.python_version(), + ] + if x is not None + ) def is_enabled(self, ubf_context=None): if ubf_context == UBF_CONTROL: # Disable `@conda` for ubf_control tasks. return False - return not next(x for x in [ - self.attributes['disabled'], - self.base_attributes['disabled'], - False] if x is not None) + return not next( + x + for x in [ + self.attributes["disabled"], + self.base_attributes["disabled"], + False, + ] + if x is not None + ) def _lib_deps(self): deps = get_pinned_conda_libs(self._python_version()) - base_deps = self.base_attributes['libraries'] + base_deps = self.base_attributes["libraries"] deps.update(base_deps) - step_deps = self.attributes['libraries'] + step_deps = self.attributes["libraries"] if isinstance(step_deps, (unicode, basestring)): - step_deps = step_deps.strip('"{}\'') + step_deps = step_deps.strip("\"{}'") if step_deps: - step_deps = dict(map(lambda x: x.strip().strip('"\''), - a.split(':')) for a in step_deps.split(',')) + step_deps = dict( + map(lambda x: x.strip().strip("\"'"), a.split(":")) + for a in step_deps.split(",") + ) deps.update(step_deps) return deps def _step_deps(self): - deps = [b'python==%s' % self._python_version().encode()] - deps.extend(b'%s==%s' % (name.encode('ascii'), ver.encode('ascii')) - for name, ver in self._lib_deps().items()) + deps = [b"python==%s" % self._python_version().encode()] + deps.extend( + b"%s==%s" % (name.encode("ascii"), ver.encode("ascii")) + for name, ver in self._lib_deps().items() + ) return deps def _env_id(self): deps = self._step_deps() - return 'metaflow_%s_%s_%s' % (self.flow.name, - self.architecture, - sha1(b' '.join(sorted(deps))).hexdigest()) + return "metaflow_%s_%s_%s" % ( + self.flow.name, + self.architecture, + sha1(b" ".join(sorted(deps))).hexdigest(), + ) def _resolve_step_environment(self, ds_root, force=False): env_id = self._env_id() cached_deps = read_conda_manifest(ds_root, self.flow.name) if CondaStepDecorator.conda is None: CondaStepDecorator.conda = Conda() - CondaStepDecorator.environments =\ - CondaStepDecorator.conda.environments(self.flow.name) - if force or env_id not in cached_deps or 'cache_urls' not in cached_deps[env_id]: + CondaStepDecorator.environments = CondaStepDecorator.conda.environments( + self.flow.name + ) + if ( + force + or env_id not in cached_deps + or "cache_urls" not in cached_deps[env_id] + ): if force or env_id not in cached_deps: deps = self._step_deps() - (exact_deps, urls, order) = \ - self.conda.create(self.step, - env_id, - deps, - architecture=self.architecture, - disable_safety_checks=self.disable_safety_checks) + (exact_deps, urls, order) = self.conda.create( + self.step, + env_id, + deps, + architecture=self.architecture, + disable_safety_checks=self.disable_safety_checks, + ) payload = { - 'explicit': exact_deps, - 'deps': [d.decode('ascii') for d in deps], - 'urls': urls, - 'order': order + "explicit": exact_deps, + "deps": [d.decode("ascii") for d in deps], + "urls": urls, + "order": order, } else: payload = cached_deps[env_id] - if self.flow_datastore.TYPE == 's3' and 'cache_urls' not in payload: - payload['cache_urls'] = self._cache_env() + if self.flow_datastore.TYPE == "s3" and "cache_urls" not in payload: + payload["cache_urls"] = self._cache_env() write_to_conda_manifest(ds_root, self.flow.name, env_id, payload) - CondaStepDecorator.environments =\ - CondaStepDecorator.conda.environments(self.flow.name) + CondaStepDecorator.environments = CondaStepDecorator.conda.environments( + self.flow.name + ) return env_id def _cache_env(self): def _download(entry): url, local_path = entry with requests.get(url, stream=True) as r: - with open(local_path, 'wb') as f: + with open(local_path, "wb") as f: shutil.copyfileobj(r.raw, f) env_id = self._env_id() files = [] to_download = [] for package_info in self.conda.package_info(env_id): - url = urlparse(package_info['url']) - path = os.path.join(CONDA_PACKAGE_S3ROOT, - url.netloc, - url.path.lstrip('/'), - package_info['md5'], - package_info['fn']) - tarball_path = package_info['package_tarball_full_path'] - if tarball_path.endswith('.conda'): + url = urlparse(package_info["url"]) + path = os.path.join( + CONDA_PACKAGE_S3ROOT, + url.netloc, + url.path.lstrip("/"), + package_info["md5"], + package_info["fn"], + ) + tarball_path = package_info["package_tarball_full_path"] + if tarball_path.endswith(".conda"): # Conda doesn't set the metadata correctly for certain fields # when the underlying OS is spoofed. tarball_path = tarball_path[:-6] - if not tarball_path.endswith('.tar.bz2'): - tarball_path = '%s.tar.bz2' % tarball_path + if not tarball_path.endswith(".tar.bz2"): + tarball_path = "%s.tar.bz2" % tarball_path if not os.path.isfile(tarball_path): # The tarball maybe missing when user invokes `conda clean`! - to_download.append((package_info['url'], tarball_path)) + to_download.append((package_info["url"], tarball_path)) files.append((path, tarball_path)) if to_download: Pool(8).map(_download, to_download) @@ -178,47 +204,52 @@ def _prepare_step_environment(self, step_name, ds_root): env_id = self._resolve_step_environment(ds_root) if env_id not in CondaStepDecorator.environments: cached_deps = read_conda_manifest(ds_root, self.flow.name) - self.conda.create(self.step, - env_id, - cached_deps[env_id]['urls'], - architecture=self.architecture, - explicit=True, - disable_safety_checks=self.disable_safety_checks) - CondaStepDecorator.environments =\ - CondaStepDecorator.conda.environments(self.flow.name) + self.conda.create( + self.step, + env_id, + cached_deps[env_id]["urls"], + architecture=self.architecture, + explicit=True, + disable_safety_checks=self.disable_safety_checks, + ) + CondaStepDecorator.environments = CondaStepDecorator.conda.environments( + self.flow.name + ) return env_id def _disable_safety_checks(self, decos): # Disable conda safety checks when creating linux-64 environments on - # a macOS. This is needed because of gotchas around inconsistently + # a macOS. This is needed because of gotchas around inconsistently # case-(in)sensitive filesystems for macOS and linux. for deco in decos: - if deco.name in ('batch', 'kubernetes') and platform.system() == 'Darwin': + if deco.name in ("batch", "kubernetes") and platform.system() == "Darwin": return True return False def _architecture(self, decos): for deco in decos: - if deco.name in ('batch', 'kubernetes'): + if deco.name in ("batch", "kubernetes"): # force conda resolution for linux-64 architectures - return 'linux-64' - bit = '32' - if platform.machine().endswith('64'): - bit = '64' - if platform.system() == 'Linux': - return 'linux-%s' % bit - elif platform.system() == 'Darwin': - return 'osx-%s' % bit + return "linux-64" + bit = "32" + if platform.machine().endswith("64"): + bit = "64" + if platform.system() == "Linux": + return "linux-%s" % bit + elif platform.system() == "Darwin": + return "osx-%s" % bit else: - raise InvalidEnvironmentException('The *@conda* decorator is not supported ' - 'outside of Linux and Darwin platforms') + raise InvalidEnvironmentException( + "The *@conda* decorator is not supported " + "outside of Linux and Darwin platforms" + ) def runtime_init(self, flow, graph, package, run_id): # Create a symlink to installed version of metaflow to execute user code against - path_to_metaflow = os.path.join(get_metaflow_root(), 'metaflow') - self.metaflow_home = tempfile.mkdtemp(dir='/tmp') + path_to_metaflow = os.path.join(get_metaflow_root(), "metaflow") + self.metaflow_home = tempfile.mkdtemp(dir="/tmp") self.addl_paths = None - os.symlink(path_to_metaflow, os.path.join(self.metaflow_home, 'metaflow')) + os.symlink(path_to_metaflow, os.path.join(self.metaflow_home, "metaflow")) # Do the same for metaflow_extensions try: import metaflow_extensions as m @@ -230,7 +261,10 @@ def runtime_init(self, flow, graph, package, run_id): custom_paths = list(m.__path__) if len(custom_paths) == 1: # Regular package - os.symlink(custom_paths[0], os.path.join(self.metaflow_home, 'metaflow_extensions')) + os.symlink( + custom_paths[0], + os.path.join(self.metaflow_home, "metaflow_extensions"), + ) else: # Namespace package; we don't symlink but add the additional paths # for the conda interpreter @@ -240,13 +274,15 @@ def runtime_init(self, flow, graph, package, run_id): # the escape to work even in non metaflow-created subprocesses generate_trampolines(self.metaflow_home) - def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger): - if environment.TYPE != 'conda': - raise InvalidEnvironmentException('The *@conda* decorator requires ' - '--environment=conda') + if environment.TYPE != "conda": + raise InvalidEnvironmentException( + "The *@conda* decorator requires " "--environment=conda" + ) + def _logger(line, **kwargs): logger(line) + self.local_root = LocalStorage.get_datastore_root_from_config(_logger) environment.set_local_root(self.local_root) self.architecture = self._architecture(decos) @@ -255,67 +291,68 @@ def _logger(line, **kwargs): self.flow = flow self.flow_datastore = flow_datastore self.base_attributes = self._get_base_attributes() - os.environ['PYTHONNOUSERSITE'] = '1' + os.environ["PYTHONNOUSERSITE"] = "1" def package_init(self, flow, step, environment): if self.is_enabled(): self._prepare_step_environment(step, self.local_root) - def runtime_task_created(self, - task_datastore, - task_id, - split_index, - input_paths, - is_cloned, - ubf_context): + def runtime_task_created( + self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context + ): if self.is_enabled(ubf_context): - self.env_id = \ - self._prepare_step_environment(self.step, self.local_root) - - def task_pre_step(self, - step_name, - task_datastore, - meta, - run_id, - task_id, - flow, - graph, - retry_count, - max_retries, - ubf_context, - inputs): + self.env_id = self._prepare_step_environment(self.step, self.local_root) + + def task_pre_step( + self, + step_name, + task_datastore, + meta, + run_id, + task_id, + flow, + graph, + retry_count, + max_retries, + ubf_context, + inputs, + ): if self.is_enabled(ubf_context): - # Add the Python interpreter's parent to the path. This is to + # Add the Python interpreter's parent to the path. This is to # ensure that any non-pythonic dependencies introduced by the conda # environment are visible to the user code. env_path = os.path.dirname(sys.executable) - if os.environ.get('PATH') is not None: - env_path = os.pathsep.join([env_path, os.environ['PATH']]) - os.environ['PATH'] = env_path - - meta.register_metadata(run_id, step_name, task_id, - [MetaDatum(field='conda_env_id', - value=self._env_id(), - type='conda_env_id', - tags=[ - "attempt_id:{0}". - format(retry_count)])]) - - def runtime_step_cli(self, - cli_args, - retry_count, - max_user_code_retries, - ubf_context): - no_batch = 'batch' not in cli_args.commands - no_kubernetes = 'kubernetes' not in cli_args.commands + if os.environ.get("PATH") is not None: + env_path = os.pathsep.join([env_path, os.environ["PATH"]]) + os.environ["PATH"] = env_path + + meta.register_metadata( + run_id, + step_name, + task_id, + [ + MetaDatum( + field="conda_env_id", + value=self._env_id(), + type="conda_env_id", + tags=["attempt_id:{0}".format(retry_count)], + ) + ], + ) + + def runtime_step_cli( + self, cli_args, retry_count, max_user_code_retries, ubf_context + ): + no_batch = "batch" not in cli_args.commands + no_kubernetes = "kubernetes" not in cli_args.commands if self.is_enabled(ubf_context) and no_batch and no_kubernetes: python_path = self.metaflow_home if self.addl_paths is not None: addl_paths = os.pathsep.join(self.addl_paths) python_path = os.pathsep.join([addl_paths, python_path]) - cli_args.env['PYTHONPATH'] = python_path - cli_args.env['_METAFLOW_CONDA_ENV'] = self.env_id + cli_args.env["PYTHONPATH"] = python_path + cli_args.env["_METAFLOW_CONDA_ENV"] = self.env_id cli_args.entrypoint[0] = self.conda.python(self.env_id) def runtime_finished(self, exception): diff --git a/metaflow/plugins/debug_logger.py b/metaflow/plugins/debug_logger.py index b1e31fea658..0634762b56e 100644 --- a/metaflow/plugins/debug_logger.py +++ b/metaflow/plugins/debug_logger.py @@ -4,10 +4,10 @@ class DebugEventLogger(object): - TYPE = 'debugLogger' + TYPE = "debugLogger" def log(self, msg): - sys.stderr.write('event_logger: ' + str(msg)+'\n') + sys.stderr.write("event_logger: " + str(msg) + "\n") def process_message(self, msg): # type: (Message) -> None diff --git a/metaflow/plugins/debug_monitor.py b/metaflow/plugins/debug_monitor.py index 18cb7041869..40324a2c86a 100644 --- a/metaflow/plugins/debug_monitor.py +++ b/metaflow/plugins/debug_monitor.py @@ -6,29 +6,31 @@ from metaflow.monitor import Timer, deserialize_metric from metaflow.monitor import MEASURE_TYPE, get_monitor_msg_type + class DebugMonitor(object): - TYPE = 'debugMonitor' + TYPE = "debugMonitor" def __init__(self): - self.logger('init') + self.logger("init") def count(self, count): pass def measure(self, timer): # type: (Timer) -> None - self.logger('elapsed time for {}: {}'. - format(timer.name, str(timer.get_duration()))) + self.logger( + "elapsed time for {}: {}".format(timer.name, str(timer.get_duration())) + ) def gauge(self, gauge): pass def process_message(self, msg): # type: (Message) -> None - self.logger('processing message %s' % str(msg.msg_type)) + self.logger("processing message %s" % str(msg.msg_type)) msg_type = get_monitor_msg_type(msg) if msg_type == MEASURE_TYPE: - timer = deserialize_metric(msg.payload.get('timer')) + timer = deserialize_metric(msg.payload.get("timer")) self.measure(timer) else: pass @@ -37,5 +39,5 @@ def shutdown(self): sys.stderr.flush() def logger(self, msg): - print('local_monitor: %s' % msg, file=sys.stderr) + print("local_monitor: %s" % msg, file=sys.stderr) sys.stderr.flush() diff --git a/metaflow/plugins/env_escape/__init__.py b/metaflow/plugins/env_escape/__init__.py index 07ee682685b..5a0c926df88 100644 --- a/metaflow/plugins/env_escape/__init__.py +++ b/metaflow/plugins/env_escape/__init__.py @@ -44,11 +44,12 @@ # it available to any sub-process that launch sa well. # We also store the maximum protocol version that we support for pickle so that # we can determine what to use -ENV_ESCAPE_PY = os.environ.get('METAFLOW_ENV_ESCAPE_PY', sys.executable) +ENV_ESCAPE_PY = os.environ.get("METAFLOW_ENV_ESCAPE_PY", sys.executable) ENV_ESCAPE_PICKLE_VERSION = os.environ.get( - 'METAFLOW_ENV_ESCAPE_PICKLE_VERSION', str(pickle.HIGHEST_PROTOCOL)) -os.environ['METAFLOW_ENV_ESCAPE_PICKLE_VERSION'] = ENV_ESCAPE_PICKLE_VERSION -os.environ['METAFLOW_ENV_ESCAPE_PY'] = ENV_ESCAPE_PY + "METAFLOW_ENV_ESCAPE_PICKLE_VERSION", str(pickle.HIGHEST_PROTOCOL) +) +os.environ["METAFLOW_ENV_ESCAPE_PICKLE_VERSION"] = ENV_ESCAPE_PICKLE_VERSION +os.environ["METAFLOW_ENV_ESCAPE_PY"] = ENV_ESCAPE_PY def generate_trampolines(python_path): @@ -58,7 +59,7 @@ def generate_trampolines(python_path): # in some cases we may want to disable environment escape # functionality, in that case, set METAFLOW_ENV_ESCAPE_DISABLED - if os.environ.get('METAFLOW_ENV_ESCAPE_DISABLED', False) in (True, 'True'): + if os.environ.get("METAFLOW_ENV_ESCAPE_DISABLED", False) in (True, "True"): return python_interpreter_path = ENV_ESCAPE_PY @@ -73,28 +74,38 @@ def generate_trampolines(python_path): # e.name is set to the name of the package that fails to load # so don't error ONLY IF the error is importing this module (but do # error if there is a transitive import error) - if not (isinstance(e, ModuleNotFoundError) and e.name in [ - 'metaflow_extensions', 'metaflow_extensions.plugins', - 'metaflow_extensions.plugins.env_escape']): + if not ( + isinstance(e, ModuleNotFoundError) + and e.name + in [ + "metaflow_extensions", + "metaflow_extensions.plugins", + "metaflow_extensions.plugins.env_escape", + ] + ): print( "Cannot load metaflow_extensions env escape configurations -- " - "if you want to ignore, uninstall metaflow_extensions package") + "if you want to ignore, uninstall metaflow_extensions package" + ) raise else: - paths.append(os.path.dirname(os.path.abspath(custom_escape.__file__)) + - "/configurations") + paths.append( + os.path.dirname(os.path.abspath(custom_escape.__file__)) + "/configurations" + ) for rootpath in paths: for path in os.listdir(rootpath): path = os.path.join(rootpath, path) if os.path.isdir(path): dir_name = os.path.basename(path) - if dir_name.startswith('emulate_'): + if dir_name.startswith("emulate_"): module_names = dir_name[8:].split("__") for module_name in module_names: - with open(os.path.join( - python_path, module_name + ".py"), mode='w') as f: - f.write(""" + with open( + os.path.join(python_path, module_name + ".py"), mode="w" + ) as f: + f.write( + """ import importlib import os import sys @@ -148,13 +159,14 @@ def load(): raise RuntimeError( "Trying to access an escaped module ({module_name}) without a valid interpreter") load() -""" .format( - python_path=python_interpreter_path, - max_pickle_version=max_pickle_version, - path=path, - prefixes=module_names, - module_name=module_name -)) +""".format( + python_path=python_interpreter_path, + max_pickle_version=max_pickle_version, + path=path, + prefixes=module_names, + module_name=module_name, + ) + ) def init(python_interpreter_path, max_pickle_version): @@ -166,8 +178,8 @@ def init(python_interpreter_path, max_pickle_version): path = os.path.join(config_dir, path) if os.path.isdir(path): dir_name = os.path.basename(path) - if dir_name.startswith('emulate_'): + if dir_name.startswith("emulate_"): module_names = dir_name[8:].split("__") create_modules( - python_interpreter_path, max_pickle_version, path, - module_names) + python_interpreter_path, max_pickle_version, path, module_names + ) diff --git a/metaflow/plugins/env_escape/client.py b/metaflow/plugins/env_escape/client.py index c6f2307b60e..4206242d764 100644 --- a/metaflow/plugins/env_escape/client.py +++ b/metaflow/plugins/env_escape/client.py @@ -58,10 +58,17 @@ def __init__(self, python_path, max_pickle_version, config_dir): if os.path.exists(self._socket_path): raise RuntimeError("Existing socket: %s" % self._socket_path) env = os.environ.copy() - #env["PYTHONPATH"] = ":".join(sys.path) + # env["PYTHONPATH"] = ":".join(sys.path) self._server_process = Popen( - [python_path, "-u", "-m", server_module, str(max_pickle_version), - config_dir, self._socket_path], + [ + python_path, + "-u", + "-m", + server_module, + str(max_pickle_version), + config_dir, + self._socket_path, + ], env=env, stdout=PIPE, stderr=PIPE, @@ -146,7 +153,7 @@ def __init__(self, python_path, max_pickle_version, config_dir): "functions": response[FIELD_CONTENT]["functions"], "values": response[FIELD_CONTENT]["values"], "exceptions": response[FIELD_CONTENT]["exceptions"], - "aliases": response[FIELD_CONTENT]["aliases"] + "aliases": response[FIELD_CONTENT]["aliases"], } self._aliases = response[FIELD_CONTENT]["aliases"] @@ -177,7 +184,7 @@ def cleanup(self): {FIELD_MSGTYPE: MSG_CONTROL, FIELD_OPTYPE: CONTROL_SHUTDOWN} ) self._channel.recv(timeout=10) # If we receive, we are sure we - # are good + # are good except: # noqa E722 pass # If there is any issue sending this message, just ignore it self._server_process.kill() @@ -240,7 +247,7 @@ def get_local_class(self, name, obj_id=None): # Gets (and creates if needed), the class mapping to the remote # class of name 'name'. name = self._get_canonical_name(name) - if name == 'function': + if name == "function": # Special handling of pickled functions. We create a new class that # simply has a __call__ method that will forward things back to # the server side. @@ -248,7 +255,8 @@ def get_local_class(self, name, obj_id=None): raise RuntimeError("Local function unpickling without an object ID") if obj_id not in self._proxied_standalone_functions: self._proxied_standalone_functions[obj_id] = create_class( - self, '__function_%s' % obj_id, {}, {}, {}, {'__call__': ''}) + self, "__function_%s" % obj_id, {}, {}, {}, {"__call__": ""} + ) return self._proxied_standalone_functions[obj_id] if name not in self._proxied_classes: @@ -259,9 +267,12 @@ def get_local_class(self, name, obj_id=None): # remote class has and remove UNSUPPORTED things and overridden things remote_methods = self.stub_request(None, OP_GETMETHODS, name) local_class = create_class( - self, name, self._overrides.get(name, {}), - self._getattr_overrides.get(name, {}), self._setattr_overrides.get(name, {}), - remote_methods + self, + name, + self._overrides.get(name, {}), + self._getattr_overrides.get(name, {}), + self._setattr_overrides.get(name, {}), + remote_methods, ) self._proxied_classes[name] = local_class return local_class @@ -302,7 +313,7 @@ def _get_canonical_name(self, name): base_name = self._aliases.get(name) if base_name is not None: return base_name - for idx in reversed([pos for pos, char in enumerate(name) if char == '.']): + for idx in reversed([pos for pos, char in enumerate(name) if char == "."]): base_name = self._aliases.get(name[:idx]) if base_name is not None: return ".".join([base_name, name[idx + 1 :]]) diff --git a/metaflow/plugins/env_escape/client_modules.py b/metaflow/plugins/env_escape/client_modules.py index ae5858b874b..c292cf87b49 100644 --- a/metaflow/plugins/env_escape/client_modules.py +++ b/metaflow/plugins/env_escape/client_modules.py @@ -13,6 +13,7 @@ def _clean_client(client): client.cleanup() + class _WrappedModule(object): def __init__(self, loader, prefix, exports, exception_classes, client): self._loader = loader @@ -73,18 +74,19 @@ def func(*args, **kwargs): "module '%s' has no attribute '%s' -- contact the author of the " "configuration if this is something " "you expect to work (support may be added if it exists in the " - "original library)" % (self._prefix, name)) + "original library)" % (self._prefix, name) + ) return m def __setattr__(self, name, value): if name in ( - "package", - "__spec__", - "_loader", - "_prefix", - "_client", - "_exports", - "_exception_classes", + "package", + "__spec__", + "_loader", + "_prefix", + "_client", + "_exports", + "_exception_classes", ): object.__setattr__(self, name, value) return @@ -130,7 +132,8 @@ def load_module(self, fullname): if self._client is None: if sys.version_info[0] < 3: raise NotImplementedError( - "Environment escape imports are not supported in Python 2") + "Environment escape imports are not supported in Python 2" + ) # We initialize a client and query the modules we handle # The max_pickle_version is the pickle version that the server (so # the underlying interpreter we call into) supports; we determine @@ -138,7 +141,9 @@ def load_module(self, fullname): # of those two max_pickle_version = min(self._max_pickle_version, pickle.HIGHEST_PROTOCOL) - self._client = Client(self._python_path, max_pickle_version, self._config_dir) + self._client = Client( + self._python_path, max_pickle_version, self._config_dir + ) atexit.register(_clean_client, self._client) exports = self._client.get_exports() @@ -153,7 +158,7 @@ def load_module(self, fullname): export_exceptions = exports.get("exceptions", []) self._aliases = exports.get("aliases", {}) for name in itertools.chain( - export_classes, export_functions, export_values + export_classes, export_functions, export_values ): splits = name.rsplit(".", 1) prefixes.add(splits[0]) @@ -229,7 +234,7 @@ def _get_canonical_name(self, name): base_name = self._aliases.get(name) if base_name is not None: return base_name - for idx in reversed([pos for pos, char in enumerate(name) if char == '.']): + for idx in reversed([pos for pos, char in enumerate(name) if char == "."]): base_name = self._aliases.get(name[:idx]) if base_name is not None: return ".".join([base_name, name[idx + 1 :]]) @@ -247,7 +252,8 @@ def create_modules(python_path, max_pickle_version, path, prefixes): else: # pass raise RuntimeError( - "Trying to override %s when module exists in system" % prefix) + "Trying to override %s when module exists in system" % prefix + ) # The first version forces the use of the environment escape even if the module # exists in the system. This is useful for testing to make sure that the @@ -255,4 +261,6 @@ def create_modules(python_path, max_pickle_version, path, prefixes): # will only use the environment escape if the module cannot be found # sys.meta_path.insert(0, ModuleImporter(python_path, path, prefixes)) - sys.meta_path.append(ModuleImporter(python_path, max_pickle_version, path, prefixes)) + sys.meta_path.append( + ModuleImporter(python_path, max_pickle_version, path, prefixes) + ) diff --git a/metaflow/plugins/env_escape/configurations/emulate_test_lib/overrides.py b/metaflow/plugins/env_escape/configurations/emulate_test_lib/overrides.py index c2e70e9cd5c..7418644e8f3 100644 --- a/metaflow/plugins/env_escape/configurations/emulate_test_lib/overrides.py +++ b/metaflow/plugins/env_escape/configurations/emulate_test_lib/overrides.py @@ -25,6 +25,7 @@ def remote_print_value(obj, func): print("Encoding for client") return v + @local_getattr_override({"test_lib.TestClass1": "override_value"}) def local_get_value2(stub, name, func): print("In local getattr override for %s" % name) @@ -85,6 +86,7 @@ def __str__(self): def _deserialize_user(self, json_obj): self.user_value = json_obj + @remote_exception_serialize("test_lib.SomeException") def some_exception_serialize(ex): return 42 diff --git a/metaflow/plugins/env_escape/configurations/emulate_test_lib/server_mappings.py b/metaflow/plugins/env_escape/configurations/emulate_test_lib/server_mappings.py index 2af9b83eb28..63de6f41369 100644 --- a/metaflow/plugins/env_escape/configurations/emulate_test_lib/server_mappings.py +++ b/metaflow/plugins/env_escape/configurations/emulate_test_lib/server_mappings.py @@ -3,8 +3,9 @@ import sys # HACK to pretend that we installed test_lib -sys.path.append(os.path.realpath( - os.path.join(os.path.dirname(__file__), '..', 'test_lib_impl'))) +sys.path.append( + os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "test_lib_impl")) +) import test_lib as lib diff --git a/metaflow/plugins/env_escape/configurations/test_lib_impl/__init__.py b/metaflow/plugins/env_escape/configurations/test_lib_impl/__init__.py index 5c8b64db148..df704cc72b1 100644 --- a/metaflow/plugins/env_escape/configurations/test_lib_impl/__init__.py +++ b/metaflow/plugins/env_escape/configurations/test_lib_impl/__init__.py @@ -1,4 +1,4 @@ # Example library that can be used to demonstrate the use of the env_escape # plugin. -# See test/env_escape/example.py for an example flow that uses this. \ No newline at end of file +# See test/env_escape/example.py for an example flow that uses this. diff --git a/metaflow/plugins/env_escape/data_transferer.py b/metaflow/plugins/env_escape/data_transferer.py index 29a799aeb88..51225b17a39 100644 --- a/metaflow/plugins/env_escape/data_transferer.py +++ b/metaflow/plugins/env_escape/data_transferer.py @@ -37,7 +37,7 @@ class InvalidUnicode: dict, defaultdict, OrderedDict, - datetime + datetime, ] _container_types = (list, tuple, set, frozenset, dict, defaultdict, OrderedDict) @@ -56,7 +56,7 @@ class InvalidUnicode: bytes, unicode, # noqa F821 long, # noqa F821 - datetime + datetime, ) _types_to_encoding = {x: idx for idx, x in enumerate(_types)} @@ -103,7 +103,10 @@ def _load_none(obj_type, transferer, json_annotation, json_obj): @_register_dumper(_simple_types) def _dump_simple(obj_type, transferer, obj): - return (None, base64.b64encode(pickle.dumps(obj, protocol=defaultProtocol)).decode("utf-8")) + return ( + None, + base64.b64encode(pickle.dumps(obj, protocol=defaultProtocol)).decode("utf-8"), + ) @_register_loader(_simple_types) @@ -216,8 +219,9 @@ def dump(self, obj): # This is primarily used to transfer a reference to an object try: json_obj = base64.b64encode( - pickle.dumps(self._connection.pickle_object(obj), - protocol=defaultProtocol) + pickle.dumps( + self._connection.pickle_object(obj), protocol=defaultProtocol + ) ).decode("utf-8") except ValueError as e: raise RuntimeError("Unable to dump non base type: %s" % e) @@ -233,8 +237,9 @@ def load(self, json_obj): # This is something that the connection handles try: return self._connection.unpickle_object( - pickle.loads(base64.b64decode(json_obj[FIELD_INLINE_VALUE]), - encoding="utf-8") + pickle.loads( + base64.b64decode(json_obj[FIELD_INLINE_VALUE]), encoding="utf-8" + ) ) except ValueError as e: raise RuntimeError("Unable to load non base type: %s" % e) @@ -270,7 +275,7 @@ def _sub_process(obj): if isinstance(obj, (tuple, set, frozenset)): cast_to = type(obj) obj = list(obj) - in_place = True # We can do in place since we copied the object + in_place = True # We can do in place since we copied the object if isinstance(obj, OrderedDict): key_change_allowed = False if isinstance(obj, defaultdict): @@ -282,9 +287,9 @@ def _sub_process(obj): if not in_place: obj = copy(obj) in_place = True - obj['__default_factory'] = obj.default_factory + obj["__default_factory"] = obj.default_factory obj.default_factory = None - elif obj.get('__default_factory') is not None: + elif obj.get("__default_factory") is not None: # This is in the unpickle path, we need to reset the factory properly update_default_factory = True has_changes = True @@ -333,8 +338,8 @@ def _sub_process(obj): if update_default_factory: # We do this here because we now unpickled the reference # to default_dict and can set it back up again. - obj.default_factory = obj['__default_factory'] - del obj['__default_factory'] + obj.default_factory = obj["__default_factory"] + del obj["__default_factory"] if has_changes: if cast_to: return cast_to(obj) diff --git a/metaflow/plugins/env_escape/server.py b/metaflow/plugins/env_escape/server.py index 1060ca71f12..119e653f534 100644 --- a/metaflow/plugins/env_escape/server.py +++ b/metaflow/plugins/env_escape/server.py @@ -83,16 +83,18 @@ def __init__(self, max_pickle_version, config_dir): # We will also proxy functions from objects as needed. This is useful # for defaultdict for example since the `default_factory` function is a # lambda that needs to be transferred. - self._class_types_to_names[type(lambda x: x)] = 'function' + self._class_types_to_names[type(lambda x: x)] = "function" # Update all alias information for base_name, aliases in itertools.chain( - a1.items(), a2.items(), a3.items(), a4.items()): + a1.items(), a2.items(), a3.items(), a4.items() + ): for alias in aliases: a = self._aliases.setdefault(alias, base_name) if a != base_name: raise ValueError( - "%s is an alias to both %s and %s" % (alias, base_name, a)) + "%s is an alias to both %s and %s" % (alias, base_name, a) + ) # Determine if we have any overrides self._overrides = {} @@ -254,7 +256,9 @@ def encode_exception(self, ex_type, ex, trace_back): extra_content = None if serializer is not None: extra_content = serializer(ex) - return dump_exception(self._datatransferer, ex_type, ex, trace_back, extra_content) + return dump_exception( + self._datatransferer, ex_type, ex, trace_back, extra_content + ) def decode(self, json_obj): # This decodes an object that was transferred in diff --git a/metaflow/plugins/env_escape/stub.py b/metaflow/plugins/env_escape/stub.py index 1095ac14104..807ab1827e4 100644 --- a/metaflow/plugins/env_escape/stub.py +++ b/metaflow/plugins/env_escape/stub.py @@ -27,8 +27,7 @@ "___identifier___", "___connection___", "___refcount___", - "___local_overrides___" - "__class__", + "___local_overrides___" "__class__", "__init__", "__del__", "__delattr__", @@ -61,6 +60,7 @@ STATIC_METHOD = 1 CLASS_METHOD = 2 + def fwd_request(stub, request_type, *args, **kwargs): connection = object.__getattribute__(stub, "___connection___") return connection.stub_request(stub, request_type, *args, **kwargs) @@ -116,7 +116,7 @@ def __del__(self): pass self.___refcount___ -= 1 if self.___refcount___ == 0: - fwd_request(self, OP_DEL) + fwd_request(self, OP_DEL) except Exception: # raised in a destructor, most likely on program termination, # when the connection might have already been closed. @@ -193,11 +193,13 @@ def method(_self, *args, **kwargs): def static_method(connection, class_name, name, *args, **kwargs): return connection.stub_request( - None, OP_CALLONCLASS, class_name, name, True, *args, **kwargs) + None, OP_CALLONCLASS, class_name, name, True, *args, **kwargs + ) def class_method(connection, class_name, name, cls, *args, **kwargs): return connection.stub_request( - None, OP_CALLONCLASS, class_name, name, False, *args, **kwargs) + None, OP_CALLONCLASS, class_name, name, False, *args, **kwargs + ) if method_type == NORMAL_METHOD: m = method @@ -252,8 +254,14 @@ def __call__(cls, *args, **kwargs): ) -def create_class(connection, class_name, overriden_methods, - getattr_overrides, setattr_overrides, class_methods): +def create_class( + connection, + class_name, + overriden_methods, + getattr_overrides, + setattr_overrides, + class_methods, +): class_dict = {"__slots__": ()} for name, doc in class_methods.items(): @@ -270,24 +278,32 @@ def create_class(connection, class_name, overriden_methods, lambda override, orig_method: lambda obj, *args, **kwargs: override( obj, functools.partial(orig_method, obj), *args, **kwargs ) - )(overriden_methods[name], - _make_method(method_type, connection, class_name, name, doc)) + )( + overriden_methods[name], + _make_method(method_type, connection, class_name, name, doc), + ) elif method_type == STATIC_METHOD: class_dict[name] = ( lambda override, orig_method: lambda *args, **kwargs: override( orig_method, *args, **kwargs ) - )(overriden_methods[name], - _make_method(method_type, connection, class_name, name, doc)) + )( + overriden_methods[name], + _make_method(method_type, connection, class_name, name, doc), + ) elif method_type == CLASS_METHOD: class_dict[name] = ( lambda override, orig_method: lambda cls, *args, **kwargs: override( cls, functools.partial(orig_method, cls), *args, **kwargs ) - )(overriden_methods[name], - _make_method(method_type, connection, class_name, name, doc)) + )( + overriden_methods[name], + _make_method(method_type, connection, class_name, name, doc), + ) elif name not in LOCAL_ATTRS: - class_dict[name] = _make_method(method_type, connection, class_name, name, doc) + class_dict[name] = _make_method( + method_type, connection, class_name, name, doc + ) # Check for any getattr/setattr overrides special_attributes = set(getattr_overrides.keys()) special_attributes.update(set(setattr_overrides.keys())) @@ -296,14 +312,17 @@ def create_class(connection, class_name, overriden_methods, getter = getattr_overrides.get(attr) setter = setattr_overrides.get(attr) if getter is not None: - getter = lambda x, name=attr, inner=getter: \ - inner(x, name, lambda y=x, name=name: y.__getattr__(name)) + getter = lambda x, name=attr, inner=getter: inner( + x, name, lambda y=x, name=name: y.__getattr__(name) + ) if setter is not None: - setter = lambda x, value, name=attr, inner=setter: \ - inner(x, name, \ - lambda val, y=x, name=name: fwd_request(y, OP_SETATTR, name, val), - value) + setter = lambda x, value, name=attr, inner=setter: inner( + x, + name, + lambda val, y=x, name=name: fwd_request(y, OP_SETATTR, name, val), + value, + ) overriden_attrs.add(attr) class_dict[attr] = property(getter, setter) - class_dict['___local_overrides___'] = overriden_attrs + class_dict["___local_overrides___"] = overriden_attrs return MetaWithConnection(class_name, (Stub,), class_dict, connection) diff --git a/metaflow/plugins/environment_decorator.py b/metaflow/plugins/environment_decorator.py index 13da4c969cd..af50f24195e 100644 --- a/metaflow/plugins/environment_decorator.py +++ b/metaflow/plugins/environment_decorator.py @@ -24,8 +24,11 @@ def myStep(self): vars : Dict Dictionary of environment variables to add/update prior to executing your step. """ - name = 'environment' - defaults = {'vars': {}} - def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context): - cli_args.env.update(self.attributes['vars'].items()) \ No newline at end of file + name = "environment" + defaults = {"vars": {}} + + def runtime_step_cli( + self, cli_args, retry_count, max_user_code_retries, ubf_context + ): + cli_args.env.update(self.attributes["vars"].items()) diff --git a/metaflow/plugins/metadata/__init__.py b/metaflow/plugins/metadata/__init__.py index d497e571c26..2552f17f6f4 100644 --- a/metaflow/plugins/metadata/__init__.py +++ b/metaflow/plugins/metadata/__init__.py @@ -1,2 +1,2 @@ from .local import LocalMetadataProvider -from .service import ServiceMetadataProvider \ No newline at end of file +from .service import ServiceMetadataProvider diff --git a/metaflow/plugins/metadata/local.py b/metaflow/plugins/metadata/local.py index 8a96ed066d1..ed4ae16da77 100644 --- a/metaflow/plugins/metadata/local.py +++ b/metaflow/plugins/metadata/local.py @@ -8,20 +8,24 @@ class LocalMetadataProvider(MetadataProvider): - TYPE = 'local' + TYPE = "local" def __init__(self, environment, flow, event_logger, monitor): - super(LocalMetadataProvider, self).__init__(environment, flow, event_logger, monitor) + super(LocalMetadataProvider, self).__init__( + environment, flow, event_logger, monitor + ) @classmethod def compute_info(cls, val): from metaflow.datastore.local_storage import LocalStorage + v = os.path.realpath(os.path.join(val, DATASTORE_LOCAL_DIR)) if os.path.isdir(v): LocalStorage.datastore_root = v return val raise ValueError( - 'Could not find directory %s in directory %s' % (DATASTORE_LOCAL_DIR, val)) + "Could not find directory %s in directory %s" % (DATASTORE_LOCAL_DIR, val) + ) @classmethod def default_info(cls): @@ -29,19 +33,24 @@ def default_info(cls): def print_clean(line, **kwargs): print(line) - v = LocalStorage.get_datastore_root_from_config(print_clean, create_on_absent=False) + + v = LocalStorage.get_datastore_root_from_config( + print_clean, create_on_absent=False + ) if v is None: - return '' % DATASTORE_LOCAL_DIR + return ( + "" % DATASTORE_LOCAL_DIR + ) return os.path.dirname(v) def version(self): - return 'local' + return "local" def new_run_id(self, tags=None, sys_tags=None): # We currently just use the timestamp to create an ID. We can be reasonably certain # that it is unique and this makes it possible to do without coordination or # reliance on POSIX locks in the filesystem. - run_id = '%d' % (time.time() * 1e6) + run_id = "%d" % (time.time() * 1e6) self._new_run(run_id, tags, sys_tags) return run_id @@ -62,63 +71,66 @@ def new_task_id(self, run_id, step_name, tags=None, sys_tags=None): self._new_task(run_id, step_name, task_id, tags, sys_tags) return task_id - def register_task_id(self, - run_id, - step_name, - task_id, - attempt=0, - tags=None, - sys_tags=None): + def register_task_id( + self, run_id, step_name, task_id, attempt=0, tags=None, sys_tags=None + ): try: # Same logic as register_run_id int(task_id) except ValueError: self._new_task(run_id, step_name, task_id, attempt, tags, sys_tags) else: - self._register_code_package_metadata( - run_id, step_name, task_id, attempt) - - def register_data_artifacts(self, - run_id, - step_name, - task_id, - attempt_id, - artifacts): - meta_dir = self._create_and_get_metadir(self._flow_name, run_id, step_name, task_id) - artlist = self._artifacts_to_json(run_id, step_name, task_id, attempt_id, artifacts) - artdict = {'%d_artifact_%s' % (attempt_id, art['name']): art for art in artlist} + self._register_code_package_metadata(run_id, step_name, task_id, attempt) + + def register_data_artifacts( + self, run_id, step_name, task_id, attempt_id, artifacts + ): + meta_dir = self._create_and_get_metadir( + self._flow_name, run_id, step_name, task_id + ) + artlist = self._artifacts_to_json( + run_id, step_name, task_id, attempt_id, artifacts + ) + artdict = {"%d_artifact_%s" % (attempt_id, art["name"]): art for art in artlist} self._save_meta(meta_dir, artdict) def register_metadata(self, run_id, step_name, task_id, metadata): - meta_dir = self._create_and_get_metadir(self._flow_name, run_id, step_name, task_id) + meta_dir = self._create_and_get_metadir( + self._flow_name, run_id, step_name, task_id + ) metalist = self._metadata_to_json(run_id, step_name, task_id, metadata) ts = int(round(time.time() * 1000)) - metadict = {'sysmeta_%s_%d' % (meta['field_name'], ts): meta for meta in metalist} + metadict = { + "sysmeta_%s_%d" % (meta["field_name"], ts): meta for meta in metalist + } self._save_meta(meta_dir, metadict) @classmethod def _get_object_internal( - cls, obj_type, obj_order, sub_type, sub_order, filters, attempt, *args): + cls, obj_type, obj_order, sub_type, sub_order, filters, attempt, *args + ): from metaflow.datastore.local_storage import LocalStorage - if obj_type == 'artifact': + + if obj_type == "artifact": # Artifacts are actually part of the tasks in the filesystem - obj_type = 'task' - sub_type = 'artifact' + obj_type = "task" + sub_type = "artifact" sub_order = obj_order obj_order = obj_order - 1 # Special handling of self, artifact, and metadata - if sub_type == 'self': + if sub_type == "self": meta_path = LocalMetadataProvider._get_metadir(*args[:obj_order]) if meta_path is None: return None - self_file = os.path.join(meta_path, '_self.json') + self_file = os.path.join(meta_path, "_self.json") if os.path.isfile(self_file): return MetadataProvider._apply_filter( - [LocalMetadataProvider._read_json_file(self_file)], filters)[0] + [LocalMetadataProvider._read_json_file(self_file)], filters + )[0] return None - if sub_type == 'artifact': + if sub_type == "artifact": meta_path = LocalMetadataProvider._get_metadir(*args[:obj_order]) result = [] if meta_path is None: @@ -126,42 +138,49 @@ def _get_object_internal( successful_attempt = attempt if successful_attempt is None: - attempt_done_files = os.path.join(meta_path, 'sysmeta_attempt-done_*') + attempt_done_files = os.path.join(meta_path, "sysmeta_attempt-done_*") attempts_done = sorted(glob.iglob(attempt_done_files)) if attempts_done: - successful_attempt = int(LocalMetadataProvider._read_json_file( - attempts_done[-1])['value']) + successful_attempt = int( + LocalMetadataProvider._read_json_file(attempts_done[-1])[ + "value" + ] + ) if successful_attempt is not None: - which_artifact = '*' + which_artifact = "*" if len(args) >= sub_order: which_artifact = args[sub_order - 1] artifact_files = os.path.join( - meta_path, '%d_artifact_%s.json' % (successful_attempt, which_artifact)) + meta_path, + "%d_artifact_%s.json" % (successful_attempt, which_artifact), + ) for obj in glob.iglob(artifact_files): result.append(LocalMetadataProvider._read_json_file(obj)) if len(result) == 1: return result[0] return result - if sub_type == 'metadata': + if sub_type == "metadata": result = [] meta_path = LocalMetadataProvider._get_metadir(*args[:obj_order]) if meta_path is None: return result - files = os.path.join(meta_path, 'sysmeta_*') + files = os.path.join(meta_path, "sysmeta_*") for obj in glob.iglob(files): result.append(LocalMetadataProvider._read_json_file(obj)) return result # For the other types, we locate all the objects we need to find and return them - obj_path = LocalMetadataProvider._make_path(*args[:obj_order], create_on_absent=False) + obj_path = LocalMetadataProvider._make_path( + *args[:obj_order], create_on_absent=False + ) result = [] if obj_path is None: return result - skip_dirs = '*/'*(sub_order - obj_order) + skip_dirs = "*/" * (sub_order - obj_order) all_meta = os.path.join(obj_path, skip_dirs, LocalStorage.METADATA_DIR) for meta_path in glob.iglob(all_meta): - self_file = os.path.join(meta_path, '_self.json') + self_file = os.path.join(meta_path, "_self.json") if os.path.isfile(self_file): result.append(LocalMetadataProvider._read_json_file(self_file)) return MetadataProvider._apply_filter(result, filters) @@ -180,47 +199,60 @@ def _makedirs(path): raise def _ensure_meta( - self, obj_type, run_id, step_name, task_id, tags=None, sys_tags=None): + self, obj_type, run_id, step_name, task_id, tags=None, sys_tags=None + ): if tags is None: tags = set() if sys_tags is None: sys_tags = set() - subpath = self._create_and_get_metadir(self._flow_name, run_id, step_name, task_id) - selfname = os.path.join(subpath, '_self.json') + subpath = self._create_and_get_metadir( + self._flow_name, run_id, step_name, task_id + ) + selfname = os.path.join(subpath, "_self.json") self._makedirs(subpath) if os.path.isfile(selfname): return # In this case, the metadata information does not exist so we create it self._save_meta( subpath, - {'_self': self._object_to_json( - obj_type, - run_id, - step_name, - task_id, - self.sticky_tags.union(tags), - self.sticky_sys_tags.union(sys_tags))}) + { + "_self": self._object_to_json( + obj_type, + run_id, + step_name, + task_id, + self.sticky_tags.union(tags), + self.sticky_sys_tags.union(sys_tags), + ) + }, + ) def _new_run(self, run_id, tags=None, sys_tags=None): - self._ensure_meta('flow', None, None, None) - self._ensure_meta('run', run_id, None, None, tags, sys_tags) - - def _new_task(self, run_id, step_name, task_id, attempt=0, tags=None, sys_tags=None): - self._ensure_meta('step', run_id, step_name, None) - self._ensure_meta('task', run_id, step_name, task_id, tags, sys_tags) + self._ensure_meta("flow", None, None, None) + self._ensure_meta("run", run_id, None, None, tags, sys_tags) + + def _new_task( + self, run_id, step_name, task_id, attempt=0, tags=None, sys_tags=None + ): + self._ensure_meta("step", run_id, step_name, None) + self._ensure_meta("task", run_id, step_name, task_id, tags, sys_tags) self._register_code_package_metadata(run_id, step_name, task_id, attempt) @staticmethod def _make_path( - flow_name=None, run_id=None, step_name=None, task_id=None, - create_on_absent=True): + flow_name=None, run_id=None, step_name=None, task_id=None, create_on_absent=True + ): from metaflow.datastore.local_storage import LocalStorage + if LocalStorage.datastore_root is None: + def print_clean(line, **kwargs): print(line) + LocalStorage.datastore_root = LocalStorage.get_datastore_root_from_config( - print_clean, create_on_absent=create_on_absent) + print_clean, create_on_absent=create_on_absent + ) if LocalStorage.datastore_root is None: return None @@ -235,14 +267,17 @@ def print_clean(line, **kwargs): components.append(step_name) if task_id: components.append(task_id) - return LocalStorage().full_uri( - LocalStorage.path_join(*components)) + return LocalStorage().full_uri(LocalStorage.path_join(*components)) @staticmethod def _create_and_get_metadir( - flow_name=None, run_id=None, step_name=None, task_id=None): + flow_name=None, run_id=None, step_name=None, task_id=None + ): from metaflow.datastore.local_storage import LocalStorage - root_path = LocalMetadataProvider._make_path(flow_name, run_id, step_name, task_id) + + root_path = LocalMetadataProvider._make_path( + flow_name, run_id, step_name, task_id + ) subpath = os.path.join(root_path, LocalStorage.METADATA_DIR) LocalMetadataProvider._makedirs(subpath) return subpath @@ -250,8 +285,10 @@ def _create_and_get_metadir( @staticmethod def _get_metadir(flow_name=None, run_id=None, step_name=None, task_id=None): from metaflow.datastore.local_storage import LocalStorage + root_path = LocalMetadataProvider._make_path( - flow_name, run_id, step_name, task_id, create_on_absent=False) + flow_name, run_id, step_name, task_id, create_on_absent=False + ) if root_path is None: return None subpath = os.path.join(root_path, LocalStorage.METADATA_DIR) @@ -260,21 +297,20 @@ def _get_metadir(flow_name=None, run_id=None, step_name=None, task_id=None): return None @staticmethod - def _dump_json_to_file( - filepath, data, allow_overwrite=False): + def _dump_json_to_file(filepath, data, allow_overwrite=False): if os.path.isfile(filepath) and not allow_overwrite: return - with open(filepath + '.tmp', 'w') as f: + with open(filepath + ".tmp", "w") as f: json.dump(data, f) - os.rename(filepath + '.tmp', filepath) + os.rename(filepath + ".tmp", filepath) @staticmethod def _read_json_file(filepath): - with open(filepath, 'r') as f: + with open(filepath, "r") as f: return json.load(f) @staticmethod def _save_meta(root_dir, metadict): for name, datum in metadict.items(): - filename = os.path.join(root_dir, '%s.json' % name) + filename = os.path.join(root_dir, "%s.json" % name) LocalMetadataProvider._dump_json_to_file(filename, datum) diff --git a/metaflow/plugins/metadata/service.py b/metaflow/plugins/metadata/service.py index 61c16d148ac..2bccd75af05 100644 --- a/metaflow/plugins/metadata/service.py +++ b/metaflow/plugins/metadata/service.py @@ -5,8 +5,11 @@ from distutils.version import LooseVersion from metaflow.exception import MetaflowException -from metaflow.metaflow_config import METADATA_SERVICE_NUM_RETRIES, METADATA_SERVICE_HEADERS, \ - METADATA_SERVICE_URL +from metaflow.metaflow_config import ( + METADATA_SERVICE_NUM_RETRIES, + METADATA_SERVICE_HEADERS, + METADATA_SERVICE_URL, +) from metaflow.metadata import MetadataProvider from metaflow.metadata.heartbeat import HB_URL_KEY from metaflow.sidecar import SidecarSubProcess @@ -17,8 +20,9 @@ class HeartbeatTypes(object): RUN = 1 TASK = 2 + class ServiceException(MetaflowException): - headline = 'Metaflow service error' + headline = "Metaflow service error" def __init__(self, msg, http_code=None, body=None): self.http_code = None if http_code is None else int(http_code) @@ -27,26 +31,33 @@ def __init__(self, msg, http_code=None, body=None): class ServiceMetadataProvider(MetadataProvider): - TYPE = 'service' + TYPE = "service" _supports_attempt_gets = None def __init__(self, environment, flow, event_logger, monitor): - super(ServiceMetadataProvider, self).__init__(environment, flow, event_logger, monitor) - self.url_task_template = os.path.join(METADATA_SERVICE_URL, - 'flows/{flow_id}/runs/{run_number}/steps/{step_name}/tasks/{task_id}/heartbeat') - self.url_run_template = os.path.join(METADATA_SERVICE_URL, - 'flows/{flow_id}/runs/{run_number}/heartbeat') + super(ServiceMetadataProvider, self).__init__( + environment, flow, event_logger, monitor + ) + self.url_task_template = os.path.join( + METADATA_SERVICE_URL, + "flows/{flow_id}/runs/{run_number}/steps/{step_name}/tasks/{task_id}/heartbeat", + ) + self.url_run_template = os.path.join( + METADATA_SERVICE_URL, "flows/{flow_id}/runs/{run_number}/heartbeat" + ) self.sidecar_process = None @classmethod def compute_info(cls, val): - v = val.rstrip('/') + v = val.rstrip("/") try: - resp = requests.get(os.path.join(v, 'ping'), headers=METADATA_SERVICE_HEADERS) + resp = requests.get( + os.path.join(v, "ping"), headers=METADATA_SERVICE_HEADERS + ) resp.raise_for_status() except: # noqa E722 - raise ValueError('Metaflow service [%s] unreachable.' % v) + raise ValueError("Metaflow service [%s] unreachable." % v) return v @classmethod @@ -71,36 +82,32 @@ def register_run_id(self, run_id, tags=None, sys_tags=None): def new_task_id(self, run_id, step_name, tags=None, sys_tags=None): return self._new_task(run_id, step_name, tags=tags, sys_tags=sys_tags) - def register_task_id(self, - run_id, - step_name, - task_id, - attempt=0, - tags=None, - sys_tags=None): + def register_task_id( + self, run_id, step_name, task_id, attempt=0, tags=None, sys_tags=None + ): try: # don't try to register an integer ID which was obtained # from the metadata service in the first place int(task_id) except ValueError: - self._new_task(run_id, - step_name, - task_id, - attempt, - tags=tags, - sys_tags=sys_tags) + self._new_task( + run_id, step_name, task_id, attempt, tags=tags, sys_tags=sys_tags + ) else: self._register_code_package_metadata(run_id, step_name, task_id, attempt) - def _start_heartbeat(self, heartbeat_type, flow_id, run_id, step_name=None, task_id=None): + def _start_heartbeat( + self, heartbeat_type, flow_id, run_id, step_name=None, task_id=None + ): if self._already_started(): # A single ServiceMetadataProvider instance can not start # multiple heartbeat side cars of any type/combination. Either a # single run heartbeat or a single task heartbeat can be started raise Exception("heartbeat already started") # start sidecar - if self.version() is None or \ - LooseVersion(self.version()) < LooseVersion('2.0.4'): + if self.version() is None or LooseVersion(self.version()) < LooseVersion( + "2.0.4" + ): # if old version of the service is running # then avoid running real heartbeat sidecar process self.sidecar_process = SidecarSubProcess("nullSidecarHeartbeat") @@ -111,13 +118,15 @@ def _start_heartbeat(self, heartbeat_type, flow_id, run_id, step_name=None, task if heartbeat_type == HeartbeatTypes.TASK: # create task heartbeat data = { - 'flow_id': flow_id, 'run_number': run_id, - 'step_name': step_name, 'task_id': task_id, - } + "flow_id": flow_id, + "run_number": run_id, + "step_name": step_name, + "task_id": task_id, + } payload[HB_URL_KEY] = self.url_task_template.format(**data) elif heartbeat_type == HeartbeatTypes.RUN: # create run heartbeat - data = {'flow_id': flow_id, 'run_number': run_id} + data = {"flow_id": flow_id, "run_number": run_id} payload[HB_URL_KEY] = self.url_run_template.format(**data) else: @@ -130,11 +139,7 @@ def start_run_heartbeat(self, flow_id, run_id): self._start_heartbeat(HeartbeatTypes.RUN, flow_id, run_id) def start_task_heartbeat(self, flow_id, run_id, step_name, task_id): - self._start_heartbeat(HeartbeatTypes.TASK, - flow_id, - run_id, - step_name, - task_id) + self._start_heartbeat(HeartbeatTypes.TASK, flow_id, run_id, step_name, task_id) def _already_started(self): return self.sidecar_process is not None @@ -143,62 +148,71 @@ def stop_heartbeat(self): msg = Message(MessageTypes.SHUTDOWN, None) self.sidecar_process.msg_handler(msg) - def register_data_artifacts(self, - run_id, - step_name, - task_id, - attempt_id, - artifacts): - url = ServiceMetadataProvider._obj_path(self._flow_name, run_id, step_name, task_id) - url += '/artifact' - data = self._artifacts_to_json(run_id, step_name, task_id, attempt_id, artifacts) + def register_data_artifacts( + self, run_id, step_name, task_id, attempt_id, artifacts + ): + url = ServiceMetadataProvider._obj_path( + self._flow_name, run_id, step_name, task_id + ) + url += "/artifact" + data = self._artifacts_to_json( + run_id, step_name, task_id, attempt_id, artifacts + ) self._request(self._monitor, url, data) def register_metadata(self, run_id, step_name, task_id, metadata): - url = ServiceMetadataProvider._obj_path(self._flow_name, run_id, step_name, task_id) - url += '/metadata' + url = ServiceMetadataProvider._obj_path( + self._flow_name, run_id, step_name, task_id + ) + url += "/metadata" data = self._metadata_to_json(run_id, step_name, task_id, metadata) self._request(self._monitor, url, data) @classmethod def _get_object_internal( - cls, obj_type, obj_order, sub_type, sub_order, filters, attempt, *args): + cls, obj_type, obj_order, sub_type, sub_order, filters, attempt, *args + ): if attempt is not None: if cls._supports_attempt_gets is None: version = cls._version(None) - cls._supports_attempt_gets = version is not None and \ - LooseVersion(version) >= LooseVersion('2.0.6') + cls._supports_attempt_gets = version is not None and LooseVersion( + version + ) >= LooseVersion("2.0.6") if not cls._supports_attempt_gets: raise ServiceException( "Getting specific attempts of Tasks or Artifacts requires " "the metaflow service to be at least version 2.0.6. Please " - "upgrade your service") + "upgrade your service" + ) - if sub_type == 'self': - if obj_type == 'artifact': + if sub_type == "self": + if obj_type == "artifact": # Special case with the artifacts; we add the attempt url = ServiceMetadataProvider._obj_path( - *args[:obj_order], attempt=attempt) + *args[:obj_order], attempt=attempt + ) else: url = ServiceMetadataProvider._obj_path(*args[:obj_order]) try: - return MetadataProvider._apply_filter([cls._request(None, url)], filters)[0] + return MetadataProvider._apply_filter( + [cls._request(None, url)], filters + )[0] except ServiceException as ex: if ex.http_code == 404: return None raise # For the other types, we locate all the objects we need to find and return them - if obj_type != 'root': + if obj_type != "root": url = ServiceMetadataProvider._obj_path(*args[:obj_order]) else: - url = '' - if sub_type == 'metadata': - url += '/metadata' - elif sub_type == 'artifact' and obj_type == 'task' and attempt is not None: - url += '/attempt/%s/artifacts' % attempt + url = "" + if sub_type == "metadata": + url += "/metadata" + elif sub_type == "artifact" and obj_type == "task" and attempt is not None: + url += "/attempt/%s/artifacts" % attempt else: - url += '/%ss' % sub_type + url += "/%ss" % sub_type try: return MetadataProvider._apply_filter(cls._request(None, url), filters) except ServiceException as ex: @@ -208,59 +222,72 @@ def _get_object_internal( def _new_run(self, run_id=None, tags=None, sys_tags=None): # first ensure that the flow exists - self._get_or_create('flow') - run = self._get_or_create('run', run_id, tags=tags, sys_tags=sys_tags) - return str(run['run_number']) - - def _new_task(self, - run_id, - step_name, - task_id=None, - attempt=0, - tags=None, - sys_tags=None): + self._get_or_create("flow") + run = self._get_or_create("run", run_id, tags=tags, sys_tags=sys_tags) + return str(run["run_number"]) + + def _new_task( + self, run_id, step_name, task_id=None, attempt=0, tags=None, sys_tags=None + ): # first ensure that the step exists - self._get_or_create('step', run_id, step_name) - task = self._get_or_create('task', run_id, step_name, task_id, tags=tags, sys_tags=sys_tags) - self._register_code_package_metadata(run_id, step_name, task['task_id'], attempt) - return task['task_id'] + self._get_or_create("step", run_id, step_name) + task = self._get_or_create( + "task", run_id, step_name, task_id, tags=tags, sys_tags=sys_tags + ) + self._register_code_package_metadata( + run_id, step_name, task["task_id"], attempt + ) + return task["task_id"] @staticmethod def _obj_path( - flow_name, run_id=None, step_name=None, task_id=None, - artifact_name=None, attempt=None): - object_path = '/flows/%s' % flow_name + flow_name, + run_id=None, + step_name=None, + task_id=None, + artifact_name=None, + attempt=None, + ): + object_path = "/flows/%s" % flow_name if run_id is not None: - object_path += '/runs/%s' % run_id + object_path += "/runs/%s" % run_id if step_name is not None: - object_path += '/steps/%s' % step_name + object_path += "/steps/%s" % step_name if task_id is not None: - object_path += '/tasks/%s' % task_id + object_path += "/tasks/%s" % task_id if artifact_name is not None: - object_path += '/artifacts/%s' % artifact_name + object_path += "/artifacts/%s" % artifact_name if attempt is not None: - object_path += '/attempt/%s' % attempt + object_path += "/attempt/%s" % attempt return object_path @staticmethod def _create_path(obj_type, flow_name, run_id=None, step_name=None): - create_path = '/flows/%s' % flow_name - if obj_type == 'flow': + create_path = "/flows/%s" % flow_name + if obj_type == "flow": return create_path - if obj_type == 'run': - return create_path + '/run' - create_path += '/runs/%s/steps/%s' % (run_id, step_name) - if obj_type == 'step': - return create_path + '/step' - return create_path + '/task' + if obj_type == "run": + return create_path + "/run" + create_path += "/runs/%s/steps/%s" % (run_id, step_name) + if obj_type == "step": + return create_path + "/step" + return create_path + "/task" def _get_or_create( - self, obj_type, run_id=None, step_name=None, task_id=None, tags=None, sys_tags=None): + self, + obj_type, + run_id=None, + step_name=None, + task_id=None, + tags=None, + sys_tags=None, + ): if tags is None: tags = set() if sys_tags is None: sys_tags = set() + def create_object(): data = self._object_to_json( obj_type, @@ -268,15 +295,16 @@ def create_object(): step_name, task_id, self.sticky_tags.union(tags), - self.sticky_sys_tags.union(sys_tags)) + self.sticky_sys_tags.union(sys_tags), + ) return self._request(self._monitor, create_path, data, obj_path) always_create = False obj_path = self._obj_path(self._flow_name, run_id, step_name, task_id) create_path = self._create_path(obj_type, self._flow_name, run_id, step_name) - if obj_type == 'run' and run_id is None: + if obj_type == "run" and run_id is None: always_create = True - elif obj_type == 'task' and task_id is None: + elif obj_type == "task" and task_id is None: always_create = True if always_create: @@ -293,26 +321,32 @@ def create_object(): @classmethod def _request(cls, monitor, path, data=None, retry_409_path=None): if cls.INFO is None: - raise MetaflowException('Missing Metaflow Service URL. ' - 'Specify with METAFLOW_SERVICE_URL environment variable') - url = os.path.join(cls.INFO, path.lstrip('/')) + raise MetaflowException( + "Missing Metaflow Service URL. " + "Specify with METAFLOW_SERVICE_URL environment variable" + ) + url = os.path.join(cls.INFO, path.lstrip("/")) for i in range(METADATA_SERVICE_NUM_RETRIES): try: if data is None: if monitor: - with monitor.measure('metaflow.service_metadata.get'): + with monitor.measure("metaflow.service_metadata.get"): resp = requests.get(url, headers=METADATA_SERVICE_HEADERS) else: resp = requests.get(url, headers=METADATA_SERVICE_HEADERS) else: if monitor: - with monitor.measure('metaflow.service_metadata.post'): - resp = requests.post(url, headers=METADATA_SERVICE_HEADERS, json=data) + with monitor.measure("metaflow.service_metadata.post"): + resp = requests.post( + url, headers=METADATA_SERVICE_HEADERS, json=data + ) else: - resp = requests.post(url, headers=METADATA_SERVICE_HEADERS, json=data) + resp = requests.post( + url, headers=METADATA_SERVICE_HEADERS, json=data + ) except: # noqa E722 if monitor: - with monitor.count('metaflow.service_metadata.failed_request'): + with monitor.count("metaflow.service_metadata.failed_request"): if i == METADATA_SERVICE_NUM_RETRIES - 1: raise else: @@ -336,39 +370,43 @@ def _request(cls, monitor, path, data=None, retry_409_path=None): else: return elif resp.status_code != 503: - raise ServiceException('Metadata request (%s) failed (code %s): %s' - % (path, resp.status_code, resp.text), - resp.status_code, - resp.text) - time.sleep(2**i) + raise ServiceException( + "Metadata request (%s) failed (code %s): %s" + % (path, resp.status_code, resp.text), + resp.status_code, + resp.text, + ) + time.sleep(2 ** i) if resp: - raise ServiceException('Metadata request (%s) failed (code %s): %s' - % (path, resp.status_code, resp.text), - resp.status_code, - resp.text) + raise ServiceException( + "Metadata request (%s) failed (code %s): %s" + % (path, resp.status_code, resp.text), + resp.status_code, + resp.text, + ) else: - raise ServiceException('Metadata request (%s) failed' % path) + raise ServiceException("Metadata request (%s) failed" % path) @classmethod def _version(cls, monitor): if cls.INFO is None: - raise MetaflowException('Missing Metaflow Service URL. ' - 'Specify with METAFLOW_SERVICE_URL environment variable') - path = 'ping' + raise MetaflowException( + "Missing Metaflow Service URL. " + "Specify with METAFLOW_SERVICE_URL environment variable" + ) + path = "ping" url = os.path.join(cls.INFO, path) for i in range(METADATA_SERVICE_NUM_RETRIES): try: if monitor: - with monitor.measure('metaflow.service_metadata.get'): - resp = requests.get(url, - headers=METADATA_SERVICE_HEADERS) + with monitor.measure("metaflow.service_metadata.get"): + resp = requests.get(url, headers=METADATA_SERVICE_HEADERS) else: resp = requests.get(url, headers=METADATA_SERVICE_HEADERS) except: if monitor: - with monitor.count( - 'metaflow.service_metadata.failed_request'): + with monitor.count("metaflow.service_metadata.failed_request"): if i == METADATA_SERVICE_NUM_RETRIES - 1: raise else: @@ -377,18 +415,21 @@ def _version(cls, monitor): resp = None else: if resp.status_code < 300: - return resp.headers.get('METADATA_SERVICE_VERSION', None) + return resp.headers.get("METADATA_SERVICE_VERSION", None) elif resp.status_code != 503: - raise ServiceException('Metadata request (%s) failed' - ' (code %s): %s' % - (url, resp.status_code, resp.text), - resp.status_code, - resp.text) - time.sleep(2**i) + raise ServiceException( + "Metadata request (%s) failed" + " (code %s): %s" % (url, resp.status_code, resp.text), + resp.status_code, + resp.text, + ) + time.sleep(2 ** i) if resp: - raise ServiceException('Metadata request (%s) failed (code %s): %s' - % (url, resp.status_code, resp.text), - resp.status_code, - resp.text) + raise ServiceException( + "Metadata request (%s) failed (code %s): %s" + % (url, resp.status_code, resp.text), + resp.status_code, + resp.text, + ) else: - raise ServiceException('Metadata request (%s) failed' % url) + raise ServiceException("Metadata request (%s) failed" % url) diff --git a/metaflow/plugins/package_cli.py b/metaflow/plugins/package_cli.py index 0d42004cc83..ded3c53e25d 100644 --- a/metaflow/plugins/package_cli.py +++ b/metaflow/plugins/package_cli.py @@ -2,49 +2,56 @@ from hashlib import sha1 from metaflow.package import MetaflowPackage + @click.group() def cli(): pass -@cli.group(help='Commands related to code packages.') + +@cli.group(help="Commands related to code packages.") @click.pass_obj def package(obj): # Prepare the package before any of the sub-commands are invoked. - obj.package = MetaflowPackage(obj.flow, - obj.environment, - obj.echo, - obj.package_suffixes) + obj.package = MetaflowPackage( + obj.flow, obj.environment, obj.echo, obj.package_suffixes + ) + -@package.command(help='Output information about the current code package.') +@package.command(help="Output information about the current code package.") @click.pass_obj def info(obj): - obj.echo('Status of the current working directory:', fg='magenta', bold=False) - obj.echo_always('Hash: *%s*' % sha1(obj.package.blob).hexdigest(), - highlight='green', - highlight_bold=False) - obj.echo_always('Package size: *%d* KB' % (len(obj.package.blob) / 1024), - highlight='green', - highlight_bold=False) + obj.echo("Status of the current working directory:", fg="magenta", bold=False) + obj.echo_always( + "Hash: *%s*" % sha1(obj.package.blob).hexdigest(), + highlight="green", + highlight_bold=False, + ) + obj.echo_always( + "Package size: *%d* KB" % (len(obj.package.blob) / 1024), + highlight="green", + highlight_bold=False, + ) num = sum(1 for _ in obj.package.path_tuples()) - obj.echo_always('Number of files: *%d*' % num, - highlight='green', - highlight_bold=False) + obj.echo_always( + "Number of files: *%d*" % num, highlight="green", highlight_bold=False + ) -@package.command(help='List files included in the code package.') + +@package.command(help="List files included in the code package.") @click.pass_obj def list(obj): - obj.echo('Files included in the code package ' - '(change with --package-suffixes):', - fg='magenta', - bold=False) - obj.echo_always('\n'.join(path for path, _ in obj.package.path_tuples())) - -@package.command(help='Save the current code package in a tar file') -@click.argument('path') + obj.echo( + "Files included in the code package " "(change with --package-suffixes):", + fg="magenta", + bold=False, + ) + obj.echo_always("\n".join(path for path, _ in obj.package.path_tuples())) + + +@package.command(help="Save the current code package in a tar file") +@click.argument("path") @click.pass_obj def save(obj, path): - with open(path, 'wb') as f: + with open(path, "wb") as f: f.write(obj.package.blob) - obj.echo('Code package saved in *%s*.' % path, - fg='magenta', - bold=False) + obj.echo("Code package saved in *%s*." % path, fg="magenta", bold=False) diff --git a/metaflow/plugins/project_decorator.py b/metaflow/plugins/project_decorator.py index 89f7929966d..a3e44c01ad9 100644 --- a/metaflow/plugins/project_decorator.py +++ b/metaflow/plugins/project_decorator.py @@ -8,97 +8,106 @@ # be careful when changing these limits. Other systems that see # these names may rely on these limits -VALID_NAME_RE = '[^a-z0-9_]' +VALID_NAME_RE = "[^a-z0-9_]" VALID_NAME_LEN = 128 + class ProjectDecorator(FlowDecorator): - name = 'project' - defaults = {'name': None} + name = "project" + defaults = {"name": None} options = { - 'production': dict( + "production": dict( is_flag=True, default=False, show_default=True, help="Use the @project's production branch. To " - "use a custom branch, use --branch."), - 'branch': dict( + "use a custom branch, use --branch.", + ), + "branch": dict( default=None, show_default=False, help="Use the given branch name under @project. " - "The default is the user name if --production is " - "not specified.") + "The default is the user name if --production is " + "not specified.", + ), } - def flow_init(self, - flow, - graph, - environment, - flow_datastore, - metadata, - logger, - echo, - options): + def flow_init( + self, flow, graph, environment, flow_datastore, metadata, logger, echo, options + ): self._option_values = options - project_name = self.attributes.get('name') - project_flow_name, branch_name = format_name(flow.name, - project_name, - options['production'], - options['branch'], - get_username()) - is_user_branch = options['branch'] is None and not options['production'] - echo("Project: *%s*, Branch: *%s*" % (project_name, branch_name), - fg='magenta', - highlight='green') - current._update_env({'project_name': project_name, - 'branch_name': branch_name, - 'is_user_branch': is_user_branch, - 'is_production': options['production'], - 'project_flow_name': project_flow_name}) - metadata.add_sticky_tags(sys_tags=[ - 'project:%s' % project_name, - 'project_branch:%s' % branch_name]) + project_name = self.attributes.get("name") + project_flow_name, branch_name = format_name( + flow.name, + project_name, + options["production"], + options["branch"], + get_username(), + ) + is_user_branch = options["branch"] is None and not options["production"] + echo( + "Project: *%s*, Branch: *%s*" % (project_name, branch_name), + fg="magenta", + highlight="green", + ) + current._update_env( + { + "project_name": project_name, + "branch_name": branch_name, + "is_user_branch": is_user_branch, + "is_production": options["production"], + "project_flow_name": project_flow_name, + } + ) + metadata.add_sticky_tags( + sys_tags=["project:%s" % project_name, "project_branch:%s" % branch_name] + ) def get_top_level_options(self): return list(self._option_values.items()) -def format_name(flow_name, - project_name, - deploy_prod, - given_branch, - user_name): + +def format_name(flow_name, project_name, deploy_prod, given_branch, user_name): if not project_name: # an empty string is not a valid project name - raise MetaflowException("@project needs a name. " - "Try @project(name='some_name')") + raise MetaflowException( + "@project needs a name. " "Try @project(name='some_name')" + ) elif re.search(VALID_NAME_RE, project_name): - raise MetaflowException("The @project name must contain only " - "lowercase alphanumeric characters " - "and underscores.") + raise MetaflowException( + "The @project name must contain only " + "lowercase alphanumeric characters " + "and underscores." + ) elif len(project_name) > VALID_NAME_LEN: - raise MetaflowException("The @project name must be shorter than " - "%d characters." % VALID_NAME_LEN) + raise MetaflowException( + "The @project name must be shorter than " "%d characters." % VALID_NAME_LEN + ) if given_branch: if re.search(VALID_NAME_RE, given_branch): - raise MetaflowException("The branch name must contain only " - "lowercase alphanumeric characters " - "and underscores.") + raise MetaflowException( + "The branch name must contain only " + "lowercase alphanumeric characters " + "and underscores." + ) elif len(given_branch) > VALID_NAME_LEN: - raise MetaflowException("Branch name is too long. " - "The maximum is %d characters."\ - % VALID_NAME_LEN) + raise MetaflowException( + "Branch name is too long. " + "The maximum is %d characters." % VALID_NAME_LEN + ) if deploy_prod: - branch = 'prod.%s' % given_branch + branch = "prod.%s" % given_branch else: - branch = 'test.%s' % given_branch + branch = "test.%s" % given_branch elif deploy_prod: - branch = 'prod' + branch = "prod" else: # For AWS Step Functions, we set the branch to the value of # environment variable `METAFLOW_OWNER`, since AWS Step Functions # has no notion of user name. - branch = 'user.%s' % os.environ.get('METAFLOW_OWNER', user_name) + branch = "user.%s" % os.environ.get("METAFLOW_OWNER", user_name) - return '.'.join((project_name, branch, flow_name)), branch \ No newline at end of file + return ".".join((project_name, branch, flow_name)), branch diff --git a/metaflow/plugins/resources_decorator.py b/metaflow/plugins/resources_decorator.py index 38288a4ce66..19433742ed1 100644 --- a/metaflow/plugins/resources_decorator.py +++ b/metaflow/plugins/resources_decorator.py @@ -4,13 +4,13 @@ class ResourcesDecorator(StepDecorator): """ Step decorator to specify the resources needed when executing this step. - + This decorator passes this information along to container orchestrator (AWS Batch, Kubernetes, etc.) when requesting resources to execute this step. - + This decorator is ignored if the execution of the step happens locally. - + To use, annotate your step as follows: ``` @resources(cpu=32) @@ -30,10 +30,6 @@ def my_step(self): The value for the size (in MiB) of the /dev/shm volume for this step. This parameter maps to the --shm-size option to docker run . """ - name = 'resources' - defaults = { - 'cpu': '1', - 'gpu': '0', - 'memory': '4096', - 'shared_memory': None - } \ No newline at end of file + + name = "resources" + defaults = {"cpu": "1", "gpu": "0", "memory": "4096", "shared_memory": None} diff --git a/metaflow/plugins/retry_decorator.py b/metaflow/plugins/retry_decorator.py index 68e52375907..e5cc5b72e0d 100644 --- a/metaflow/plugins/retry_decorator.py +++ b/metaflow/plugins/retry_decorator.py @@ -29,16 +29,18 @@ def myStep(self): minutes_between_retries : int Number of minutes between retries """ - name = 'retry' - defaults = {'times': '3', - 'minutes_between_retries': '2'} + + name = "retry" + defaults = {"times": "3", "minutes_between_retries": "2"} def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger): # The total number of attempts must not exceed MAX_ATTEMPTS. # attempts = normal task (1) + retries (N) + @catch fallback (1) - if int(self.attributes['times']) + 2 > MAX_ATTEMPTS: - raise MetaflowException('The maximum number of retries is ' - '@retry(times=%d).' % (MAX_ATTEMPTS - 2)) + if int(self.attributes["times"]) + 2 > MAX_ATTEMPTS: + raise MetaflowException( + "The maximum number of retries is " + "@retry(times=%d)." % (MAX_ATTEMPTS - 2) + ) def step_task_retry_count(self): - return int(self.attributes['times']), 0 + return int(self.attributes["times"]), 0 diff --git a/metaflow/plugins/test_unbounded_foreach_decorator.py b/metaflow/plugins/test_unbounded_foreach_decorator.py index c7693ff039b..ff6dfd0ad48 100644 --- a/metaflow/plugins/test_unbounded_foreach_decorator.py +++ b/metaflow/plugins/test_unbounded_foreach_decorator.py @@ -11,12 +11,14 @@ from metaflow.unbounded_foreach import UnboundedForeachInput, UBF_CONTROL, UBF_TASK from metaflow.util import to_unicode + class InternalTestUnboundedForeachInput(UnboundedForeachInput): """ Test class that wraps around values (any iterator) and simulates an unbounded-foreach instead of a bounded foreach. """ - NAME = 'InternalTestUnboundedForeachInput' + + NAME = "InternalTestUnboundedForeachInput" def __init__(self, iterable): self.iterable = iterable @@ -41,98 +43,96 @@ def __str__(self): return str(self.iterable) def __repr__(self): - return '%s(%s)' % (self.NAME, self.iterable) + return "%s(%s)" % (self.NAME, self.iterable) + class InternalTestUnboundedForeachDecorator(StepDecorator): - name = 'unbounded_test_foreach_internal' + name = "unbounded_test_foreach_internal" results_dict = {} - def __init__(self, - attributes=None, - statically_defined=False): + def __init__(self, attributes=None, statically_defined=False): super(InternalTestUnboundedForeachDecorator, self).__init__( - attributes, statically_defined) - - def step_init(self, - flow, - graph, - step_name, - decorators, - environment, - flow_datastore, - logger): + attributes, statically_defined + ) + + def step_init( + self, flow, graph, step_name, decorators, environment, flow_datastore, logger + ): self.environment = environment def control_task_step_func(self, flow, graph, retry_count): from metaflow import current + run_id = current.run_id step_name = current.step_name control_task_id = current.task_id - (_, split_step_name, split_task_id) = control_task_id.split('-')[1:] + (_, split_step_name, split_task_id) = control_task_id.split("-")[1:] # If we are running inside Conda, we use the base executable FIRST; # the conda environment will then be used when runtime_step_cli is # called. This is so that it can properly set up all the metaflow # aliases needed. - env_to_use = getattr(self.environment, 'base_env', self.environment) + env_to_use = getattr(self.environment, "base_env", self.environment) executable = env_to_use.executable(step_name) script = sys.argv[0] # Access the `unbounded_foreach` param using `flow` (as datastore). - assert(flow._unbounded_foreach) + assert flow._unbounded_foreach foreach_iter = flow.input if not isinstance(foreach_iter, InternalTestUnboundedForeachInput): - raise MetaflowException('Expected type to be '\ - 'InternalTestUnboundedForeachInput. Found %s'\ - % (type(foreach_iter))) + raise MetaflowException( + "Expected type to be " + "InternalTestUnboundedForeachInput. Found %s" % (type(foreach_iter)) + ) foreach_num_splits = sum(1 for _ in foreach_iter) - print('Simulating UnboundedForeach over value:', - foreach_iter, 'num_splits:', foreach_num_splits) + print( + "Simulating UnboundedForeach over value:", + foreach_iter, + "num_splits:", + foreach_num_splits, + ) mapper_tasks = [] for i in range(foreach_num_splits): - task_id = \ - '%s-%d' % (control_task_id.replace('control-', 'test-ubf-'), i) - pathspec = '%s/%s/%s' % (run_id, step_name, task_id) + task_id = "%s-%d" % (control_task_id.replace("control-", "test-ubf-"), i) + pathspec = "%s/%s/%s" % (run_id, step_name, task_id) mapper_tasks.append(to_unicode(pathspec)) - input_paths = '%s/%s/%s' % (run_id, split_step_name, split_task_id) + input_paths = "%s/%s/%s" % (run_id, split_step_name, split_task_id) # Override specific `step` kwargs. kwargs = cli_args.step_kwargs - kwargs['split_index'] = str(i) - kwargs['run_id'] = run_id - kwargs['task_id'] = task_id - kwargs['input_paths'] = input_paths - kwargs['ubf_context'] = UBF_TASK - kwargs['retry_count'] = 0 - - cmd = cli_args.step_command(executable, script, step_name, - step_kwargs=kwargs) - step_cli = u' '.join(cmd) + kwargs["split_index"] = str(i) + kwargs["run_id"] = run_id + kwargs["task_id"] = task_id + kwargs["input_paths"] = input_paths + kwargs["ubf_context"] = UBF_TASK + kwargs["retry_count"] = 0 + + cmd = cli_args.step_command( + executable, script, step_name, step_kwargs=kwargs + ) + step_cli = u" ".join(cmd) # Print cmdline for execution. Doesn't work without the temporary # unicode object while using `print`. - print(u'[${cwd}] Starting split#{split} with cmd:{cmd}'\ - .format(cwd=os.getcwd(), - split=i, - cmd=step_cli)) + print( + u"[${cwd}] Starting split#{split} with cmd:{cmd}".format( + cwd=os.getcwd(), split=i, cmd=step_cli + ) + ) output_bytes = subprocess.check_output(cmd) output = to_unicode(output_bytes) for line in output.splitlines(): - print('[Split#%d] %s' % (i, line)) + print("[Split#%d] %s" % (i, line)) # Save the list of (child) mapper task pathspec(s) into a designated # artifact `_control_mapper_tasks`. flow._control_mapper_tasks = mapper_tasks - - def task_decorate(self, - step_func, - flow, - graph, - retry_count, - max_user_code_retries, - ubf_context): + def task_decorate( + self, step_func, flow, graph, retry_count, max_user_code_retries, ubf_context + ): if ubf_context == UBF_CONTROL: from functools import partial + return partial(self.control_task_step_func, flow, graph, retry_count) else: return step_func @@ -140,4 +140,4 @@ def task_decorate(self, def step_task_retry_count(self): # UBF plugins don't want retry for the control task. We signal this # intent to the runtime by returning (None, None). - return None, None \ No newline at end of file + return None, None diff --git a/metaflow/plugins/timeout_decorator.py b/metaflow/plugins/timeout_decorator.py index f5278177fbb..bfede534bb0 100644 --- a/metaflow/plugins/timeout_decorator.py +++ b/metaflow/plugins/timeout_decorator.py @@ -7,7 +7,7 @@ class TimeoutException(MetaflowException): - headline = '@timeout' + headline = "@timeout" class TimeoutDecorator(StepDecorator): @@ -41,10 +41,9 @@ def myStep(self): minutes_between_retries : int Number of minutes between retries """ - name = 'timeout' - defaults = {'seconds': 0, - 'minutes': 0, - 'hours': 0} + + name = "timeout" + defaults = {"seconds": 0, "minutes": 0, "hours": 0} def __init__(self, *args, **kwargs): super(TimeoutDecorator, self).__init__(*args, **kwargs) @@ -52,54 +51,61 @@ def __init__(self, *args, **kwargs): # value without worrying about decorator order. # Convert values in attributes to type:int since they can be type:str # when passed using the CLI option --with. - self.secs = int(self.attributes['hours']) * 3600 +\ - int(self.attributes['minutes']) * 60 +\ - int(self.attributes['seconds']) + self.secs = ( + int(self.attributes["hours"]) * 3600 + + int(self.attributes["minutes"]) * 60 + + int(self.attributes["seconds"]) + ) def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger): self.logger = logger if not self.secs: - raise MetaflowException('Specify a duration for @timeout.') - - def task_pre_step(self, - step_name, - task_datastore, - metadata, - run_id, - task_id, - flow, - graph, - retry_count, - max_user_code_retries, - ubf_context, - inputs): + raise MetaflowException("Specify a duration for @timeout.") + + def task_pre_step( + self, + step_name, + task_datastore, + metadata, + run_id, + task_id, + flow, + graph, + retry_count, + max_user_code_retries, + ubf_context, + inputs, + ): if ubf_context != UBF_CONTROL and retry_count <= max_user_code_retries: # enable timeout only when executing user code self.step_name = step_name signal.signal(signal.SIGALRM, self._sigalrm_handler) signal.alarm(self.secs) - def task_post_step(self, - step_name, - flow, - graph, - retry_count, - max_user_code_retries): + def task_post_step( + self, step_name, flow, graph, retry_count, max_user_code_retries + ): signal.alarm(0) def _sigalrm_handler(self, signum, frame): def pretty_print_stack(): for line in traceback.format_stack(): - if 'timeout_decorators.py' not in line: + if "timeout_decorators.py" not in line: for part in line.splitlines(): - yield '> %s' % part - - msg = 'Step {step_name} timed out after {hours} hours, '\ - '{minutes} minutes, {seconds} seconds'\ - .format(step_name=self.step_name, **self.attributes) + yield "> %s" % part + + msg = ( + "Step {step_name} timed out after {hours} hours, " + "{minutes} minutes, {seconds} seconds".format( + step_name=self.step_name, **self.attributes + ) + ) self.logger(msg) - raise TimeoutException('%s\nStack when the timeout was raised:\n%s' - % (msg, '\n'.join(pretty_print_stack()))) + raise TimeoutException( + "%s\nStack when the timeout was raised:\n%s" + % (msg, "\n".join(pretty_print_stack())) + ) + def get_run_time_limit_for_task(step_decos): run_time_limit = 5 * 24 * 60 * 60 # 5 days. diff --git a/metaflow/procpoll.py b/metaflow/procpoll.py index 5b42f38f563..08358a47d20 100644 --- a/metaflow/procpoll.py +++ b/metaflow/procpoll.py @@ -1,14 +1,15 @@ import platform import select + class ProcPollEvent(object): def __init__(self, fd, can_read=False, is_terminated=False): self.fd = fd self.can_read = can_read self.is_terminated = is_terminated -class ProcPoll(object): +class ProcPoll(object): def poll(self): raise NotImplementedError() @@ -18,79 +19,80 @@ def add(self, fd): def remove(self, fd): raise NotImplementedError() -class LinuxProcPoll(ProcPoll): +class LinuxProcPoll(ProcPoll): def __init__(self): self._poll = select.poll() def add(self, fd): - self._poll.register(fd, select.POLLIN | - select.POLLERR | - select.POLLHUP) + self._poll.register(fd, select.POLLIN | select.POLLERR | select.POLLHUP) def remove(self, fd): self._poll.unregister(fd) def poll(self, timeout): for (fd, event) in self._poll.poll(timeout): - yield ProcPollEvent(fd=fd, - can_read=bool(event & select.POLLIN), - is_terminated=bool(event & select.POLLHUP) or - bool(event & select.POLLERR)) + yield ProcPollEvent( + fd=fd, + can_read=bool(event & select.POLLIN), + is_terminated=bool(event & select.POLLHUP) + or bool(event & select.POLLERR), + ) class DarwinProcPoll(ProcPoll): - def __init__(self): self._kq = select.kqueue() def add(self, fd): - ev = select.kevent(fd, - filter=select.KQ_FILTER_READ, - flags=select.KQ_EV_ADD) + ev = select.kevent(fd, filter=select.KQ_FILTER_READ, flags=select.KQ_EV_ADD) self._kq.control([ev], 0, 0) def remove(self, fd): - ev = select.kevent(fd, - flags=select.KQ_EV_DELETE) + ev = select.kevent(fd, flags=select.KQ_EV_DELETE) self._kq.control([ev], 0, 0) def poll(self, timeout): for event in self._kq.control(None, 100, timeout): - yield ProcPollEvent(fd=event.ident, - can_read=True, - is_terminated=event.flags & select.KQ_EV_EOF) + yield ProcPollEvent( + fd=event.ident, + can_read=True, + is_terminated=event.flags & select.KQ_EV_EOF, + ) + def make_poll(): os = platform.system() - if os == 'Linux': + if os == "Linux": return LinuxProcPoll() - elif os == 'Darwin': + elif os == "Darwin": return DarwinProcPoll() else: - raise Exception("Polling is not supported on " - "your operating system (%s)" % os) + raise Exception( + "Polling is not supported on " "your operating system (%s)" % os + ) -if __name__ == '__main__': + +if __name__ == "__main__": import subprocess - p1 = subprocess.Popen(['bash', '-c', - 'for ((i=0;i<10;i++)); ' - 'do echo "first $i"; sleep 1; done'], - bufsize=1, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - p2 = subprocess.Popen(['bash', '-c', - 'for ((i=0;i<5;i++)); ' - 'do echo "second $i"; sleep 2; done'], - bufsize=1, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - - fds = {p1.stdout.fileno(): ('p1', p1.stdout), - p2.stdout.fileno(): ('p2', p2.stdout)} + + p1 = subprocess.Popen( + ["bash", "-c", "for ((i=0;i<10;i++)); " 'do echo "first $i"; sleep 1; done'], + bufsize=1, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + p2 = subprocess.Popen( + ["bash", "-c", "for ((i=0;i<5;i++)); " 'do echo "second $i"; sleep 2; done'], + bufsize=1, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + fds = {p1.stdout.fileno(): ("p1", p1.stdout), p2.stdout.fileno(): ("p2", p2.stdout)} poll = make_poll() - print('poller is %s' % poll) + print("poller is %s" % poll) for fd in fds: poll.add(fd) @@ -99,8 +101,8 @@ def make_poll(): while n > 0: for event in poll.poll(0.5): name, fileobj = fds[event.fd] - print('[%s] %s' % (name, fileobj.readline().strip())) + print("[%s] %s" % (name, fileobj.readline().strip())) if event.is_terminated: - print('[%s] terminated' % name) + print("[%s] terminated" % name) poll.remove(event.fd) n -= 1 diff --git a/metaflow/pylint_wrapper.py b/metaflow/pylint_wrapper.py index f56e659c692..158a57009b0 100644 --- a/metaflow/pylint_wrapper.py +++ b/metaflow/pylint_wrapper.py @@ -7,15 +7,17 @@ from .exception import MetaflowException + class PyLintWarn(MetaflowException): - headline="Pylint is not happy" + headline = "Pylint is not happy" -class PyLint(object): +class PyLint(object): def __init__(self, fname): self._fname = fname try: from pylint.lint import Run + self._run = Run except: self._run = None @@ -26,7 +28,7 @@ def has_pylint(self): def run(self, logger=None, warnings=False, pylint_config=[]): args = [self._fname] if not warnings: - args.append('--errors-only') + args.append("--errors-only") if pylint_config: args.extend(pylint_config) stdout = sys.stdout @@ -50,14 +52,14 @@ def run(self, logger=None, warnings=False, pylint_config=[]): warnings = True if warnings: - raise PyLintWarn('*Fix Pylint warnings listed above or say --no-pylint.*') + raise PyLintWarn("*Fix Pylint warnings listed above or say --no-pylint.*") return pylint_is_happy, pylint_exception_msg def _filter_lines(self, output): for line in output.splitlines(): # Ignore headers - if '***' in line: + if "***" in line: continue # Ignore complaints about decorators missing in the metaflow module. # Automatic generation of decorators confuses Pylint. diff --git a/metaflow/runtime.py b/metaflow/runtime.py index f02e1cbf9cf..a2cdc0e121e 100644 --- a/metaflow/runtime.py +++ b/metaflow/runtime.py @@ -18,9 +18,11 @@ from . import get_namespace from .metaflow_config import MAX_ATTEMPTS -from .exception import MetaflowException,\ - MetaflowInternalError,\ - METAFLOW_EXIT_DISALLOW_RETRY +from .exception import ( + MetaflowException, + MetaflowInternalError, + METAFLOW_EXIT_DISALLOW_RETRY, +) from . import procpoll from .datastore import TaskDataStoreSet from .datastore.exceptions import DataException @@ -31,14 +33,14 @@ from .util import to_unicode, compress_list, unicode_type from .unbounded_foreach import CONTROL_TASK_TAG, UBF_CONTROL, UBF_TASK -MAX_WORKERS=16 -MAX_NUM_SPLITS=100 -MAX_LOG_SIZE=1024*1024 -PROGRESS_INTERVAL = 1000 #ms +MAX_WORKERS = 16 +MAX_NUM_SPLITS = 100 +MAX_LOG_SIZE = 1024 * 1024 +PROGRESS_INTERVAL = 1000 # ms # The following is a list of the (data) artifacts used by the runtime while # executing a flow. These are prefetched during the resume operation by # leveraging the TaskDataStoreSet. -PREFETCH_DATA_ARTIFACTS = ['_foreach_stack', '_task_ok', '_transition'] +PREFETCH_DATA_ARTIFACTS = ["_foreach_stack", "_task_ok", "_transition"] # Runtime must use logsource=RUNTIME_LOG_SOURCE for all loglines that it # formats according to mflog. See a comment in mflog.__init__ @@ -46,25 +48,27 @@ # TODO option: output dot graph periodically about execution -class NativeRuntime(object): - def __init__(self, - flow, - graph, - flow_datastore, - metadata, - environment, - package, - logger, - entrypoint, - event_logger, - monitor, - run_id=None, - clone_run_id=None, - clone_steps=None, - max_workers=MAX_WORKERS, - max_num_splits=MAX_NUM_SPLITS, - max_log_size=MAX_LOG_SIZE): +class NativeRuntime(object): + def __init__( + self, + flow, + graph, + flow_datastore, + metadata, + environment, + package, + logger, + entrypoint, + event_logger, + monitor, + run_id=None, + clone_run_id=None, + clone_steps=None, + max_workers=MAX_WORKERS, + max_num_splits=MAX_NUM_SPLITS, + max_log_size=MAX_LOG_SIZE, + ): if run_id is None: self._run_id = metadata.new_run_id() @@ -112,12 +116,13 @@ def __init__(self, # access the relevant data from cache (instead of going to the datastore # after the first prefetch). logger( - 'Gathering required information to resume run (this may take a bit of time)...') - self._origin_ds_set = \ - TaskDataStoreSet( - flow_datastore, - clone_run_id, - prefetch_data_artifacts=PREFETCH_DATA_ARTIFACTS) + "Gathering required information to resume run (this may take a bit of time)..." + ) + self._origin_ds_set = TaskDataStoreSet( + flow_datastore, + clone_run_id, + prefetch_data_artifacts=PREFETCH_DATA_ARTIFACTS, + ) self._run_queue = [] self._poll = procpoll.make_poll() self._workers = {} # fd -> subprocess mapping @@ -131,10 +136,7 @@ def __init__(self, for step in flow: for deco in step.decorators: - deco.runtime_init(flow, - graph, - package, - self._run_id) + deco.runtime_init(flow, graph, package, self._run_id) def _new_task(self, step, input_paths=None, **kwargs): @@ -146,34 +148,36 @@ def _new_task(self, step, input_paths=None, **kwargs): if step in self._clone_steps: may_clone = False - if step == '_parameters': + if step == "_parameters": decos = [] else: decos = getattr(self._flow, step).decorators - return Task(self._flow_datastore, - self._flow, - step, - self._run_id, - self._metadata, - self._environment, - self._entrypoint, - self.event_logger, - self._monitor, - input_paths=input_paths, - may_clone=may_clone, - clone_run_id=self._clone_run_id, - origin_ds_set=self._origin_ds_set, - decos=decos, - logger=self._logger, - **kwargs) + return Task( + self._flow_datastore, + self._flow, + step, + self._run_id, + self._metadata, + self._environment, + self._entrypoint, + self.event_logger, + self._monitor, + input_paths=input_paths, + may_clone=may_clone, + clone_run_id=self._clone_run_id, + origin_ds_set=self._origin_ds_set, + decos=decos, + logger=self._logger, + **kwargs + ) @property def run_id(self): return self._run_id def persist_parameters(self, task_id=None): - task = self._new_task('_parameters', task_id=task_id) + task = self._new_task("_parameters", task_id=task_id) if not task.is_cloned: task.persist(self._flow) self._params_task = task.path @@ -181,22 +185,20 @@ def persist_parameters(self, task_id=None): def execute(self): - self._logger('Workflow starting (run-id %s):' % self._run_id, - system_msg=True) + self._logger("Workflow starting (run-id %s):" % self._run_id, system_msg=True) self._metadata.start_run_heartbeat(self._flow.name, self._run_id) if self._params_task: - self._queue_push('start', {'input_paths': [self._params_task]}) + self._queue_push("start", {"input_paths": [self._params_task]}) else: - self._queue_push('start', {}) + self._queue_push("start", {}) progress_tstamp = time.time() try: # main scheduling loop exception = None - while self._run_queue or\ - self._num_active_workers > 0: + while self._run_queue or self._num_active_workers > 0: # 1. are any of the current workers finished? finished_tasks = list(self._poll_workers()) @@ -208,23 +210,23 @@ def execute(self): if time.time() - progress_tstamp > PROGRESS_INTERVAL: progress_tstamp = time.time() - msg = "%d tasks are running: %s." %\ - (self._num_active_workers, 'e.g. ...') # TODO + msg = "%d tasks are running: %s." % ( + self._num_active_workers, + "e.g. ...", + ) # TODO self._logger(msg, system_msg=True) - msg = "%d tasks are waiting in the queue." %\ - len(self._run_queue) + msg = "%d tasks are waiting in the queue." % len(self._run_queue) self._logger(msg, system_msg=True) - msg = "%d steps are pending: %s." %\ - (0, 'e.g. ...') # TODO + msg = "%d steps are pending: %s." % (0, "e.g. ...") # TODO self._logger(msg, system_msg=True) except KeyboardInterrupt as ex: - self._logger('Workflow interrupted.', system_msg=True, bad=True) + self._logger("Workflow interrupted.", system_msg=True, bad=True) self._killall() exception = ex raise except Exception as ex: - self._logger('Workflow failed.', system_msg=True, bad=True) + self._logger("Workflow failed.", system_msg=True, bad=True) self._killall() exception = ex raise @@ -237,30 +239,37 @@ def execute(self): self._metadata.stop_heartbeat() # assert that end was executed and it was successful - if ('end', ()) in self._finished: - self._logger('Done!', system_msg=True) + if ("end", ()) in self._finished: + self._logger("Done!", system_msg=True) else: - raise MetaflowInternalError('The *end* step was not successful ' - 'by the end of flow.') + raise MetaflowInternalError( + "The *end* step was not successful " "by the end of flow." + ) def _killall(self): # If we are here, all children have received a signal and are shutting down. # We want to give them an opportunity to do so and then kill live_workers = set(self._workers.values()) now = int(time.time()) - self._logger('Terminating %d active tasks...' % len(live_workers), - system_msg=True, bad=True) + self._logger( + "Terminating %d active tasks..." % len(live_workers), + system_msg=True, + bad=True, + ) while live_workers and int(time.time()) - now < 5: # While not all workers are dead and we have waited less than 5 seconds live_workers = [worker for worker in live_workers if not worker.clean()] if live_workers: - self._logger('Killing %d remaining tasks after having waited for %d seconds -- ' - 'some tasks may not exit cleanly' % (len(live_workers), - int(time.time()) - now), - system_msg=True, bad=True) + self._logger( + "Killing %d remaining tasks after having waited for %d seconds -- " + "some tasks may not exit cleanly" + % (len(live_workers), int(time.time()) - now), + system_msg=True, + bad=True, + ) for worker in live_workers: worker.kill() - self._logger('Flushing logs...', system_msg=True, bad=True) + self._logger("Flushing logs...", system_msg=True, bad=True) # give killed workers a chance to flush their logs to datastore for _ in range(3): list(self._poll_workers()) @@ -280,49 +289,64 @@ def _queue_task_join(self, task, next_steps): # CHECK: this condition should be enforced by the linter but # let's assert that the assumption holds if len(next_steps) > 1: - msg = 'Step *{step}* transitions to a join and another '\ - 'step. The join must be the only transition.' + msg = ( + "Step *{step}* transitions to a join and another " + "step. The join must be the only transition." + ) raise MetaflowInternalError(task, msg.format(step=task.step)) else: next_step = next_steps[0] - - unbounded_foreach = not task.results.is_none('_unbounded_foreach') + unbounded_foreach = not task.results.is_none("_unbounded_foreach") if unbounded_foreach: # Before we queue the join, do some post-processing of runtime state # (_finished, _is_cloned) for the (sibling) mapper tasks. # Update state of (sibling) mapper tasks for control task. if task.ubf_context == UBF_CONTROL: - mapper_tasks = task.results.get('_control_mapper_tasks') + mapper_tasks = task.results.get("_control_mapper_tasks") if not mapper_tasks: - msg = "Step *{step}* has a control task which didn't "\ - "specify the artifact *_control_mapper_tasks* for "\ - "the subsequent *{join}* step." - raise MetaflowInternalError(msg.format(step=task.step, - join=next_steps[0])) - elif not (isinstance(mapper_tasks, list) and\ - isinstance(mapper_tasks[0], unicode_type)): - msg = "Step *{step}* has a control task which didn't "\ - "specify the artifact *_control_mapper_tasks* as a "\ - "list of strings but instead specified it as {typ} "\ - "with elements of {elem_typ}." + msg = ( + "Step *{step}* has a control task which didn't " + "specify the artifact *_control_mapper_tasks* for " + "the subsequent *{join}* step." + ) raise MetaflowInternalError( - msg.format(step=task.step, - typ=type(mapper_tasks), - elem_type=type(mapper_tasks[0]))) + msg.format(step=task.step, join=next_steps[0]) + ) + elif not ( + isinstance(mapper_tasks, list) + and isinstance(mapper_tasks[0], unicode_type) + ): + msg = ( + "Step *{step}* has a control task which didn't " + "specify the artifact *_control_mapper_tasks* as a " + "list of strings but instead specified it as {typ} " + "with elements of {elem_typ}." + ) + raise MetaflowInternalError( + msg.format( + step=task.step, + typ=type(mapper_tasks), + elem_type=type(mapper_tasks[0]), + ) + ) num_splits = len(mapper_tasks) self._control_num_splits[task.path] = num_splits if task.is_cloned: # Add mapper tasks to be cloned. for i in range(num_splits): - # NOTE: For improved robustness, introduce - # `clone_options` as an enum so that we can force that + # NOTE: For improved robustness, introduce + # `clone_options` as an enum so that we can force that # clone must occur for this task. - self._queue_push(task.step, - {'input_paths': task.input_paths, - 'split_index': str(i), - 'ubf_context': UBF_TASK}) + self._queue_push( + task.step, + { + "input_paths": task.input_paths, + "split_index": str(i), + "ubf_context": UBF_TASK, + }, + ) else: # Update _finished since these tasks were successfully # run elsewhere so that join will be unblocked. @@ -354,16 +378,16 @@ def _queue_task_join(self, task, next_steps): if all(required_tasks): # all tasks to be joined are ready. Schedule the next join step. - self._queue_push(next_step, - {'input_paths': required_tasks, - 'join_type': 'foreach'}) + self._queue_push( + next_step, + {"input_paths": required_tasks, "join_type": "foreach"}, + ) else: # matching_split is the split-parent of the finished task - matching_split = \ - self._graph[self._graph[next_step].split_parents[-1]] + matching_split = self._graph[self._graph[next_step].split_parents[-1]] step_name, foreach_stack = task.finished_id - if matching_split.type == 'foreach': + if matching_split.type == "foreach": # next step is a foreach join def siblings(foreach_stack): @@ -373,57 +397,67 @@ def siblings(foreach_stack): yield tuple(bottom + [top._replace(index=index)]) # required tasks are all split-siblings of the finished task - required_tasks = [self._finished.get((task.step, s)) - for s in siblings(foreach_stack)] - join_type = 'foreach' + required_tasks = [ + self._finished.get((task.step, s)) for s in siblings(foreach_stack) + ] + join_type = "foreach" else: # next step is a split-and # required tasks are all branches joined by the next step - required_tasks = [self._finished.get((step, foreach_stack)) - for step in self._graph[next_step].in_funcs] - join_type = 'linear' + required_tasks = [ + self._finished.get((step, foreach_stack)) + for step in self._graph[next_step].in_funcs + ] + join_type = "linear" if all(required_tasks): # all tasks to be joined are ready. Schedule the next join step. - self._queue_push(next_step, - {'input_paths': required_tasks, - 'join_type': join_type}) + self._queue_push( + next_step, {"input_paths": required_tasks, "join_type": join_type} + ) def _queue_task_foreach(self, task, next_steps): # CHECK: this condition should be enforced by the linter but # let's assert that the assumption holds if len(next_steps) > 1: - msg = 'Step *{step}* makes a foreach split but it defines '\ - 'multiple transitions. Specify only one transition '\ - 'for foreach.' + msg = ( + "Step *{step}* makes a foreach split but it defines " + "multiple transitions. Specify only one transition " + "for foreach." + ) raise MetaflowInternalError(msg.format(step=task.step)) else: next_step = next_steps[0] - unbounded_foreach = not task.results.is_none('_unbounded_foreach') + unbounded_foreach = not task.results.is_none("_unbounded_foreach") if unbounded_foreach: # Need to push control process related task. - self._queue_push(next_step, - {'input_paths': [task.path], - 'ubf_context': UBF_CONTROL}) + self._queue_push( + next_step, {"input_paths": [task.path], "ubf_context": UBF_CONTROL} + ) else: - num_splits = task.results['_foreach_num_splits'] + num_splits = task.results["_foreach_num_splits"] if num_splits > self._max_num_splits: - msg = 'Foreach in step *{step}* yielded {num} child steps '\ - 'which is more than the current maximum of {max} '\ - 'children. You can raise the maximum with the '\ - '--max-num-splits option. ' - raise TaskFailed(task, msg.format(step=task.step, - num=num_splits, - max=self._max_num_splits)) + msg = ( + "Foreach in step *{step}* yielded {num} child steps " + "which is more than the current maximum of {max} " + "children. You can raise the maximum with the " + "--max-num-splits option. " + ) + raise TaskFailed( + task, + msg.format( + step=task.step, num=num_splits, max=self._max_num_splits + ), + ) # schedule all splits for i in range(num_splits): - self._queue_push(next_step, - {'split_index': str(i), - 'input_paths': [task.path]}) + self._queue_push( + next_step, {"split_index": str(i), "input_paths": [task.path]} + ) def _queue_tasks(self, finished_tasks): # finished tasks include only successful tasks @@ -435,7 +469,7 @@ def _queue_tasks(self, finished_tasks): # statically inferred transitions. Make an exception for control # tasks, where we just rely on static analysis since we don't # execute user code. - trans = task.results.get('_transition') + trans = task.results.get("_transition") if trans: next_steps = trans[0] foreach = trans[1] @@ -444,19 +478,24 @@ def _queue_tasks(self, finished_tasks): foreach = None expected = self._graph[task.step].out_funcs if next_steps != expected: - msg = 'Based on static analysis of the code, step *{step}* '\ - 'was expected to transition to step(s) *{expected}*. '\ - 'However, when the code was executed, self.next() was '\ - 'called with *{actual}*. Make sure there is only one '\ - 'unconditional self.next() call in the end of your '\ - 'step. ' - raise MetaflowInternalError(msg.format(step=task.step, - expected=', '.join( - expected), - actual=', '.join(next_steps))) + msg = ( + "Based on static analysis of the code, step *{step}* " + "was expected to transition to step(s) *{expected}*. " + "However, when the code was executed, self.next() was " + "called with *{actual}*. Make sure there is only one " + "unconditional self.next() call in the end of your " + "step. " + ) + raise MetaflowInternalError( + msg.format( + step=task.step, + expected=", ".join(expected), + actual=", ".join(next_steps), + ) + ) # Different transition types require different treatment - if any(self._graph[f].type == 'join' for f in next_steps): + if any(self._graph[f].type == "join" for f in next_steps): # Next step is a join self._queue_task_join(task, next_steps) elif foreach: @@ -465,7 +504,7 @@ def _queue_tasks(self, finished_tasks): else: # Next steps are normal linear steps for step in next_steps: - self._queue_push(step, {'input_paths': [task.path]}) + self._queue_push(step, {"input_paths": [task.path]}) def _poll_workers(self): if self._workers: @@ -485,13 +524,19 @@ def _poll_workers(self): task = worker.task if returncode: # worker did not finish successfully - if worker.cleaned or \ - returncode == METAFLOW_EXIT_DISALLOW_RETRY: - self._logger("This failed task will not be " - "retried.", system_msg=True) + if ( + worker.cleaned + or returncode == METAFLOW_EXIT_DISALLOW_RETRY + ): + self._logger( + "This failed task will not be " "retried.", + system_msg=True, + ) else: - if task.retries < task.user_code_retries +\ - task.error_retries: + if ( + task.retries + < task.user_code_retries + task.error_retries + ): self._retry_worker(worker) else: raise TaskFailed(task) @@ -514,9 +559,10 @@ def _retry_worker(self, worker): # any results with an attempt ID >= MAX_ATTEMPTS will be ignored # by datastore, so running a task with such a retry_could would # be pointless and dangerous - raise MetaflowInternalError("Too many task attempts (%d)! " - "MAX_ATTEMPTS exceeded." - % worker.task.retries) + raise MetaflowInternalError( + "Too many task attempts (%d)! " + "MAX_ATTEMPTS exceeded." % worker.task.retries + ) worker.task.new_attempt() self._launch_worker(worker.task) @@ -532,44 +578,47 @@ def _launch_worker(self, task): class Task(object): clone_pathspec_mapping = {} - def __init__(self, - flow_datastore, - flow, - step, - run_id, - metadata, - environment, - entrypoint, - event_logger, - monitor, - input_paths=None, - split_index=None, - ubf_context=None, - clone_run_id=None, - origin_ds_set=None, - may_clone=False, - join_type=None, - logger=None, - task_id=None, - decos=[]): + def __init__( + self, + flow_datastore, + flow, + step, + run_id, + metadata, + environment, + entrypoint, + event_logger, + monitor, + input_paths=None, + split_index=None, + ubf_context=None, + clone_run_id=None, + origin_ds_set=None, + may_clone=False, + join_type=None, + logger=None, + task_id=None, + decos=[], + ): if ubf_context == UBF_CONTROL: [input_path] = input_paths - run, input_step, input_task = input_path.split('/') + run, input_step, input_task = input_path.split("/") # We associate the control task-id to be 1:1 with the split node # where the unbounded-foreach was defined. # We prefer encoding the corresponding split into the task_id of # the control node; so it has access to this information quite # easily. There is anyway a corresponding int id stored in the # metadata backend - so this should be fine. - task_id = 'control-%s-%s-%s' % (run, input_step, input_task) + task_id = "control-%s-%s-%s" % (run, input_step, input_task) # Register only regular Metaflow (non control) tasks. if task_id is None: task_id = str(metadata.new_task_id(run_id, step)) else: # task_id is preset only by persist_parameters() or control tasks. if ubf_context == UBF_CONTROL: - metadata.register_task_id(run_id, step, task_id, 0, - sys_tags=[CONTROL_TASK_TAG]) + metadata.register_task_id( + run_id, step, task_id, 0, sys_tags=[CONTROL_TASK_TAG] + ) else: metadata.register_task_id(run_id, step, task_id, 0) @@ -592,7 +641,7 @@ def __init__(self, self.monitor = monitor self._logger = logger - self._path = '%s/%s/%s' % (self.run_id, self.step, self.task_id) + self._path = "%s/%s/%s" % (self.run_id, self.step, self.task_id) self.retries = 0 self.user_code_retries = 0 @@ -618,29 +667,34 @@ def __init__(self, self.new_attempt() for deco in decos: - deco.runtime_task_created(self._ds, - task_id, - split_index, - input_paths, - self._is_cloned, - ubf_context) + deco.runtime_task_created( + self._ds, + task_id, + split_index, + input_paths, + self._is_cloned, + ubf_context, + ) # determine the number of retries of this task user_code_retries, error_retries = deco.step_task_retry_count() if user_code_retries is None and error_retries is None: # This signals the runtime that the task doesn't want any # retries indifferent to other decorator opinions. - # NOTE: This is needed since we don't statically disallow + # NOTE: This is needed since we don't statically disallow # specifying `@retry` in combination with decorators which # implement `unbounded_foreach` semantics. This allows for # ergonomic user invocation of `--with retry`; instead # choosing to specially handle this way in the runtime. self.user_code_retries = None self.error_retries = None - if self.user_code_retries is not None and \ - self.error_retries is not None: - self.user_code_retries = max(self.user_code_retries, - user_code_retries) + if ( + self.user_code_retries is not None + and self.error_retries is not None + ): + self.user_code_retries = max( + self.user_code_retries, user_code_retries + ) self.error_retries = max(self.error_retries, error_retries) if self.user_code_retries is None and self.error_retries is None: self.user_code_retries = 0 @@ -648,72 +702,73 @@ def __init__(self, def new_attempt(self): self._ds = self._flow_datastore.get_task_datastore( - self.run_id, self.step, self.task_id, attempt=self.retries, mode='w') + self.run_id, self.step, self.task_id, attempt=self.retries, mode="w" + ) self._ds.init_task() def log(self, msg, system_msg=False, pid=None, timestamp=True): if pid: - prefix = '[%s (pid %s)] ' % (self._path, pid) + prefix = "[%s (pid %s)] " % (self._path, pid) else: - prefix = '[%s] ' % self._path + prefix = "[%s] " % self._path - self._logger(msg, - head=prefix, - system_msg=system_msg, - timestamp=timestamp) + self._logger(msg, head=prefix, system_msg=system_msg, timestamp=timestamp) sys.stdout.flush() def _find_origin_task(self, clone_run_id, join_type): - if self.step == '_parameters': - pathspec = '%s/_parameters[]' % clone_run_id + if self.step == "_parameters": + pathspec = "%s/_parameters[]" % clone_run_id origin = self.origin_ds_set.get_with_pathspec_index(pathspec) if origin is None: # This is just for usability: We could rerun the whole flow # if an unknown clone_run_id is provided but probably this is # not what the user intended, so raise a warning - raise MetaflowException("Resume could not find run id *%s*" % - clone_run_id) + raise MetaflowException( + "Resume could not find run id *%s*" % clone_run_id + ) else: return origin else: # all inputs must have the same foreach stack, so we can safely # pick the first one parent_pathspec = self.input_paths[0] - origin_parent_pathspec = \ - self.clone_pathspec_mapping[parent_pathspec] + origin_parent_pathspec = self.clone_pathspec_mapping[parent_pathspec] parent = self.origin_ds_set.get_with_pathspec(origin_parent_pathspec) # Parent should be non-None since only clone the child if the parent # was successfully cloned. - foreach_stack = parent['_foreach_stack'] - if join_type == 'foreach': + foreach_stack = parent["_foreach_stack"] + if join_type == "foreach": # foreach-join pops the topmost index - index = ','.join(str(s.index) for s in foreach_stack[:-1]) + index = ",".join(str(s.index) for s in foreach_stack[:-1]) elif self.split_index or self.ubf_context == UBF_CONTROL: # foreach-split pushes a new index - index = ','.join([str(s.index) for s in foreach_stack] + - [str(self.split_index)]) + index = ",".join( + [str(s.index) for s in foreach_stack] + [str(self.split_index)] + ) else: # all other transitions keep the parent's foreach stack intact - index = ','.join(str(s.index) for s in foreach_stack) - pathspec = '%s/%s[%s]' % (clone_run_id, self.step, index) + index = ",".join(str(s.index) for s in foreach_stack) + pathspec = "%s/%s[%s]" % (clone_run_id, self.step, index) return self.origin_ds_set.get_with_pathspec_index(pathspec) def _attempt_clone(self, clone_run_id, join_type): origin = self._find_origin_task(clone_run_id, join_type) - if origin and origin['_task_ok']: + if origin and origin["_task_ok"]: # Store the mapping from current_pathspec -> origin_pathspec which # will be useful for looking up origin_ds_set in find_origin_task. self.clone_pathspec_mapping[self._path] = origin.pathspec - if self.step == '_parameters': + if self.step == "_parameters": # Clone in place without relying on run_queue. self.new_attempt() self._ds.clone(origin) self._ds.done() else: - self.log("Cloning results of a previously run task %s" - % origin.pathspec, system_msg=True) + self.log( + "Cloning results of a previously run task %s" % origin.pathspec, + system_msg=True, + ) # Store the origin pathspec in clone_origin so this can be run # as a task by the runtime. self.clone_origin = origin.pathspec @@ -733,13 +788,14 @@ def results(self): return self._results_ds else: self._results_ds = self._flow_datastore.get_task_datastore( - self.run_id, self.step, self.task_id) + self.run_id, self.step, self.task_id + ) return self._results_ds @property def finished_id(self): # note: id is not available before the task has finished - return (self.step, tuple(self.results['_foreach_stack'])) + return (self.step, tuple(self.results["_foreach_stack"])) @property def is_cloned(self): @@ -759,19 +815,18 @@ def save_metadata(self, name, metadata): self._ds.save_metadata({name: metadata}) def __str__(self): - return ' '.join(self._args) + return " ".join(self._args) class TaskFailed(MetaflowException): headline = "Step failure" - def __init__(self, task, msg=''): - body = "Step *%s* (task-id %s) failed" % (task.step, - task.task_id) + def __init__(self, task, msg=""): + body = "Step *%s* (task-id %s) failed" % (task.step, task.task_id) if msg: - body = '%s: %s' % (body, msg) + body = "%s: %s" % (body, msg) else: - body += '.' + body += "." super(TaskFailed, self).__init__(body) @@ -792,7 +847,7 @@ def write(self, bytedata, system_msg=False): self._buffer.write(bytedata) self._size += len(bytedata) else: - msg = b'[TRUNCATED - MAXIMUM LOG FILE SIZE REACHED]\n' + msg = b"[TRUNCATED - MAXIMUM LOG FILE SIZE REACHED]\n" self._buffer.write(mflog_msg(msg)) self._eof = True @@ -814,16 +869,19 @@ def __init__(self, task): self.task = task self.entrypoint = list(task.entrypoint) self.top_level_options = { - 'quiet': True, - 'coverage': 'coverage' in sys.modules, - 'metadata': self.task.metadata_type, - 'environment': self.task.environment_type, - 'datastore': self.task.datastore_type, - 'event-logger': self.task.event_logger_type, - 'monitor': self.task.monitor_type, - 'datastore-root': self.task.datastore_sysroot, - 'with': [deco.make_decorator_spec() for deco in self.task.decos - if not deco.statically_defined] + "quiet": True, + "coverage": "coverage" in sys.modules, + "metadata": self.task.metadata_type, + "environment": self.task.environment_type, + "datastore": self.task.datastore_type, + "event-logger": self.task.event_logger_type, + "monitor": self.task.monitor_type, + "datastore-root": self.task.datastore_sysroot, + "with": [ + deco.make_decorator_spec() + for deco in self.task.decos + if not deco.statically_defined + ], } # FlowDecorators can define their own top-level options. They are @@ -832,18 +890,18 @@ def __init__(self, task): for deco in flow_decorators(): self.top_level_options.update(deco.get_top_level_options()) - self.commands = ['step'] + self.commands = ["step"] self.command_args = [self.task.step] self.command_options = { - 'run-id': task.run_id, - 'task-id': task.task_id, - 'input-paths': compress_list(task.input_paths), - 'split-index': task.split_index, - 'retry-count': task.retries, - 'max-user-code-retries': task.user_code_retries, - 'tag': task.tags, - 'namespace': get_namespace() or '', - 'ubf-context': task.ubf_context + "run-id": task.run_id, + "task-id": task.task_id, + "input-paths": compress_list(task.input_paths), + "split-index": task.split_index, + "retry-count": task.retries, + "max-user-code-retries": task.user_code_retries, + "tag": task.tags, + "namespace": get_namespace() or "", + "ubf-context": task.ubf_context, } self.env = {} @@ -855,12 +913,12 @@ def _options(mapping): if v: # we need special handling for 'with' since it is a reserved # keyword in Python, so we call it 'decospecs' in click args - if k == 'decospecs': - k = 'with' - k = k.replace('_', '-') + if k == "decospecs": + k = "with" + k = k.replace("_", "-") v = v if isinstance(v, (list, tuple, set)) else [v] for value in v: - yield '--%s' % k + yield "--%s" % k if not isinstance(value, bool): yield to_unicode(value) @@ -875,73 +933,78 @@ def get_env(self): return self.env def __str__(self): - return ' '.join(self.get_args()) + return " ".join(self.get_args()) class Worker(object): - def __init__(self, task, max_logs_size): self.task = task self._proc = self._launch() if task.retries > task.user_code_retries: - self.task.log('Task fallback is starting to handle the failure.', - system_msg=True, - pid=self._proc.pid) + self.task.log( + "Task fallback is starting to handle the failure.", + system_msg=True, + pid=self._proc.pid, + ) elif not task.is_cloned: - suffix = ' (retry).' if task.retries else '.' - self.task.log('Task is starting' + suffix, - system_msg=True, - pid=self._proc.pid) + suffix = " (retry)." if task.retries else "." + self.task.log( + "Task is starting" + suffix, system_msg=True, pid=self._proc.pid + ) - self._stdout = TruncatedBuffer('stdout', max_logs_size) - self._stderr = TruncatedBuffer('stderr', max_logs_size) + self._stdout = TruncatedBuffer("stdout", max_logs_size) + self._stderr = TruncatedBuffer("stderr", max_logs_size) - self._logs = {self._proc.stderr.fileno(): (self._proc.stderr, - self._stderr), - self._proc.stdout.fileno(): (self._proc.stdout, - self._stdout)} + self._logs = { + self._proc.stderr.fileno(): (self._proc.stderr, self._stderr), + self._proc.stdout.fileno(): (self._proc.stdout, self._stdout), + } - self._encoding = sys.stdout.encoding or 'UTF-8' + self._encoding = sys.stdout.encoding or "UTF-8" self.killed = False # Killed indicates that the task was forcibly killed - # with SIGKILL by the master process. - # A killed task is always considered cleaned + # with SIGKILL by the master process. + # A killed task is always considered cleaned self.cleaned = False # A cleaned task is one that is shutting down and has been - # noticed by the runtime and queried for its state (whether or - # not is is properly shut down) + # noticed by the runtime and queried for its state (whether or + # not is is properly shut down) def _launch(self): args = CLIArgs(self.task) env = dict(os.environ) if self.task.clone_run_id: - args.command_options['clone-run-id'] = self.task.clone_run_id + args.command_options["clone-run-id"] = self.task.clone_run_id if self.task.is_cloned and self.task.clone_origin: - args.command_options['clone-only'] = self.task.clone_origin + args.command_options["clone-only"] = self.task.clone_origin # disabling atlas sidecar for cloned tasks due to perf reasons - args.top_level_options['monitor'] = 'nullSidecarMonitor' + args.top_level_options["monitor"] = "nullSidecarMonitor" else: # decorators may modify the CLIArgs object in-place for deco in self.task.decos: - deco.runtime_step_cli(args, - self.task.retries, - self.task.user_code_retries, - self.task.ubf_context) + deco.runtime_step_cli( + args, + self.task.retries, + self.task.user_code_retries, + self.task.ubf_context, + ) env.update(args.get_env()) - env['PYTHONUNBUFFERED'] = 'x' + env["PYTHONUNBUFFERED"] = "x" # NOTE bufsize=1 below enables line buffering which is required # by read_logline() below that relies on readline() not blocking # print('running', args) cmdline = args.get_args() debug.subcommand_exec(cmdline) - return subprocess.Popen(cmdline, - env=env, - bufsize=1, - stdin=subprocess.PIPE, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE) + return subprocess.Popen( + cmdline, + env=env, + bufsize=1, + stdin=subprocess.PIPE, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) def emit_log(self, msg, buf, system_msg=False): if mflog.is_structured(msg): @@ -970,11 +1033,13 @@ def emit_log(self, msg, buf, system_msg=False): # persisted, assuming that all previously formatted loglines have # been already persisted at the source. buf.write(mflog_msg(msg, now=timestamp), system_msg=system_msg) - text = plain.strip().decode(self._encoding, errors='replace') - self.task.log(text, - pid=self._proc.pid, - timestamp=mflog.utc_to_local(timestamp), - system_msg=system_msg) + text = plain.strip().decode(self._encoding, errors="replace") + self.task.log( + text, + pid=self._proc.pid, + timestamp=mflog.utc_to_local(timestamp), + system_msg=system_msg, + ) def read_logline(self, fd): fileobj, buf = self._logs[fd] @@ -988,15 +1053,14 @@ def read_logline(self, fd): return False def fds(self): - return (self._proc.stderr.fileno(), - self._proc.stdout.fileno()) + return (self._proc.stderr.fileno(), self._proc.stdout.fileno()) def clean(self): if self.killed: return True if not self.cleaned: for fileobj, buf in self._logs.values(): - msg = b'[KILLED BY ORCHESTRATOR]\n' + msg = b"[KILLED BY ORCHESTRATOR]\n" self.emit_log(msg, buf, system_msg=True) self.cleaned = True return self._proc.poll() is not None @@ -1035,33 +1099,43 @@ def terminate(self): # Return early if the task is cloned since we don't want to # perform any log collection. if not self.task.is_cloned: - self.task.save_metadata('runtime', {'return_code': returncode, - 'killed': self.killed, - 'success': returncode == 0}) + self.task.save_metadata( + "runtime", + { + "return_code": returncode, + "killed": self.killed, + "success": returncode == 0, + }, + ) if returncode: if not self.killed: if returncode == -11: - self.emit_log(b'Task failed with a segmentation fault.', - self._stderr, - system_msg=True) + self.emit_log( + b"Task failed with a segmentation fault.", + self._stderr, + system_msg=True, + ) else: - self.emit_log(b'Task failed.', - self._stderr, - system_msg=True) + self.emit_log(b"Task failed.", self._stderr, system_msg=True) else: - num = self.task.results['_foreach_num_splits'] + num = self.task.results["_foreach_num_splits"] if num: - self.task.log('Foreach yields %d child steps.' % num, - system_msg=True, - pid=self._proc.pid) - self.task.log('Task finished successfully.', - system_msg=True, - pid=self._proc.pid) - self.task.save_logs({ - 'stdout': self._stdout.get_buffer(), - 'stderr': self._stderr.get_buffer()}) + self.task.log( + "Foreach yields %d child steps." % num, + system_msg=True, + pid=self._proc.pid, + ) + self.task.log( + "Task finished successfully.", system_msg=True, pid=self._proc.pid + ) + self.task.save_logs( + { + "stdout": self._stdout.get_buffer(), + "stderr": self._stderr.get_buffer(), + } + ) return returncode def __str__(self): - return 'Worker[%d]: %s' % (self._proc.pid, self.task.path) + return "Worker[%d]: %s" % (self._proc.pid, self.task.path) diff --git a/metaflow/sidecar.py b/metaflow/sidecar.py index afebd600cf0..74a47f41c35 100644 --- a/metaflow/sidecar.py +++ b/metaflow/sidecar.py @@ -23,18 +23,22 @@ except: blockingError = OSError + class PipeUnavailableError(Exception): """raised when unable to write to pipe given allotted time""" + pass class NullSidecarError(Exception): """raised when trying to poll or interact with the fake subprocess in the null sidecar""" + pass class MsgTimeoutError(Exception): """raised when trying unable to send message to sidecar in allocated time""" + pass @@ -44,7 +48,6 @@ def poll(self, timeout): class SidecarSubProcess(object): - def __init__(self, worker_type): # type: (str) -> None self.__worker_type = worker_type @@ -54,24 +57,28 @@ def __init__(self, worker_type): def start(self): - if (self.__worker_type is not None and \ - self.__worker_type.startswith(NULL_SIDECAR_PREFIX)) or \ - (platform.system() == 'Darwin' and sys.version_info < (3, 0)): - # if on darwin and running python 2 disable sidecars - # there is a bug with importing poll from select in some cases - # - # TODO: Python 2 shipped by Anaconda allows for - # `from select import poll`. We can consider enabling sidecars - # for that distribution if needed at a later date. - self.__poller = NullPoller() + if ( + self.__worker_type is not None + and self.__worker_type.startswith(NULL_SIDECAR_PREFIX) + ) or (platform.system() == "Darwin" and sys.version_info < (3, 0)): + # if on darwin and running python 2 disable sidecars + # there is a bug with importing poll from select in some cases + # + # TODO: Python 2 shipped by Anaconda allows for + # `from select import poll`. We can consider enabling sidecars + # for that distribution if needed at a later date. + self.__poller = NullPoller() else: from select import poll + python_version = sys.executable - cmdline = [python_version, - '-u', - os.path.dirname(__file__) + '/sidecar_worker.py', - self.__worker_type] + cmdline = [ + python_version, + "-u", + os.path.dirname(__file__) + "/sidecar_worker.py", + self.__worker_type, + ] debug.sidecar_exec(cmdline) self.__process = self.__start_subprocess(cmdline) @@ -79,12 +86,12 @@ def start(self): if self.__process is not None: fcntl.fcntl(self.__process.stdin, F_SETFL, O_NONBLOCK) self.__poller = poll() - self.__poller.register(self.__process.stdin.fileno(), - select.POLLOUT) + self.__poller.register(self.__process.stdin.fileno(), select.POLLOUT) else: # unable to start subprocess, fallback to Null sidecar - self.logger("unable to start subprocess for sidecar %s" - % self.__worker_type) + self.logger( + "unable to start subprocess for sidecar %s" % self.__worker_type + ) self.__poller = NullPoller() def __start_subprocess(self, cmdline): @@ -92,10 +99,12 @@ def __start_subprocess(self, cmdline): try: # Set stdout=sys.stdout & stderr=sys.stderr # to print to console the output of sidecars. - return subprocess.Popen(cmdline, - stdin=subprocess.PIPE, - stdout=open(os.devnull, 'w'), - bufsize=0) + return subprocess.Popen( + cmdline, + stdin=subprocess.PIPE, + stdout=open(os.devnull, "w"), + bufsize=0, + ) except blockingError as be: self.logger("warning: sidecar popen failed: %s" % repr(be)) except Exception as e: @@ -110,7 +119,7 @@ def kill(self): pass def emit_msg(self, msg): - msg_ser = msg.serialize().encode('utf-8') + msg_ser = msg.serialize().encode("utf-8") written_bytes = 0 while written_bytes < len(msg_ser): try: @@ -119,7 +128,7 @@ def emit_msg(self, msg): raise MsgTimeoutError("poller timed out") for fd, event in fds: if event & select.POLLERR: - raise PipeUnavailableError('pipe unavailable') + raise PipeUnavailableError("pipe unavailable") f = os.write(fd, msg_ser[written_bytes:]) written_bytes += f except NullSidecarError: @@ -138,7 +147,7 @@ def msg_handler(self, msg, retries=3): self.start() if retries > 0: self.logger("retrying msg send to sidecar") - self.msg_handler(msg, retries-1) + self.msg_handler(msg, retries - 1) else: self.logger("error sending log message") self.logger(repr(ex)) diff --git a/metaflow/sidecar_messages.py b/metaflow/sidecar_messages.py index 7bee9426e70..a7b6d183c39 100644 --- a/metaflow/sidecar_messages.py +++ b/metaflow/sidecar_messages.py @@ -6,18 +6,19 @@ class MessageTypes(object): SHUTDOWN, LOG_EVENT = range(1, 3) -class Message(object): +class Message(object): def __init__(self, msg_type, payload): self.msg_type = msg_type self.payload = payload def serialize(self): msg = { - 'msg_type': self.msg_type, - 'payload': self.payload, + "msg_type": self.msg_type, + "payload": self.payload, } - return json.dumps(msg)+"\n" + return json.dumps(msg) + "\n" + def deserialize(json_msg): - return Message(**json.loads(json_msg)) \ No newline at end of file + return Message(**json.loads(json_msg)) diff --git a/metaflow/sidecar_worker.py b/metaflow/sidecar_worker.py index e31103018b8..09c81d5879b 100644 --- a/metaflow/sidecar_worker.py +++ b/metaflow/sidecar_worker.py @@ -17,6 +17,7 @@ class WorkershutdownError(Exception): """raised when terminating sidecar""" + pass @@ -44,7 +45,7 @@ def process_messages(worker): @click.command(help="Initialize workers") -@click.argument('worker-type') +@click.argument("worker-type") def main(worker_type): worker_process = SIDECARS.get(worker_type) diff --git a/metaflow/task.py b/metaflow/task.py index b49bf77b0aa..9a97c596d10 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -6,20 +6,18 @@ from .metaflow_config import MAX_ATTEMPTS from .metadata import MetaDatum from .datastore import Inputs, TaskDataStoreSet -from .exception import MetaflowInternalError,\ - MetaflowDataMissing,\ - MetaflowExceptionWrapper +from .exception import ( + MetaflowInternalError, + MetaflowDataMissing, + MetaflowExceptionWrapper, +) from .unbounded_foreach import UBF_CONTROL -from .util import all_equal,\ - get_username,\ - resolve_identity, \ - unicode_type +from .util import all_equal, get_username, resolve_identity, unicode_type from .current import current from collections import namedtuple -ForeachFrame = namedtuple('ForeachFrame', - ['step', 'var', 'num_splits', 'index']) +ForeachFrame = namedtuple("ForeachFrame", ["step", "var", "num_splits", "index"]) class MetaflowTask(object): @@ -27,15 +25,17 @@ class MetaflowTask(object): MetaflowTask prepares a Flow instance for execution of a single step. """ - def __init__(self, - flow, - flow_datastore, - metadata, - environment, - console_logger, - event_logger, - monitor, - ubf_context): + def __init__( + self, + flow, + flow_datastore, + metadata, + environment, + console_logger, + event_logger, + monitor, + ubf_context, + ): self.flow = flow self.flow_datastore = flow_datastore self.metadata = metadata @@ -59,14 +59,17 @@ def _init_parameters(self, parameter_ds, passdown=True): # make the parameter a read-only property # note x=x binds the current value of x to the closure def property_setter( - _, cls=self.flow.__class__, param=param, var=var, - parameter_ds=parameter_ds): + _, + cls=self.flow.__class__, + param=param, + var=var, + parameter_ds=parameter_ds, + ): v = param.load_parameter(parameter_ds[var]) setattr(cls, var, property(fget=lambda _, val=v: val)) return v - setattr(self.flow.__class__, var, - property(fget=property_setter)) + setattr(self.flow.__class__, var, property(fget=property_setter)) vars.append(var) if passdown: self.flow._datastore.passdown_partial(parameter_ds, vars) @@ -79,34 +82,41 @@ def _init_data(self, run_id, join_type, input_paths): # init takes ~200-300ms when run sequentially. if len(input_paths) > 4: prefetch_data_artifacts = None - if join_type and join_type == 'foreach': + if join_type and join_type == "foreach": # Prefetch 'foreach' related artifacts to improve time taken by # _init_foreach. - prefetch_data_artifacts = \ - ['_foreach_stack', '_foreach_num_splits', '_foreach_var'] + prefetch_data_artifacts = [ + "_foreach_stack", + "_foreach_num_splits", + "_foreach_var", + ] # Note: Specify `pathspecs` while creating the datastore set to # guarantee strong consistency and guard against missing input. - datastore_set = \ - TaskDataStoreSet(self.flow_datastore, - run_id, - pathspecs=input_paths, - prefetch_data_artifacts=prefetch_data_artifacts) + datastore_set = TaskDataStoreSet( + self.flow_datastore, + run_id, + pathspecs=input_paths, + prefetch_data_artifacts=prefetch_data_artifacts, + ) ds_list = [ds for ds in datastore_set] if len(ds_list) != len(input_paths): - raise MetaflowDataMissing("Some input datastores are missing. " - "Expected: %d Actual: %d" % - (len(input_paths), len(ds_list))) + raise MetaflowDataMissing( + "Some input datastores are missing. " + "Expected: %d Actual: %d" % (len(input_paths), len(ds_list)) + ) else: # initialize directly in the single input case. ds_list = [] for input_path in input_paths: - run_id, step_name, task_id = input_path.split('/') + run_id, step_name, task_id = input_path.split("/") ds_list.append( - self.flow_datastore.get_task_datastore(run_id, step_name, task_id)) + self.flow_datastore.get_task_datastore(run_id, step_name, task_id) + ) if not ds_list: # this guards against errors in input paths - raise MetaflowDataMissing("Input paths *%s* resolved to zero " - "inputs" % ','.join(input_paths)) + raise MetaflowDataMissing( + "Input paths *%s* resolved to zero " "inputs" % ",".join(input_paths) + ) return ds_list def _init_foreach(self, step_name, join_type, inputs, split_index): @@ -121,7 +131,7 @@ def _init_foreach(self, step_name, join_type, inputs, split_index): # 3) step following a split - push a new frame in the stack # case 1) - reset the stack - if step_name == 'start': + if step_name == "start": self.flow._foreach_stack = [] # case 2) - this is a join step @@ -129,66 +139,73 @@ def _init_foreach(self, step_name, join_type, inputs, split_index): # assert the lineage of incoming branches def lineage(): for i in inputs: - if join_type == 'foreach': - top = i['_foreach_stack'][-1] - bottom = i['_foreach_stack'][:-1] + if join_type == "foreach": + top = i["_foreach_stack"][-1] + bottom = i["_foreach_stack"][:-1] # the topmost indices in the stack are all # different naturally, so ignore them in the # assertion yield bottom + [top._replace(index=0)] else: - yield i['_foreach_stack'] + yield i["_foreach_stack"] if not all_equal(lineage()): - raise MetaflowInternalError("Step *%s* tried to join branches " - "whose lineages don't match." - % step_name) + raise MetaflowInternalError( + "Step *%s* tried to join branches " + "whose lineages don't match." % step_name + ) # assert that none of the inputs are splits - we don't # allow empty foreaches (joins immediately following splits) - if any(not i.is_none('_foreach_var') for i in inputs): - raise MetaflowInternalError("Step *%s* tries to join a foreach " - "split with no intermediate steps." - % step_name) + if any(not i.is_none("_foreach_var") for i in inputs): + raise MetaflowInternalError( + "Step *%s* tries to join a foreach " + "split with no intermediate steps." % step_name + ) inp = inputs[0] - if join_type == 'foreach': + if join_type == "foreach": # Make sure that the join got all splits as its inputs. # Datastore.resolve() leaves out all undone tasks, so if # something strange happened upstream, the inputs list # may not contain all inputs which should raise an exception - stack = inp['_foreach_stack'] + stack = inp["_foreach_stack"] if stack[-1].num_splits and len(inputs) != stack[-1].num_splits: - raise MetaflowDataMissing("Foreach join *%s* expected %d " - "splits but only %d inputs were " - "found" % (step_name, - stack[-1].num_splits, - len(inputs))) + raise MetaflowDataMissing( + "Foreach join *%s* expected %d " + "splits but only %d inputs were " + "found" % (step_name, stack[-1].num_splits, len(inputs)) + ) # foreach-join pops the topmost frame from the stack self.flow._foreach_stack = stack[:-1] else: # a non-foreach join doesn't change the stack - self.flow._foreach_stack = inp['_foreach_stack'] + self.flow._foreach_stack = inp["_foreach_stack"] # case 3) - our parent was a split. Initialize a new foreach frame. - elif not inputs[0].is_none('_foreach_var'): + elif not inputs[0].is_none("_foreach_var"): if len(inputs) != 1: - raise MetaflowInternalError("Step *%s* got multiple inputs " - "although it follows a split step." - % step_name) + raise MetaflowInternalError( + "Step *%s* got multiple inputs " + "although it follows a split step." % step_name + ) if self.ubf_context != UBF_CONTROL and split_index is None: - raise MetaflowInternalError("Step *%s* follows a split step " - "but no split_index is " - "specified." % step_name) + raise MetaflowInternalError( + "Step *%s* follows a split step " + "but no split_index is " + "specified." % step_name + ) # push a new index after a split to the stack - frame = ForeachFrame(step_name, - inputs[0]['_foreach_var'], - inputs[0]['_foreach_num_splits'], - split_index) - - stack = inputs[0]['_foreach_stack'] + frame = ForeachFrame( + step_name, + inputs[0]["_foreach_var"], + inputs[0]["_foreach_num_splits"], + split_index, + ) + + stack = inputs[0]["_foreach_stack"] stack.append(frame) self.flow._foreach_stack = stack @@ -199,19 +216,21 @@ def _clone_flow(self, datastore): def clone_only(self, step_name, run_id, task_id, clone_origin_task): if not clone_origin_task: - raise MetaflowInternalError("task.clone_only needs a valid " - "clone_origin_task value.") + raise MetaflowInternalError( + "task.clone_only needs a valid " "clone_origin_task value." + ) # 1. initialize output datastore output = self.flow_datastore.get_task_datastore( - run_id, step_name, task_id, attempt=0, mode='w') + run_id, step_name, task_id, attempt=0, mode="w" + ) output.init_task() - origin_run_id, origin_step_name, origin_task_id =\ - clone_origin_task.split('/') + origin_run_id, origin_step_name, origin_task_id = clone_origin_task.split("/") # 2. initialize origin datastore origin = self.flow_datastore.get_task_datastore( - origin_run_id, origin_step_name, origin_task_id) + origin_run_id, origin_step_name, origin_task_id + ) output.clone(origin) output.done() @@ -225,81 +244,112 @@ def _finalize_control_task(self): # Throw an error if `_control_mapper_tasks` isn't populated. mapper_tasks = self.flow._control_mapper_tasks if not mapper_tasks: - msg = "Step *{step}* has a control task which didn't "\ - "specify the artifact *_control_mapper_tasks* for "\ - "the subsequent *{join}* step." - raise MetaflowInternalError(msg.format(step=step_name, - join=next_steps[0])) - elif not (isinstance(mapper_tasks, list) and\ - isinstance(mapper_tasks[0], unicode_type)): - msg = "Step *{step}* has a control task which didn't "\ - "specify the artifact *_control_mapper_tasks* as a "\ - "list of strings but instead specified it as {typ} "\ - "with elements of {elem_typ}." - raise MetaflowInternalError(msg.format(step=step_name, - typ=type(mapper_tasks), - elem_typ=type(mapper_tasks[0]))) - - def run_step(self, - step_name, - run_id, - task_id, - origin_run_id, - input_paths, - split_index, - retry_count, - max_user_code_retries): + msg = ( + "Step *{step}* has a control task which didn't " + "specify the artifact *_control_mapper_tasks* for " + "the subsequent *{join}* step." + ) + raise MetaflowInternalError( + msg.format(step=step_name, join=next_steps[0]) + ) + elif not ( + isinstance(mapper_tasks, list) + and isinstance(mapper_tasks[0], unicode_type) + ): + msg = ( + "Step *{step}* has a control task which didn't " + "specify the artifact *_control_mapper_tasks* as a " + "list of strings but instead specified it as {typ} " + "with elements of {elem_typ}." + ) + raise MetaflowInternalError( + msg.format( + step=step_name, + typ=type(mapper_tasks), + elem_typ=type(mapper_tasks[0]), + ) + ) + + def run_step( + self, + step_name, + run_id, + task_id, + origin_run_id, + input_paths, + split_index, + retry_count, + max_user_code_retries, + ): if run_id and task_id: self.metadata.register_run_id(run_id) self.metadata.register_task_id(run_id, step_name, task_id, retry_count) else: - raise MetaflowInternalError("task.run_step needs a valid run_id " - "and task_id") + raise MetaflowInternalError( + "task.run_step needs a valid run_id " "and task_id" + ) if retry_count >= MAX_ATTEMPTS: # any results with an attempt ID >= MAX_ATTEMPTS will be ignored # by datastore, so running a task with such a retry_could would # be pointless and dangerous - raise MetaflowInternalError("Too many task attempts (%d)! " - "MAX_ATTEMPTS exceeded." % retry_count) + raise MetaflowInternalError( + "Too many task attempts (%d)! " "MAX_ATTEMPTS exceeded." % retry_count + ) metadata_tags = ["attempt_id:{0}".format(retry_count)] - self.metadata.register_metadata(run_id, - step_name, - task_id, - [MetaDatum(field='attempt', - value=str(retry_count), - type='attempt', - tags=metadata_tags), - MetaDatum(field='origin-run-id', - value=str(origin_run_id), - type='origin-run-id', - tags=metadata_tags), - MetaDatum(field='ds-type', - value=self.flow_datastore.TYPE, - type='ds-type', - tags=metadata_tags), - MetaDatum(field='ds-root', - value=self.flow_datastore.datastore_root, - type='ds-root', - tags=metadata_tags)]) + self.metadata.register_metadata( + run_id, + step_name, + task_id, + [ + MetaDatum( + field="attempt", + value=str(retry_count), + type="attempt", + tags=metadata_tags, + ), + MetaDatum( + field="origin-run-id", + value=str(origin_run_id), + type="origin-run-id", + tags=metadata_tags, + ), + MetaDatum( + field="ds-type", + value=self.flow_datastore.TYPE, + type="ds-type", + tags=metadata_tags, + ), + MetaDatum( + field="ds-root", + value=self.flow_datastore.datastore_root, + type="ds-root", + tags=metadata_tags, + ), + ], + ) step_func = getattr(self.flow, step_name) node = self.flow._graph[step_name] join_type = None - if node.type == 'join': + if node.type == "join": join_type = self.flow._graph[node.split_parents[-1]].type # 1. initialize output datastore output = self.flow_datastore.get_task_datastore( - run_id, step_name, task_id, attempt=retry_count, mode='w') + run_id, step_name, task_id, attempt=retry_count, mode="w" + ) output.init_task() if input_paths: - control_paths = [path for path in input_paths - if path.split('/')[-1].startswith('control-')] + control_paths = [ + path + for path in input_paths + if path.split("/")[-1].startswith("control-") + ] if control_paths: [control_path] = control_paths input_paths.remove(control_path) @@ -310,39 +360,43 @@ def run_step(self, self._init_foreach(step_name, join_type, inputs, split_index) # 4. initialize the current singleton - current._set_env(flow_name=self.flow.name, - run_id=run_id, - step_name=step_name, - task_id=task_id, - retry_count=retry_count, - origin_run_id=origin_run_id, - namespace=resolve_identity(), - username=get_username(), - is_running=True) + current._set_env( + flow_name=self.flow.name, + run_id=run_id, + step_name=step_name, + task_id=task_id, + retry_count=retry_count, + origin_run_id=origin_run_id, + namespace=resolve_identity(), + username=get_username(), + is_running=True, + ) # 5. run task - output.save_metadata({'task_begin': + output.save_metadata( { - 'code_package_sha': os.environ.get('METAFLOW_CODE_SHA'), - 'code_package_ds': os.environ.get('METAFLOW_CODE_DS'), - 'code_package_url': os.environ.get('METAFLOW_CODE_URL'), - 'retry_count': retry_count - }}) + "task_begin": { + "code_package_sha": os.environ.get("METAFLOW_CODE_SHA"), + "code_package_ds": os.environ.get("METAFLOW_CODE_DS"), + "code_package_url": os.environ.get("METAFLOW_CODE_URL"), + "retry_count": retry_count, + } + } + ) logger = self.event_logger start = time.time() - self.metadata.start_task_heartbeat(self.flow.name, run_id, step_name, - task_id) + self.metadata.start_task_heartbeat(self.flow.name, run_id, step_name, task_id) try: # init side cars logger.start() msg = { "task_id": task_id, - "msg": 'task starting', + "msg": "task starting", "step_name": step_name, "run_id": run_id, "flow_name": self.flow.name, - "ts": round(time.time()) + "ts": round(time.time()), } logger.log(msg) @@ -360,14 +414,12 @@ def run_step(self, # Ensure that we have the right number of inputs. The # foreach case is checked above. - if join_type != 'foreach' and\ - len(inputs) != len(node.in_funcs): - raise MetaflowDataMissing("Join *%s* expected %d " - "inputs but only %d inputs " - "were found" - % (step_name, - len(node.in_funcs), - len(inputs))) + if join_type != "foreach" and len(inputs) != len(node.in_funcs): + raise MetaflowDataMissing( + "Join *%s* expected %d " + "inputs but only %d inputs " + "were found" % (step_name, len(node.in_funcs), len(inputs)) + ) # Multiple input contexts are passed in as an argument # to the step function. @@ -376,7 +428,9 @@ def run_step(self, # initialize parameters (if they exist) # We take Parameter values from the first input, # which is always safe since parameters are read-only - current._update_env({'parameter_names': self._init_parameters(inputs[0], passdown=True)}) + current._update_env( + {"parameter_names": self._init_parameters(inputs[0], passdown=True)} + ) else: # Linear step: # We are running with a single input context. @@ -384,42 +438,54 @@ def run_step(self, if len(inputs) > 1: # This should be captured by static checking but # let's assert this again - raise MetaflowInternalError("Step *%s* is not a join " - "step but it gets multiple " - "inputs." % step_name) + raise MetaflowInternalError( + "Step *%s* is not a join " + "step but it gets multiple " + "inputs." % step_name + ) self.flow._set_datastore(inputs[0]) if input_paths: # initialize parameters (if they exist) # We take Parameter values from the first input, # which is always safe since parameters are read-only - current._update_env({'parameter_names': self._init_parameters(inputs[0], passdown=False)}) + current._update_env( + { + "parameter_names": self._init_parameters( + inputs[0], passdown=False + ) + } + ) decorators = step_func.decorators for deco in decorators: - deco.task_pre_step(step_name, - output, - self.metadata, - run_id, - task_id, - self.flow, - self.flow._graph, - retry_count, - max_user_code_retries, - self.ubf_context, - inputs) + deco.task_pre_step( + step_name, + output, + self.metadata, + run_id, + task_id, + self.flow, + self.flow._graph, + retry_count, + max_user_code_retries, + self.ubf_context, + inputs, + ) # decorators can actually decorate the step function, # or they can replace it altogether. This functionality # is used e.g. by catch_decorator which switches to a # fallback code if the user code has failed too many # times. - step_func = deco.task_decorate(step_func, - self.flow, - self.flow._graph, - retry_count, - max_user_code_retries, - self.ubf_context) + step_func = deco.task_decorate( + step_func, + self.flow, + self.flow._graph, + retry_count, + max_user_code_retries, + self.ubf_context, + ) if join_type: self._exec_step_function(step_func, input_obj) @@ -427,11 +493,13 @@ def run_step(self, self._exec_step_function(step_func) for deco in decorators: - deco.task_post_step(step_name, - self.flow, - self.flow._graph, - retry_count, - max_user_code_retries) + deco.task_post_step( + step_name, + self.flow, + self.flow._graph, + retry_count, + max_user_code_retries, + ) self.flow._task_ok = True self.flow._success = True @@ -440,21 +508,23 @@ def run_step(self, tsk_msg = { "task_id": task_id, "exception_msg": str(ex), - "msg": 'task failed with exception', + "msg": "task failed with exception", "step_name": step_name, "run_id": run_id, - "flow_name": self.flow.name + "flow_name": self.flow.name, } logger.log(tsk_msg) exception_handled = False for deco in decorators: - res = deco.task_exception(ex, - step_name, - self.flow, - self.flow._graph, - retry_count, - max_user_code_retries) + res = deco.task_exception( + ex, + step_name, + self.flow, + self.flow._graph, + retry_count, + max_user_code_retries, + ) exception_handled = bool(res) or exception_handled if exception_handled: @@ -462,7 +532,7 @@ def run_step(self, else: self.flow._task_ok = False self.flow._exception = MetaflowExceptionWrapper(ex) - print('%s failed:' % self.flow, file=sys.stderr) + print("%s failed:" % self.flow, file=sys.stderr) raise finally: @@ -473,27 +543,31 @@ def run_step(self, msg = { "task_id": task_id, - "msg": 'task ending', + "msg": "task ending", "step_name": step_name, "run_id": run_id, "flow_name": self.flow.name, "ts": round(time.time()), - "runtime": round(end) + "runtime": round(end), } logger.log(msg) attempt_ok = str(bool(self.flow._task_ok)) - self.metadata.register_metadata(run_id, - step_name, - task_id, - [MetaDatum(field='attempt_ok', - value=attempt_ok, - type='internal_attempt_status', - tags=["attempt_id:{0}". - format(retry_count)]) - ]) - - output.save_metadata({'task_end': {}}) + self.metadata.register_metadata( + run_id, + step_name, + task_id, + [ + MetaDatum( + field="attempt_ok", + value=attempt_ok, + type="internal_attempt_status", + tags=["attempt_id:{0}".format(retry_count)], + ) + ], + ) + + output.save_metadata({"task_end": {}}) output.persist(self.flow) # this writes a success marker indicating that the @@ -503,12 +577,14 @@ def run_step(self, # final decorator hook: The task results are now # queryable through the client API / datastore for deco in decorators: - deco.task_finished(step_name, - self.flow, - self.flow._graph, - self.flow._task_ok, - retry_count, - max_user_code_retries) + deco.task_finished( + step_name, + self.flow, + self.flow._graph, + self.flow._task_ok, + retry_count, + max_user_code_retries, + ) # terminate side cars logger.terminate() diff --git a/metaflow/tutorials/00-helloworld/helloworld.py b/metaflow/tutorials/00-helloworld/helloworld.py index 8495729a5d3..2e073b5131c 100644 --- a/metaflow/tutorials/00-helloworld/helloworld.py +++ b/metaflow/tutorials/00-helloworld/helloworld.py @@ -8,6 +8,7 @@ class HelloFlow(FlowSpec): Run this flow to validate that Metaflow is installed correctly. """ + @step def start(self): """ @@ -37,5 +38,5 @@ def end(self): print("HelloFlow is all done.") -if __name__ == '__main__': +if __name__ == "__main__": HelloFlow() diff --git a/metaflow/tutorials/01-playlist/playlist.py b/metaflow/tutorials/01-playlist/playlist.py index 98e7d56ba87..16b653cac1d 100644 --- a/metaflow/tutorials/01-playlist/playlist.py +++ b/metaflow/tutorials/01-playlist/playlist.py @@ -27,18 +27,22 @@ class PlayListFlow(FlowSpec): 4) Displays the top entries from the playlist. """ - movie_data = IncludeFile("movie_data", - help="The path to a movie metadata file.", - default=script_path('movies.csv')) - genre = Parameter('genre', - help="Filter movies for a particular genre.", - default='Sci-Fi') + movie_data = IncludeFile( + "movie_data", + help="The path to a movie metadata file.", + default=script_path("movies.csv"), + ) - recommendations = Parameter('recommendations', - help="The number of movies to recommend in " - "the playlist.", - default=5) + genre = Parameter( + "genre", help="Filter movies for a particular genre.", default="Sci-Fi" + ) + + recommendations = Parameter( + "recommendations", + help="The number of movies to recommend in " "the playlist.", + default=5, + ) @step def start(self): @@ -47,15 +51,14 @@ def start(self): """ # For this example, we only need the movie title and the genres. - columns = ['movie_title', 'genres'] + columns = ["movie_title", "genres"] # Create a simple data frame as a dictionary of lists. - self.dataframe = dict((column, list()) \ - for column in columns) + self.dataframe = dict((column, list()) for column in columns) # Parse the CSV header. - lines = self.movie_data.split('\n') - header = lines[0].split(',') + lines = self.movie_data.split("\n") + header = lines[0].split(",") idx = {column: header.index(column) for column in columns} # Populate our dataframe from the lines of the CSV file. @@ -63,7 +66,7 @@ def start(self): if not line: continue - fields = line.rsplit(',', 4) + fields = line.rsplit(",", 4) for column in columns: self.dataframe[column].append(fields[idx[column]]) @@ -79,11 +82,13 @@ def bonus_movie(self): from random import choice # Find all the movies that are not in the provided genre. - movies = [(movie, genres) \ - for movie, genres \ - in zip(self.dataframe['movie_title'], - self.dataframe['genres']) \ - if self.genre.lower() not in genres.lower()] + movies = [ + (movie, genres) + for movie, genres in zip( + self.dataframe["movie_title"], self.dataframe["genres"] + ) + if self.genre.lower() not in genres.lower() + ] # Choose one randomly. self.bonus = choice(movies) @@ -99,11 +104,13 @@ def genre_movies(self): from random import shuffle # Find all the movies titles in the specified genre. - self.movies = [movie \ - for movie, genres \ - in zip(self.dataframe['movie_title'], - self.dataframe['genres']) \ - if self.genre.lower() in genres.lower()] + self.movies = [ + movie + for movie, genres in zip( + self.dataframe["movie_title"], self.dataframe["genres"] + ) + if self.genre.lower() in genres.lower() + ] # Randomize the title names. shuffle(self.movies) @@ -137,5 +144,5 @@ def end(self): print("Bonus Pick: '%s' from '%s'" % (self.bonus[0], self.bonus[1])) -if __name__ == '__main__': +if __name__ == "__main__": PlayListFlow() diff --git a/metaflow/tutorials/02-statistics/stats.py b/metaflow/tutorials/02-statistics/stats.py index bfe692baba0..ca1419f8697 100644 --- a/metaflow/tutorials/02-statistics/stats.py +++ b/metaflow/tutorials/02-statistics/stats.py @@ -25,9 +25,12 @@ class MovieStatsFlow(FlowSpec): 4) Save a dictionary of genre specific statistics. """ - movie_data = IncludeFile("movie_data", - help="The path to a movie metadata file.", - default=script_path('movies.csv')) + + movie_data = IncludeFile( + "movie_data", + help="The path to a movie metadata file.", + default=script_path("movies.csv"), + ) @step def start(self): @@ -46,15 +49,15 @@ def start(self): # The column 'genres' has a list of genres for each movie. Let's get # all the unique genres. - self.genres = {genre for genres \ - in self.dataframe['genres'] \ - for genre in genres.split('|')} + self.genres = { + genre for genres in self.dataframe["genres"] for genre in genres.split("|") + } self.genres = list(self.genres) # We want to compute some statistics for each genre. The 'foreach' # keyword argument allows us to compute the statistics for each genre in # parallel (i.e. a fan-out). - self.next(self.compute_statistics, foreach='genres') + self.next(self.compute_statistics, foreach="genres") @step def compute_statistics(self): @@ -69,14 +72,13 @@ def compute_statistics(self): # Find all the movies that have this genre and build a dataframe with # just those movies and just the columns of interest. - selector = self.dataframe['genres'].\ - apply(lambda row: self.genre in row) + selector = self.dataframe["genres"].apply(lambda row: self.genre in row) self.dataframe = self.dataframe[selector] - self.dataframe = self.dataframe[['movie_title', 'genres', 'gross']] + self.dataframe = self.dataframe[["movie_title", "genres", "gross"]] # Get some statistics on the gross box office for these titles. - points = [.25, .5, .75] - self.quartiles = self.dataframe['gross'].quantile(points).values + points = [0.25, 0.5, 0.75] + self.quartiles = self.dataframe["gross"].quantile(points).values # Join the results from other genres. self.next(self.join) @@ -88,10 +90,10 @@ def join(self, inputs): """ # Merge results from the genre specific computations. - self.genre_stats = {inp.genre.lower(): \ - {'quartiles': inp.quartiles, - 'dataframe': inp.dataframe} \ - for inp in inputs} + self.genre_stats = { + inp.genre.lower(): {"quartiles": inp.quartiles, "dataframe": inp.dataframe} + for inp in inputs + } self.next(self.end) @@ -104,5 +106,5 @@ def end(self): pass -if __name__ == '__main__': +if __name__ == "__main__": MovieStatsFlow() diff --git a/metaflow/tutorials/03-playlist-redux/playlist.py b/metaflow/tutorials/03-playlist-redux/playlist.py index 50abcef35f3..43153802e64 100644 --- a/metaflow/tutorials/03-playlist-redux/playlist.py +++ b/metaflow/tutorials/03-playlist-redux/playlist.py @@ -15,14 +15,16 @@ class PlayListFlow(FlowSpec): 3) Join the two to create a movie playlist and display it. """ - genre = Parameter('genre', - help="Filter movies for a particular genre.", - default='Sci-Fi') - recommendations = Parameter('recommendations', - help="The number of movies recommended for " - "the playlist.", - default=5) + genre = Parameter( + "genre", help="Filter movies for a particular genre.", default="Sci-Fi" + ) + + recommendations = Parameter( + "recommendations", + help="The number of movies recommended for " "the playlist.", + default=5, + ) @step def start(self): @@ -37,7 +39,7 @@ def start(self): print("Using metadata provider: %s" % get_metadata()) # Load the analysis from the MovieStatsFlow. - run = Flow('MovieStatsFlow').latest_successful_run + run = Flow("MovieStatsFlow").latest_successful_run print("Using analysis from '%s'" % str(run)) self.genre_stats = run.data.genre_stats @@ -55,12 +57,15 @@ def bonus_movie(self): # Concatenate all the genre specific data frames and choose a random # movie. - df = pandas.concat([data['dataframe'] \ - for genre, data in self.genre_stats.items() \ - if genre != self.genre.lower()]) + df = pandas.concat( + [ + data["dataframe"] + for genre, data in self.genre_stats.items() + if genre != self.genre.lower() + ] + ) df = df.sample(n=1) - self.bonus = (df['movie_title'].values[0], - df['genres'].values[0]) + self.bonus = (df["movie_title"].values[0], df["genres"].values[0]) self.next(self.join) @@ -79,10 +84,10 @@ def genre_movies(self): self.movies = [] else: - df = self.genre_stats[genre]['dataframe'] - quartiles = self.genre_stats[genre]['quartiles'] - selector = df['gross'] >= quartiles[-1] - self.movies = list(df[selector]['movie_title']) + df = self.genre_stats[genre]["dataframe"] + quartiles = self.genre_stats[genre]["quartiles"] + selector = df["gross"] >= quartiles[-1] + self.movies = list(df[selector]["movie_title"]) # Shuffle the playlist. shuffle(self.movies) @@ -116,5 +121,5 @@ def end(self): print("Bonus Pick: '%s' from '%s'" % (self.bonus[0], self.bonus[1])) -if __name__ == '__main__': +if __name__ == "__main__": PlayListFlow() diff --git a/metaflow/tutorials/04-playlist-plus/playlist.py b/metaflow/tutorials/04-playlist-plus/playlist.py index 81a4e371110..cf7b5a95db4 100644 --- a/metaflow/tutorials/04-playlist-plus/playlist.py +++ b/metaflow/tutorials/04-playlist-plus/playlist.py @@ -9,8 +9,8 @@ def get_python_version(): """ import platform - versions = {'2' : '2.7.15', - '3' : '3.7.3'} + + versions = {"2": "2.7.15", "3": "3.7.3"} return versions[platform.python_version_tuple()[0]] @@ -31,20 +31,24 @@ class PlayListFlow(FlowSpec): 3) Join the two to create a movie playlist and display it. """ - genre = Parameter('genre', - help="Filter movies for a particular genre.", - default='Sci-Fi') - hint = Parameter('hint', - help="Give a hint to the bonus movie algorithm.", - default='Metaflow Release') + genre = Parameter( + "genre", help="Filter movies for a particular genre.", default="Sci-Fi" + ) + + hint = Parameter( + "hint", + help="Give a hint to the bonus movie algorithm.", + default="Metaflow Release", + ) - recommendations = Parameter('recommendations', - help="The number of movies recommended for " - "the playlist.", - default=5) + recommendations = Parameter( + "recommendations", + help="The number of movies recommended for " "the playlist.", + default=5, + ) - @conda(libraries={'pandas' : '1.3.3'}) + @conda(libraries={"pandas": "1.3.3"}) @step def start(self): """ @@ -63,12 +67,12 @@ def start(self): print("Using metadata provider: %s" % get_metadata()) # Load the analysis from the MovieStatsFlow. - run = Flow('MovieStatsFlow').latest_successful_run + run = Flow("MovieStatsFlow").latest_successful_run print("Using analysis from '%s'" % str(run)) # Get the dataframe from the start step before we sliced into into # genre specific dataframes. - self.dataframe = run['start'].task.data.dataframe + self.dataframe = run["start"].task.data.dataframe # Also grab the summary statistics. self.genre_stats = run.data.genre_stats @@ -76,7 +80,7 @@ def start(self): # Compute our two recommendation types in parallel. self.next(self.bonus_movie, self.genre_movies) - @conda(libraries={'editdistance': '0.5.3', 'pandas' : '1.3.3'}) + @conda(libraries={"editdistance": "0.5.3", "pandas": "1.3.3"}) @step def bonus_movie(self): """ @@ -96,16 +100,17 @@ def bonus_movie(self): def _edit_distance(movie_title): return editdistance.eval(self.hint, movie_title) - # Compute the distance and take the argmin to find the closest title. - distance = self.dataframe['movie_title'].apply(_edit_distance) + distance = self.dataframe["movie_title"].apply(_edit_distance) index = distance.idxmin() - self.bonus = (self.dataframe['movie_title'].values[index], - self.dataframe['genres'].values[index]) + self.bonus = ( + self.dataframe["movie_title"].values[index], + self.dataframe["genres"].values[index], + ) self.next(self.join) - @conda(libraries={'pandas' : '1.3.3'}) + @conda(libraries={"pandas": "1.3.3"}) @step def genre_movies(self): """ @@ -126,10 +131,10 @@ def genre_movies(self): self.movies = [] else: - df = self.genre_stats[genre]['dataframe'] - quartiles = self.genre_stats[genre]['quartiles'] - selector = df['gross'] >= quartiles[-1] - self.movies = list(df[selector]['movie_title']) + df = self.genre_stats[genre]["dataframe"] + quartiles = self.genre_stats[genre]["quartiles"] + selector = df["gross"] >= quartiles[-1] + self.movies = list(df[selector]["movie_title"]) # Shuffle the content. shuffle(self.movies) @@ -163,5 +168,5 @@ def end(self): print("Bonus Pick: '%s' from '%s'" % (self.bonus[0], self.bonus[1])) -if __name__ == '__main__': +if __name__ == "__main__": PlayListFlow() diff --git a/metaflow/tutorials/05-helloaws/helloaws.py b/metaflow/tutorials/05-helloaws/helloaws.py index 71a852b72db..97218108e85 100644 --- a/metaflow/tutorials/05-helloaws/helloaws.py +++ b/metaflow/tutorials/05-helloaws/helloaws.py @@ -8,6 +8,7 @@ class HelloAWSFlow(FlowSpec): Run this flow to validate your AWS configuration. """ + @step def start(self): """ @@ -40,7 +41,7 @@ def hello(self): goes wrong, the step will be automatically retried. """ - self.message = 'Hi from AWS!' + self.message = "Hi from AWS!" print("Metaflow says: %s" % self.message) self.next(self.end) @@ -54,5 +55,5 @@ def end(self): print("HelloAWS is finished.") -if __name__ == '__main__': +if __name__ == "__main__": HelloAWSFlow() diff --git a/metaflow/unbounded_foreach.py b/metaflow/unbounded_foreach.py index 06f9d75d0ec..1560d3510de 100644 --- a/metaflow/unbounded_foreach.py +++ b/metaflow/unbounded_foreach.py @@ -1,6 +1,7 @@ -CONTROL_TASK_TAG = 'control_task' -UBF_CONTROL = 'ubf_control' -UBF_TASK = 'ubf_task' +CONTROL_TASK_TAG = "control_task" +UBF_CONTROL = "ubf_control" +UBF_TASK = "ubf_task" + class UnboundedForeachInput(object): """ diff --git a/metaflow/util.py b/metaflow/util.py index 360cf3cac77..e57f646391b 100644 --- a/metaflow/util.py +++ b/metaflow/util.py @@ -31,7 +31,6 @@ def unquote_bytes(x): # this is used e.g. by mflog/save_logs.py to identify paths class Path(object): - def __init__(self, path): self.path = path @@ -53,9 +52,11 @@ def unquote_bytes(x): if sys.version_info.major >= 3 and sys.version_info.minor >= 7: from collections import namedtuple + namedtuple_with_defaults = namedtuple else: from collections import namedtuple + def namedtuple_with_defaults(typename, field_names, defaults=()): T = namedtuple(typename, field_names) T.__new__.__defaults__ = (None,) * len(T._fields) @@ -63,6 +64,7 @@ def namedtuple_with_defaults(typename, field_names, defaults=()): T.__new__.__defaults__ = tuple(prototype) return T + class TempDir(object): # Provide a temporary directory since Python 2.7 does not have it inbuilt def __enter__(self): @@ -76,10 +78,11 @@ def __exit__(self, exc_type, exc_value, traceback): def cached_property(getter): @wraps(getter) def exec_once(self): - saved_name = '__%s' % getter.__name__ + saved_name = "__%s" % getter.__name__ if not hasattr(self, saved_name): setattr(self, saved_name, getter(self)) return getattr(self, saved_name) + return property(exec_once) @@ -109,7 +112,8 @@ def url_quote(url): # # We mark colon as a safe character to keep simple ASCII urls # nice looking, e.g. "http://google.com" - return to_bytes(quote(to_bytes(url), safe='/:')) + return to_bytes(quote(to_bytes(url), safe="/:")) + def url_unquote(url_bytes): """ @@ -117,6 +121,7 @@ def url_unquote(url_bytes): """ return unquote_bytes(url_bytes) + def is_stringish(x): """ Returns true if the object is a unicode or a bytes object @@ -136,7 +141,7 @@ def to_unicode(x): Convert any object to a unicode object """ if isinstance(x, bytes_type): - return x.decode('utf-8') + return x.decode("utf-8") else: return unicode_type(x) @@ -146,13 +151,13 @@ def to_bytes(x): Convert any object to a byte string """ if isinstance(x, unicode_type): - return x.encode('utf-8') + return x.encode("utf-8") elif isinstance(x, bytes_type): return x elif isinstance(x, float): - return repr(x).encode('utf-8') + return repr(x).encode("utf-8") else: - return str(x).encode('utf-8') + return str(x).encode("utf-8") def get_username(): @@ -161,32 +166,35 @@ def get_username(): could not be determined. """ # note: the order of the list matters - ENVVARS = ['METAFLOW_USER', 'SUDO_USER', 'USERNAME', 'USER'] + ENVVARS = ["METAFLOW_USER", "SUDO_USER", "USERNAME", "USER"] for var in ENVVARS: user = os.environ.get(var) - if user and user != 'root': + if user and user != "root": return user return None def resolve_identity(): - prod_token = os.environ.get('METAFLOW_PRODUCTION_TOKEN') + prod_token = os.environ.get("METAFLOW_PRODUCTION_TOKEN") if prod_token: - return 'production:%s' % prod_token + return "production:%s" % prod_token user = get_username() - if user and user != 'root': - return 'user:%s' % user + if user and user != "root": + return "user:%s" % user else: raise MetaflowUnknownUser() def get_latest_run_id(echo, flow_name): from metaflow.datastore.local_storage import LocalStorage + local_root = LocalStorage.datastore_root if local_root is None: - local_root = LocalStorage.get_datastore_root_from_config(echo, create_on_absent=False) + local_root = LocalStorage.get_datastore_root_from_config( + echo, create_on_absent=False + ) if local_root: - path = os.path.join(local_root, flow_name, 'latest_run') + path = os.path.join(local_root, flow_name, "latest_run") if os.path.exists(path): with open(path) as f: return f.read() @@ -195,8 +203,11 @@ def get_latest_run_id(echo, flow_name): def write_latest_run_id(obj, run_id): from metaflow.datastore.local_storage import LocalStorage + if LocalStorage.datastore_root is None: - LocalStorage.datastore_root = LocalStorage.get_datastore_root_from_config(obj.echo) + LocalStorage.datastore_root = LocalStorage.get_datastore_root_from_config( + obj.echo + ) path = LocalStorage.path_join(LocalStorage.datastore_root, obj.flow.name) try: os.makedirs(path) @@ -204,20 +215,20 @@ def write_latest_run_id(obj, run_id): if x.errno != 17: # Directories exists in other case which is fine raise - with open(os.path.join(path, 'latest_run'), 'w') as f: + with open(os.path.join(path, "latest_run"), "w") as f: f.write(str(run_id)) def get_object_package_version(obj): - """ + """ Return the top level package name and package version that defines the class of the given object. """ try: module_name = obj.__class__.__module__ - if '.' in module_name: - top_package_name = module_name.split('.')[0] + if "." in module_name: + top_package_name = module_name.split(".")[0] else: top_package_name = module_name @@ -232,17 +243,14 @@ class of the given object. return top_package_name, None -def compress_list(lst, - separator=',', - rangedelim=':', - zlibmarker='!', - zlibmin=500): +def compress_list(lst, separator=",", rangedelim=":", zlibmarker="!", zlibmin=500): - bad_items = [x for x in lst - if separator in x or rangedelim in x or zlibmarker in x] + bad_items = [x for x in lst if separator in x or rangedelim in x or zlibmarker in x] if bad_items: - raise MetaflowInternalError("Item '%s' includes a delimiter character " - "so it can't be compressed" % bad_items[0]) + raise MetaflowInternalError( + "Item '%s' includes a delimiter character " + "so it can't be compressed" % bad_items[0] + ) # Three output modes: lcp = longest_common_prefix(lst) if len(lst) < 2 or not lcp: @@ -262,14 +270,15 @@ def compress_list(lst, # has plenty of redundancy. Decoding the data *twice* helps a # lot compressed = zlib.compress(zlib.compress(to_bytes(res))) - return zlibmarker + base64.b64encode(compressed).decode('utf-8') + return zlibmarker + base64.b64encode(compressed).decode("utf-8") + -def decompress_list(lststr, separator=',', rangedelim=':', zlibmarker='!'): +def decompress_list(lststr, separator=",", rangedelim=":", zlibmarker="!"): # Three input modes: if lststr[0] == zlibmarker: # 3. zlib-compressed, base64-encoded lstbytes = base64.b64decode(lststr[1:]) - decoded = zlib.decompress(zlib.decompress(lstbytes)).decode('utf-8') + decoded = zlib.decompress(zlib.decompress(lstbytes)).decode("utf-8") else: decoded = lststr @@ -284,15 +293,17 @@ def decompress_list(lststr, separator=',', rangedelim=':', zlibmarker='!'): def longest_common_prefix(lst): if lst: - return ''.join(a for a, _ in takewhile(lambda t: t[0] == t[1], - zip(min(lst), max(lst)))) + return "".join( + a for a, _ in takewhile(lambda t: t[0] == t[1], zip(min(lst), max(lst))) + ) else: - return '' + return "" def get_metaflow_root(): return os.path.dirname(os.path.dirname(__file__)) + def dict_to_cli_options(params): for k, v in params.items(): # Omit boolean options set to false or None, but preserve options with an empty @@ -300,13 +311,13 @@ def dict_to_cli_options(params): if v is not False and v is not None: # we need special handling for 'with' since it is a reserved # keyword in Python, so we call it 'decospecs' in click args - if k == 'decospecs': - k = 'with' - k = k.replace('_', '-') + if k == "decospecs": + k = "with" + k = k.replace("_", "-") if not isinstance(v, tuple): v = [v] for value in v: - yield '--%s' % k + yield "--%s" % k if not isinstance(value, bool): value = to_unicode(value) @@ -336,8 +347,10 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): # directories pass the os.access check. try: # Forced testing from shutil import which as w + return w(cmd, mode, path) - except ImportError: + except ImportError: + def _access_check(fn, mode): return os.path.exists(fn) and os.access(fn, mode) and not os.path.isdir(fn) @@ -379,9 +392,9 @@ def to_pascalcase(obj): if isinstance(obj, dict): res = obj.__class__() for k in obj: - res[re.sub('([a-zA-Z])', - lambda x: x.groups()[0].upper(), k, 1)] = \ - to_pascalcase(obj[k]) + res[ + re.sub("([a-zA-Z])", lambda x: x.groups()[0].upper(), k, 1) + ] = to_pascalcase(obj[k]) elif isinstance(obj, (list, set, tuple)): res = obj.__class__(to_pascalcase(v) for v in obj) else: diff --git a/setup.py b/setup.py index 2e81d1266cd..7ed8f1c21c7 100644 --- a/setup.py +++ b/setup.py @@ -1,26 +1,23 @@ from setuptools import setup, find_packages -version = '2.4.2' +version = "2.4.2" -setup(name='metaflow', - version=version, - description='Metaflow: More Data Science, Less Engineering', - author='Machine Learning Infrastructure Team at Netflix', - author_email='help@metaflow.org', - license='Apache License 2.0', - packages=find_packages(exclude=['metaflow_test']), - py_modules=['metaflow', ], - package_data={'metaflow' : ['tutorials/*/*']}, - entry_points=''' +setup( + name="metaflow", + version=version, + description="Metaflow: More Data Science, Less Engineering", + author="Machine Learning Infrastructure Team at Netflix", + author_email="help@metaflow.org", + license="Apache License 2.0", + packages=find_packages(exclude=["metaflow_test"]), + py_modules=[ + "metaflow", + ], + package_data={"metaflow": ["tutorials/*/*"]}, + entry_points=""" [console_scripts] metaflow=metaflow.main_cli:main - ''', - install_requires = [ - 'click>=7.0', - 'requests', - 'boto3', - 'pylint' - ], - tests_require = [ - 'coverage' - ]) + """, + install_requires=["click>=7.0", "requests", "boto3", "pylint"], + tests_require=["coverage"], +) diff --git a/test/core/metaflow_extensions/exceptions/__init__.py b/test/core/metaflow_extensions/exceptions/__init__.py index 82bdaa135e0..9ac044b1258 100644 --- a/test/core/metaflow_extensions/exceptions/__init__.py +++ b/test/core/metaflow_extensions/exceptions/__init__.py @@ -1,8 +1,9 @@ from metaflow.exception import MetaflowException + class MetaflowTestException(MetaflowException): - headline = 'Subservice error' + headline = "Subservice error" def __init__(self, error): - msg = 'Test error: \'%s\'' % error - super(MetaflowTestException, self).__init__(msg) \ No newline at end of file + msg = "Test error: '%s'" % error + super(MetaflowTestException, self).__init__(msg) diff --git a/test/core/metaflow_extensions/plugins/__init__.py b/test/core/metaflow_extensions/plugins/__init__.py index 7e4c23d5760..22cc7b23486 100644 --- a/test/core/metaflow_extensions/plugins/__init__.py +++ b/test/core/metaflow_extensions/plugins/__init__.py @@ -3,9 +3,11 @@ def get_plugin_cli(): from .flow_options import FlowDecoratorWithOptions + FLOW_DECORATORS = [FlowDecoratorWithOptions] from .test_step_decorator import TestStepDecorator + STEP_DECORATORS = [TestStepDecorator] ENVIRONMENTS = [] @@ -18,4 +20,4 @@ def get_plugin_cli(): MONITOR_SIDECARS = {} -__mf_promote_submodules__ = ['nondecoplugin'] \ No newline at end of file +__mf_promote_submodules__ = ["nondecoplugin"] diff --git a/test/core/metaflow_extensions/plugins/flow_options.py b/test/core/metaflow_extensions/plugins/flow_options.py index 9e860a66069..7747a09cb90 100644 --- a/test/core/metaflow_extensions/plugins/flow_options.py +++ b/test/core/metaflow_extensions/plugins/flow_options.py @@ -3,15 +3,11 @@ class FlowDecoratorWithOptions(FlowDecorator): - name = 'test_flow_decorator' + name = "test_flow_decorator" - options = { - 'foobar': dict( - default=None, - show_default=False, - help='Test flag' - ) - } + options = {"foobar": dict(default=None, show_default=False, help="Test flag")} - def flow_init(self, flow, graph, environment, flow_datastore, metadata, logger, echo, options): - current._update_env({'foobar_value': options['foobar']}) \ No newline at end of file + def flow_init( + self, flow, graph, environment, flow_datastore, metadata, logger, echo, options + ): + current._update_env({"foobar_value": options["foobar"]}) diff --git a/test/core/metaflow_extensions/plugins/nondecoplugin/__init__.py b/test/core/metaflow_extensions/plugins/nondecoplugin/__init__.py index ea8fe1156e1..e2cad733295 100644 --- a/test/core/metaflow_extensions/plugins/nondecoplugin/__init__.py +++ b/test/core/metaflow_extensions/plugins/nondecoplugin/__init__.py @@ -1 +1 @@ -my_value = 42 \ No newline at end of file +my_value = 42 diff --git a/test/core/metaflow_extensions/plugins/test_step_decorator.py b/test/core/metaflow_extensions/plugins/test_step_decorator.py index b83b21189d7..7e7de606c75 100644 --- a/test/core/metaflow_extensions/plugins/test_step_decorator.py +++ b/test/core/metaflow_extensions/plugins/test_step_decorator.py @@ -2,7 +2,9 @@ class TestStepDecorator(StepDecorator): - name = 'test_step_decorator' + name = "test_step_decorator" - def task_post_step(self, step_name, flow, graph, retry_count, max_user_code_retries): + def task_post_step( + self, step_name, flow, graph, retry_count, max_user_code_retries + ): flow.plugin_set_value = step_name diff --git a/test/core/metaflow_extensions/toplevel/toplevel.py b/test/core/metaflow_extensions/toplevel/toplevel.py index a8a47dc3e15..52f6dc38fa1 100644 --- a/test/core/metaflow_extensions/toplevel/toplevel.py +++ b/test/core/metaflow_extensions/toplevel/toplevel.py @@ -1,5 +1,5 @@ -__mf_extensions__ = 'test' +__mf_extensions__ = "test" tl_value = 42 -__version__ = None \ No newline at end of file +__version__ = None diff --git a/test/core/metaflow_test/__init__.py b/test/core/metaflow_test/__init__.py index e47e82c55cf..02ddacd6465 100644 --- a/test/core/metaflow_test/__init__.py +++ b/test/core/metaflow_test/__init__.py @@ -3,6 +3,7 @@ from metaflow.exception import MetaflowException from metaflow import current + def steps(prio, quals, required=False): def wrapper(f): f.is_step = True @@ -11,55 +12,69 @@ def wrapper(f): f.required = required f.tags = [] return f + return wrapper + def tag(tagspec, **kwargs): def wrapper(f): f.tags.append(tagspec) return f + return wrapper + def truncate(var): var = str(var) if len(var) > 500: - var = '%s...' % var[:500] + var = "%s..." % var[:500] return var + class AssertArtifactFailed(Exception): pass + class AssertLogFailed(Exception): pass -class ExpectationFailed(Exception): +class ExpectationFailed(Exception): def __init__(self, expected, got): - super(ExpectationFailed, self).__init__("Expected result: %s, got %s"\ - % (truncate(expected), - truncate(got))) + super(ExpectationFailed, self).__init__( + "Expected result: %s, got %s" % (truncate(expected), truncate(got)) + ) + class ResumeFromHere(MetaflowException): headline = "Resume requested" + def __init__(self): - super(ResumeFromHere, self).__init__("This is not an error. " - "Testing resume...") + super(ResumeFromHere, self).__init__( + "This is not an error. " "Testing resume..." + ) + class TestRetry(MetaflowException): headline = "Testing retry" + def __init__(self): - super(TestRetry, self).__init__("This is not an error. " - "Testing retry...") + super(TestRetry, self).__init__("This is not an error. " "Testing retry...") + def is_resumed(): return current.origin_run_id is not None + def origin_run_id_for_resume(): return current.origin_run_id + def assert_equals(expected, got): if expected != got: raise ExpectationFailed(expected, got) + def assert_exception(func, exception): try: func() @@ -68,7 +83,8 @@ def assert_exception(func, exception): except Exception as ex: raise ExpectationFailed(exception, ex) else: - raise ExpectationFailed(exception, 'no exception') + raise ExpectationFailed(exception, "no exception") + class MetaflowTest(object): PRIORITY = 999999999 @@ -79,8 +95,8 @@ class MetaflowTest(object): def check_results(self, flow, checker): return False -class MetaflowCheck(object): +class MetaflowCheck(object): def __init__(self, flow): pass @@ -104,11 +120,13 @@ def artifact_dict(step, name): def assert_log(self, step, logtype, value, exact_match=True): raise NotImplementedError() + def new_checker(flow): from . import cli_check, metadata_check + CHECKER = { - 'CliCheck': cli_check.CliCheck, - 'MetadataCheck': metadata_check.MetadataCheck + "CliCheck": cli_check.CliCheck, + "MetadataCheck": metadata_check.MetadataCheck, } CLASSNAME = sys.argv[1] return CHECKER[CLASSNAME](flow) diff --git a/test/core/metaflow_test/cli_check.py b/test/core/metaflow_test/cli_check.py index 7c46785c617..078a5d47f51 100644 --- a/test/core/metaflow_test/cli_check.py +++ b/test/core/metaflow_test/cli_check.py @@ -15,14 +15,14 @@ # Python 3 import pickle -class CliCheck(MetaflowCheck): +class CliCheck(MetaflowCheck): def run_cli(self, args, capture_output=False): - cmd = [sys.executable, 'test_flow.py'] + cmd = [sys.executable, "test_flow.py"] # remove --quiet from top level options to capture output from echo # we will add --quiet in args if needed - cmd.extend([opt for opt in self.cli_options if opt != '--quiet']) + cmd.extend([opt for opt in self.cli_options if opt != "--quiet"]) cmd.extend(args) @@ -43,53 +43,62 @@ def assert_artifact(self, step, name, value, fields=None): data = artifact if not isinstance(data, dict): raise AssertArtifactFailed( - "Task '%s' expected %s to be a dictionary (got %s)" % - (task, name, type(data))) + "Task '%s' expected %s to be a dictionary (got %s)" + % (task, name, type(data)) + ) if data.get(field, None) != v: raise AssertArtifactFailed( - "Task '%s' expected %s[%s]=%r but got %s[%s]=%s" % - (task, name, field, truncate(value), name, field, - truncate(data[field]))) + "Task '%s' expected %s[%s]=%r but got %s[%s]=%s" + % ( + task, + name, + field, + truncate(value), + name, + field, + truncate(data[field]), + ) + ) elif artifact != value: raise AssertArtifactFailed( - "Task '%s' expected %s=%r but got %s=%s" % - (task, name, truncate(value), name, truncate(artifact))) + "Task '%s' expected %s=%r but got %s=%s" + % (task, name, truncate(value), name, truncate(artifact)) + ) else: - raise AssertArtifactFailed("Task '%s' expected %s=%s but " - "the key was not found" %\ - (task, name, truncate(value))) + raise AssertArtifactFailed( + "Task '%s' expected %s=%s but " + "the key was not found" % (task, name, truncate(value)) + ) return True def artifact_dict(self, step, name): - with NamedTemporaryFile(dir='.') as tmp: - cmd = ['dump', - '--max-value-size', '100000000000', - '--private', - '--include', name, - '--file', tmp.name, - '%s/%s' % (self.run_id, step)] + with NamedTemporaryFile(dir=".") as tmp: + cmd = [ + "dump", + "--max-value-size", + "100000000000", + "--private", + "--include", + name, + "--file", + tmp.name, + "%s/%s" % (self.run_id, step), + ] self.run_cli(cmd) - with open(tmp.name, 'rb') as f: + with open(tmp.name, "rb") as f: # if the step had multiple tasks, this will fail return pickle.load(f) def assert_log(self, step, logtype, value, exact_match=True): log = self.get_log(step, logtype) - if (exact_match and log != value) or\ - (not exact_match and value not in log): + if (exact_match and log != value) or (not exact_match and value not in log): raise AssertLogFailed( - "Task '%s/%s' expected %s log '%s' but got '%s'" %\ - (self.run_id, - step, - logtype, - repr(value), - repr(log))) + "Task '%s/%s' expected %s log '%s' but got '%s'" + % (self.run_id, step, logtype, repr(value), repr(log)) + ) return True - + def get_log(self, step, logtype): - cmd = ['--quiet', - 'logs', - '--%s' % logtype, - '%s/%s' % (self.run_id, step)] - return self.run_cli(cmd, capture_output=True).decode('utf-8') \ No newline at end of file + cmd = ["--quiet", "logs", "--%s" % logtype, "%s/%s" % (self.run_id, step)] + return self.run_cli(cmd, capture_output=True).decode("utf-8") diff --git a/test/core/metaflow_test/formatter.py b/test/core/metaflow_test/formatter.py index e4a202fb6db..2c56f306d19 100644 --- a/test/core/metaflow_test/formatter.py +++ b/test/core/metaflow_test/formatter.py @@ -3,14 +3,14 @@ INDENT = 4 -class FlowFormatter(object): +class FlowFormatter(object): def __init__(self, graphspec, test): self.graphspec = graphspec self.test = test - self.should_resume = getattr(test, 'RESUME', False) - self.should_fail = getattr(test, 'SHOULD_FAIL', False) - self.flow_name = '%sFlow' % self.test.__class__.__name__ + self.should_resume = getattr(test, "RESUME", False) + self.should_fail = getattr(test, "SHOULD_FAIL", False) + self.flow_name = "%sFlow" % self.test.__class__.__name__ self.used = set() self._code_cache = {} self.steps = self._index_steps(test) @@ -23,13 +23,12 @@ def __init__(self, graphspec, test): self.valid = False def _format_method(self, step): - def lines(): lines, lineno = inspect.getsourcelines(step) lines_iter = iter(lines) for line in lines_iter: head = line.lstrip() - if head and (head[0] == '@' or head.startswith('def ')): + if head and (head[0] == "@" or head.startswith("def ")): continue first_line = line break @@ -47,20 +46,20 @@ def _index_steps(self, test): steps = [] for attr in dir(test): obj = getattr(test, attr) - if hasattr(obj, 'is_step'): + if hasattr(obj, "is_step"): steps.append(obj) return list(sorted(steps, key=lambda x: x.prio)) def _node_quals(self, name, node): - quals = {'all'} - quals.update(node.get('quals', [])) - if name in ('start', 'end'): + quals = {"all"} + quals.update(node.get("quals", [])) + if name in ("start", "end"): quals.add(name) - if 'join' in node: - quals.add('join') - if 'linear' in node: - quals.add('linear') - for qual in node.get('quals', []): + if "join" in node: + quals.add("join") + if "linear" in node: + quals.add("linear") + for qual in node.get("quals", []): quals.add(qual) return quals @@ -69,99 +68,95 @@ def _choose_step(self, name, node): for step in self.steps: if step.quals & node_quals: return step - raise Exception("Test %s doesn't have a match for step %s in graph %s"\ - % (self.test, name, self.graphspec['name'])) + raise Exception( + "Test %s doesn't have a match for step %s in graph %s" + % (self.test, name, self.graphspec["name"]) + ) def _flow_lines(self): tags = [] for step in self.steps: - tags.extend(tag.split('(')[0] for tag in step.tags) - - yield 0, '# -*- coding: utf-8 -*-' - yield 0, 'from metaflow import FlowSpec, step, Parameter, project, IncludeFile, JSONType, current' - yield 0, 'from metaflow_test import assert_equals, '\ - 'assert_exception, '\ - 'ExpectationFailed, '\ - 'is_resumed, '\ - 'ResumeFromHere, '\ - 'TestRetry' + tags.extend(tag.split("(")[0] for tag in step.tags) + + yield 0, "# -*- coding: utf-8 -*-" + yield 0, "from metaflow import FlowSpec, step, Parameter, project, IncludeFile, JSONType, current" + yield 0, "from metaflow_test import assert_equals, " "assert_exception, " "ExpectationFailed, " "is_resumed, " "ResumeFromHere, " "TestRetry" if tags: - yield 0, 'from metaflow import %s' % ','.join(tags) + yield 0, "from metaflow import %s" % ",".join(tags) yield 0, self.test.HEADER - yield 0, 'class %s(FlowSpec):' % self.flow_name + yield 0, "class %s(FlowSpec):" % self.flow_name for var, parameter in self.test.PARAMETERS.items(): - kwargs = ['%s=%s' % (k, v) for k, v in parameter.items()] - yield 1, '%s = Parameter("%s", %s)' % (var, var, ','.join(kwargs)) + kwargs = ["%s=%s" % (k, v) for k, v in parameter.items()] + yield 1, '%s = Parameter("%s", %s)' % (var, var, ",".join(kwargs)) for var, include in self.test.INCLUDE_FILES.items(): - kwargs = ['%s=%s' % (k, v) for k, v in include.items()] - yield 1, '%s = IncludeFile("%s", %s)' % (var, var, ','.join(kwargs)) + kwargs = ["%s=%s" % (k, v) for k, v in include.items()] + yield 1, '%s = IncludeFile("%s", %s)' % (var, var, ",".join(kwargs)) - for name, node in self.graphspec['graph'].items(): + for name, node in self.graphspec["graph"].items(): step = self._choose_step(name, node) self.used.add(step) for tagspec in step.tags: - yield 1, '@%s' % tagspec - yield 1, '@step' + yield 1, "@%s" % tagspec + yield 1, "@step" - if 'join' in node: - yield 1, 'def %s(self, inputs):' % name + if "join" in node: + yield 1, "def %s(self, inputs):" % name else: - yield 1, 'def %s(self):' % name + yield 1, "def %s(self):" % name - if 'foreach' in node: - yield 2, 'self.%s = %s' % (node['foreach_var'], - node['foreach_var_default']) + if "foreach" in node: + yield 2, "self.%s = %s" % ( + node["foreach_var"], + node["foreach_var_default"], + ) for line in self._format_method(step): yield 2, line - if 'linear' in node: - yield 2, 'self.next(self.%s)' % node['linear'] - elif 'branch' in node: - branches = ','.join('self.%s' % x for x in node['branch']) - yield 2, 'self.next(%s)' % branches - elif 'foreach' in node: - yield 2, 'self.next(self.%s, foreach="%s")' %\ - (node['foreach'], node['foreach_var']) + if "linear" in node: + yield 2, "self.next(self.%s)" % node["linear"] + elif "branch" in node: + branches = ",".join("self.%s" % x for x in node["branch"]) + yield 2, "self.next(%s)" % branches + elif "foreach" in node: + yield 2, 'self.next(self.%s, foreach="%s")' % ( + node["foreach"], + node["foreach_var"], + ) yield 0, "if __name__ == '__main__':" - yield 1, '%s()' % self.flow_name + yield 1, "%s()" % self.flow_name def _check_lines(self): - yield 0, '# -*- coding: utf-8 -*-' - yield 0, 'from coverage import Coverage' - yield 0, 'cov = Coverage(data_suffix=True, '\ - 'auto_data=True, '\ - 'branch=True, '\ - 'omit=["check_flow.py", '\ - '"test_flow.py", '\ - '"*/click/*", '\ - '"*/site-packages/*", '\ - '"*/core/metaflow_extensions/*", '\ - '"*/core/metaflow_test/*"])' - yield 0, 'cov.start()' - yield 0, 'import sys' - yield 0, 'from metaflow_test import assert_equals, new_checker' - yield 0, 'def check_results(flow, checker):' + yield 0, "# -*- coding: utf-8 -*-" + yield 0, "from coverage import Coverage" + yield 0, "cov = Coverage(data_suffix=True, " "auto_data=True, " "branch=True, " 'omit=["check_flow.py", ' '"test_flow.py", ' '"*/click/*", ' '"*/site-packages/*", ' '"*/core/metaflow_extensions/*", ' '"*/core/metaflow_test/*"])' + yield 0, "cov.start()" + yield 0, "import sys" + yield 0, "from metaflow_test import assert_equals, new_checker" + yield 0, "def check_results(flow, checker):" for line in self._format_method(self.test.check_results): yield 1, line yield 0, "if __name__ == '__main__':" - yield 1, 'from test_flow import %s' % self.flow_name - yield 1, 'flow = %s(use_cli=False)' % self.flow_name - yield 1, 'check = new_checker(flow)' - yield 1, 'check_results(flow, check)' + yield 1, "from test_flow import %s" % self.flow_name + yield 1, "flow = %s(use_cli=False)" % self.flow_name + yield 1, "check = new_checker(flow)" + yield 1, "check_results(flow, check)" def _pretty_print(self, lines): def _lines(): for indent, line in lines: - yield ''.join((' ' * (indent * INDENT), line)) - return '\n'.join(_lines()) + yield "".join((" " * (indent * INDENT), line)) + + return "\n".join(_lines()) def __str__(self): - return "test '%s' graph '%s'" % (self.test.__class__.__name__, - self.graphspec['name']) + return "test '%s' graph '%s'" % ( + self.test.__class__.__name__, + self.graphspec["name"], + ) diff --git a/test/core/metaflow_test/metadata_check.py b/test/core/metaflow_test/metadata_check.py index cc9d0bfd21e..f7c82163027 100644 --- a/test/core/metaflow_test/metadata_check.py +++ b/test/core/metaflow_test/metadata_check.py @@ -1,37 +1,43 @@ import json from metaflow.util import is_stringish -from . import MetaflowCheck, AssertArtifactFailed, AssertLogFailed, assert_equals, assert_exception, truncate +from . import ( + MetaflowCheck, + AssertArtifactFailed, + AssertLogFailed, + assert_equals, + assert_exception, + truncate, +) -class MetadataCheck(MetaflowCheck): +class MetadataCheck(MetaflowCheck): def __init__(self, flow): from metaflow.client import Flow, get_namespace + self.flow = flow self.run = Flow(flow.name)[self.run_id] - assert_equals(sorted(step.name for step in flow), - sorted(step.id for step in self.run)) + assert_equals( + sorted(step.name for step in flow), sorted(step.id for step in self.run) + ) self._test_namespace() def _test_namespace(self): - from metaflow.client import Flow,\ - get_namespace,\ - namespace,\ - default_namespace + from metaflow.client import Flow, get_namespace, namespace, default_namespace from metaflow.exception import MetaflowNamespaceMismatch import os + # test 1) METAFLOW_USER should be the default - assert_equals('user:%s' % os.environ.get('METAFLOW_USER'), - get_namespace()) + assert_equals("user:%s" % os.environ.get("METAFLOW_USER"), get_namespace()) # test 2) Run should be in the listing - assert_equals(True, - self.run_id in [run.id for run in Flow(self.flow.name)]) + assert_equals(True, self.run_id in [run.id for run in Flow(self.flow.name)]) # test 3) changing namespace should change namespace - namespace('user:nobody') - assert_equals(get_namespace(), 'user:nobody') + namespace("user:nobody") + assert_equals(get_namespace(), "user:nobody") # test 4) fetching results in the incorrect namespace should fail - assert_exception(lambda: Flow(self.flow.name)[self.run_id], - MetaflowNamespaceMismatch) + assert_exception( + lambda: Flow(self.flow.name)[self.run_id], MetaflowNamespaceMismatch + ) # test 5) global namespace should work namespace(None) assert_equals(get_namespace(), None) @@ -53,21 +59,32 @@ def assert_artifact(self, step, name, value, fields=None): data = artifact if not isinstance(data, dict): raise AssertArtifactFailed( - "Task '%s' expected %s to be a dictionary (got %s)" % - (task, name, type(data))) + "Task '%s' expected %s to be a dictionary (got %s)" + % (task, name, type(data)) + ) if data.get(field, None) != v: raise AssertArtifactFailed( - "Task '%s' expected %s[%s]=%r but got %s[%s]=%s" % - (task, name, field, truncate(value), name, field, - truncate(data[field]))) + "Task '%s' expected %s[%s]=%r but got %s[%s]=%s" + % ( + task, + name, + field, + truncate(value), + name, + field, + truncate(data[field]), + ) + ) elif artifact != value: raise AssertArtifactFailed( - "Task '%s' expected %s=%r but got %s=%s" % - (task, name, truncate(value), name, truncate(artifact))) + "Task '%s' expected %s=%r but got %s=%s" + % (task, name, truncate(value), name, truncate(artifact)) + ) else: - raise AssertArtifactFailed("Task '%s' expected %s=%s but " - "the key was not found" %\ - (task, name, truncate(value))) + raise AssertArtifactFailed( + "Task '%s' expected %s=%s but " + "the key was not found" % (task, name, truncate(value)) + ) return True def artifact_dict(self, step, name): @@ -80,12 +97,10 @@ def assert_log(self, step, logtype, value, exact_match=True): elif not exact_match and value in log_value: return True else: - raise AssertLogFailed("Step '%s' expected task.%s='%s' but got task.%s='%s'" %\ - (step, - logtype, - repr(value), - logtype, - repr(log_value))) + raise AssertLogFailed( + "Step '%s' expected task.%s='%s' but got task.%s='%s'" + % (step, logtype, repr(value), logtype, repr(log_value)) + ) def get_log(self, step, logtype): - return ''.join(getattr(task, logtype) for task in self.run[step]) + return "".join(getattr(task, logtype) for task in self.run[step]) diff --git a/test/core/run_tests.py b/test/core/run_tests.py index 3e9eb365aa6..8fe7f4442a8 100644 --- a/test/core/run_tests.py +++ b/test/core/run_tests.py @@ -14,76 +14,84 @@ from metaflow_test import MetaflowTest from metaflow_test.formatter import FlowFormatter + def iter_graphs(): - root = os.path.join(os.path.dirname(__file__), 'graphs') + root = os.path.join(os.path.dirname(__file__), "graphs") for graphfile in os.listdir(root): - if graphfile.endswith('.json') and not graphfile[0] == '.': + if graphfile.endswith(".json") and not graphfile[0] == ".": with open(os.path.join(root, graphfile)) as f: yield json.load(f) + def iter_tests(): - root = os.path.join(os.path.dirname(__file__), 'tests') + root = os.path.join(os.path.dirname(__file__), "tests") sys.path.insert(0, root) for testfile in os.listdir(root): - if testfile.endswith('.py') and not testfile[0] == '.': - mod = importlib.import_module(testfile[:-3], 'metaflow_test') + if testfile.endswith(".py") and not testfile[0] == ".": + mod = importlib.import_module(testfile[:-3], "metaflow_test") for name in dir(mod): obj = getattr(mod, name) - if name != 'MetaflowTest' and\ - isinstance(obj, type) and\ - issubclass(obj, MetaflowTest): + if ( + name != "MetaflowTest" + and isinstance(obj, type) + and issubclass(obj, MetaflowTest) + ): yield obj() + def log(msg, formatter=None, context=None, real_bad=False, real_good=False): - cstr = '' - fstr = '' + cstr = "" + fstr = "" if context: - cstr = " in context '%s'" % context['name'] + cstr = " in context '%s'" % context["name"] if formatter: - fstr = ' %s' % formatter + fstr = " %s" % formatter if cstr or fstr: line = "###%s%s: %s ###" % (fstr, cstr, msg) else: line = "### %s ###" % msg if real_bad: - line = click.style(line, fg='red', bold=True) + line = click.style(line, fg="red", bold=True) elif real_good: - line = click.style(line, fg='green', bold=True) + line = click.style(line, fg="green", bold=True) else: - line = click.style(line, fg='white', bold=True) + line = click.style(line, fg="white", bold=True) pid = os.getpid() - click.echo('[pid %s] %s' % (pid, line)) + click.echo("[pid %s] %s" % (pid, line)) + def copy_coverage_files(dstdir): - for fname in glob.glob('.coverage.*'): + for fname in glob.glob(".coverage.*"): shutil.copy(fname, dstdir) -def run_test(formatter, context, coverage_dir, debug, checks, env_base): +def run_test(formatter, context, coverage_dir, debug, checks, env_base): def run_cmd(mode): - cmd = [context['python'], '-B', 'test_flow.py'] - cmd.extend(context['top_options']) - cmd.extend((mode, '--run-id-file', 'run-id')) - cmd.extend(context['run_options']) + cmd = [context["python"], "-B", "test_flow.py"] + cmd.extend(context["top_options"]) + cmd.extend((mode, "--run-id-file", "run-id")) + cmd.extend(context["run_options"]) return cmd cwd = os.getcwd() - tempdir = tempfile.mkdtemp('_metaflow_test') + tempdir = tempfile.mkdtemp("_metaflow_test") package = os.path.dirname(os.path.abspath(__file__)) try: # write scripts os.chdir(tempdir) - with open('test_flow.py', 'w') as f: + with open("test_flow.py", "w") as f: f.write(formatter.flow_code) - with open('check_flow.py', 'w') as f: + with open("check_flow.py", "w") as f: f.write(formatter.check_code) - with open('.coveragerc', 'w') as f: + with open(".coveragerc", "w") as f: f.write("[run]\ndisable_warnings = module-not-measured\n") - shutil.copytree(os.path.join(cwd, "metaflow_test"), os.path.join(tempdir, "metaflow_test")) + shutil.copytree( + os.path.join(cwd, "metaflow_test"), os.path.join(tempdir, "metaflow_test") + ) - path = os.path.join(tempdir, 'test_flow.py') + path = os.path.join(tempdir, "test_flow.py") env = {} env.update(env_base) @@ -91,64 +99,73 @@ def run_cmd(mode): # nonce can be used to insert entropy in env vars. # This is useful e.g. for separating S3 paths of # runs, which may have clashing run_ids - env.update(dict((k, v.format(nonce=str(uuid.uuid4()))) - for k, v in context['env'].items())) - - pythonpath = os.environ.get('PYTHONPATH', '.') - env.update({'LANG': 'C.UTF-8', - 'LC_ALL': 'C.UTF-8', - 'PATH': os.environ.get('PATH', '.'), - 'PYTHONIOENCODING': 'utf_8', - 'PYTHONPATH': "%s:%s" % (package, pythonpath)}) - - if 'pre_command' in context: - if context['pre_command'].get('metaflow_command'): - cmd = [context['python'], 'test_flow.py'] - cmd.extend(context['top_options']) - cmd.extend(context['pre_command']['command']) + env.update( + dict( + (k, v.format(nonce=str(uuid.uuid4()))) + for k, v in context["env"].items() + ) + ) + + pythonpath = os.environ.get("PYTHONPATH", ".") + env.update( + { + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": os.environ.get("PATH", "."), + "PYTHONIOENCODING": "utf_8", + "PYTHONPATH": "%s:%s" % (package, pythonpath), + } + ) + + if "pre_command" in context: + if context["pre_command"].get("metaflow_command"): + cmd = [context["python"], "test_flow.py"] + cmd.extend(context["top_options"]) + cmd.extend(context["pre_command"]["command"]) else: - cmd = context['pre_command']['command'] + cmd = context["pre_command"]["command"] pre_ret = subprocess.call(cmd, env=env) - if pre_ret and not context['pre_command'].get('ignore_errors', - False): + if pre_ret and not context["pre_command"].get("ignore_errors", False): log("pre-command failed", formatter, context) return pre_ret, path # run flow - flow_ret = subprocess.call(run_cmd('run'), env=env) + flow_ret = subprocess.call(run_cmd("run"), env=env) if flow_ret: if formatter.should_fail: log("Flow failed as expected.") elif formatter.should_resume: log("Resuming flow", formatter, context) - flow_ret = subprocess.call(run_cmd('resume'), env=env) + flow_ret = subprocess.call(run_cmd("resume"), env=env) else: log("flow failed", formatter, context) return flow_ret, path elif formatter.should_fail: - log("The flow should have failed but it didn't. Error!", - formatter, - context) + log("The flow should have failed but it didn't. Error!", formatter, context) return 1, path # check results - run_id = open('run-id').read() + run_id = open("run-id").read() ret = 0 - for check_name in context['checks']: + for check_name in context["checks"]: check = checks[check_name] - python = check['python'] - cmd = [python, 'check_flow.py', check['class'], run_id] - cmd.extend(context['top_options']) + python = check["python"] + cmd = [python, "check_flow.py", check["class"], run_id] + cmd.extend(context["top_options"]) check_ret = subprocess.call(cmd, env=env) if check_ret: - log("checker '%s' says that results failed" % check_name, + log( + "checker '%s' says that results failed" % check_name, formatter, - context) + context, + ) ret = check_ret else: - log("checker '%s' says that results are ok" % check_name, + log( + "checker '%s' says that results are ok" % check_name, formatter, - context) + context, + ) # copy coverage files if coverage_dir: @@ -159,16 +176,16 @@ def run_cmd(mode): if not debug: shutil.rmtree(tempdir) -def run_all(ok_tests, - ok_contexts, - ok_graphs, - coverage_dir, - debug, - num_parallel, - inherit_env): - tests = [test for test in sorted(iter_tests(), key=lambda x: x.PRIORITY)\ - if not ok_tests or test.__class__.__name__.lower() in ok_tests] +def run_all( + ok_tests, ok_contexts, ok_graphs, coverage_dir, debug, num_parallel, inherit_env +): + + tests = [ + test + for test in sorted(iter_tests(), key=lambda x: x.PRIORITY) + if not ok_tests or test.__class__.__name__.lower() in ok_tests + ] failed = [] if inherit_env: @@ -178,59 +195,62 @@ def run_all(ok_tests, if debug or num_parallel is None: for test in tests: - failed.extend(run_test_cases((test, - ok_contexts, - ok_graphs, - coverage_dir, - debug, - base_env))) + failed.extend( + run_test_cases( + (test, ok_contexts, ok_graphs, coverage_dir, debug, base_env) + ) + ) else: - args = [(test, ok_contexts, ok_graphs, coverage_dir, debug, base_env) - for test in tests] + args = [ + (test, ok_contexts, ok_graphs, coverage_dir, debug, base_env) + for test in tests + ] for fail in Pool(num_parallel).imap_unordered(run_test_cases, args): failed.extend(fail) return failed + def run_test_cases(args): test, ok_contexts, ok_graphs, coverage_dir, debug, base_env = args - contexts = json.load(open('contexts.json')) + contexts = json.load(open("contexts.json")) graphs = list(iter_graphs()) test_name = test.__class__.__name__ - log('Loaded test %s' % test_name) + log("Loaded test %s" % test_name) failed = [] for graph in graphs: - if ok_graphs and graph['name'].lower() not in ok_graphs: + if ok_graphs and graph["name"].lower() not in ok_graphs: continue formatter = FlowFormatter(graph, test) if formatter.valid: - for context in contexts['contexts']: + for context in contexts["contexts"]: if ok_contexts: - if context['name'].lower() not in ok_contexts: + if context["name"].lower() not in ok_contexts: continue - elif context.get('disabled', False): + elif context.get("disabled", False): continue - if (test_name in map(str, context.get('disabled_tests', []))): + if test_name in map(str, context.get("disabled_tests", [])): continue - enabled_tests = context.get('enabled_tests', []) + enabled_tests = context.get("enabled_tests", []) if enabled_tests and (test_name not in map(str, enabled_tests)): continue log("running", formatter, context) - ret, path = run_test(formatter, - context, - coverage_dir, - debug, - contexts['checks'], - base_env) + ret, path = run_test( + formatter, + context, + coverage_dir, + debug, + contexts["checks"], + base_env, + ) if ret: - tstid = '%s in context %s' % (formatter, - context['name']) + tstid = "%s in context %s" % (formatter, context["name"]) failed.append((tstid, path)) log("failed", formatter, context, real_bad=True) if debug: @@ -241,85 +261,100 @@ def run_test_cases(args): log("not a valid combination. Skipped.", formatter) return failed + def produce_coverage_report(coverage_dir, coverage_output): - COVERAGE = sys.executable + ' -m coverage ' + COVERAGE = sys.executable + " -m coverage " cwd = os.getcwd() try: os.chdir(coverage_dir) - if os.listdir('.'): - subprocess.check_call(COVERAGE + 'combine .coverage*', shell=True) - subprocess.check_call(COVERAGE + 'xml -o %s.xml' % coverage_output, - shell=True) - subprocess.check_call(COVERAGE + 'html -d %s' % coverage_output, - shell=True) - log("Coverage report written to %s" % coverage_output, - real_good=True) + if os.listdir("."): + subprocess.check_call(COVERAGE + "combine .coverage*", shell=True) + subprocess.check_call( + COVERAGE + "xml -o %s.xml" % coverage_output, shell=True + ) + subprocess.check_call(COVERAGE + "html -d %s" % coverage_output, shell=True) + log("Coverage report written to %s" % coverage_output, real_good=True) else: log("No coverage data was produced", real_bad=True) finally: os.chdir(cwd) + @click.command(help="Run tests") -@click.option("--contexts", - default='', - type=str, - help="A comma-separated list of contexts to include (default: all).") -@click.option("--tests", - default='', - type=str, - help="A comma-separate list of graphs to include (default: all).") -@click.option("--graphs", - default='', - type=str, - help="A comma-separate list of graphs to include (default: all).") -@click.option("--coverage-output", - default=None, - type=str, - show_default=True, - help="Output prefix for the coverage reports (default: None)") -@click.option("--debug", - is_flag=True, - default=False, - help="Debug mode: Stop at the first failure, " - "don't delete test directory") -@click.option("--inherit-env", - is_flag=True, - default=False, - help="Inherit env variables") -@click.option("--num-parallel", - show_default=True, - default=None, - type=int, - help="Number of parallel tests to run. By default, " - "tests are run sequentially.") -def cli(tests=None, - contexts=None, - graphs=None, - coverage_output=None, - num_parallel=None, - debug=False, - inherit_env=False): - - parse = lambda x: {t.lower() for t in x.split(',') if t} - coverage_dir = tempfile.mkdtemp('_metaflow_test_coverage') if coverage_output else None +@click.option( + "--contexts", + default="", + type=str, + help="A comma-separated list of contexts to include (default: all).", +) +@click.option( + "--tests", + default="", + type=str, + help="A comma-separate list of graphs to include (default: all).", +) +@click.option( + "--graphs", + default="", + type=str, + help="A comma-separate list of graphs to include (default: all).", +) +@click.option( + "--coverage-output", + default=None, + type=str, + show_default=True, + help="Output prefix for the coverage reports (default: None)", +) +@click.option( + "--debug", + is_flag=True, + default=False, + help="Debug mode: Stop at the first failure, " "don't delete test directory", +) +@click.option( + "--inherit-env", is_flag=True, default=False, help="Inherit env variables" +) +@click.option( + "--num-parallel", + show_default=True, + default=None, + type=int, + help="Number of parallel tests to run. By default, " "tests are run sequentially.", +) +def cli( + tests=None, + contexts=None, + graphs=None, + coverage_output=None, + num_parallel=None, + debug=False, + inherit_env=False, +): + + parse = lambda x: {t.lower() for t in x.split(",") if t} + coverage_dir = ( + tempfile.mkdtemp("_metaflow_test_coverage") if coverage_output else None + ) try: - failed = run_all(parse(tests), - parse(contexts), - parse(graphs), - coverage_dir, - debug, - num_parallel, - inherit_env) + failed = run_all( + parse(tests), + parse(contexts), + parse(graphs), + coverage_dir, + debug, + num_parallel, + inherit_env, + ) if coverage_output and not debug: - produce_coverage_report(coverage_dir, - os.path.abspath(coverage_output)) + produce_coverage_report(coverage_dir, os.path.abspath(coverage_output)) if failed: log("The following tests failed:") for fail, path in failed: if debug: - log('%s (path %s)' % (fail, path), real_bad=True) + log("%s (path %s)" % (fail, path), real_bad=True) else: log(fail, real_bad=True) sys.exit(1) @@ -330,5 +365,6 @@ def cli(tests=None, if coverage_dir: shutil.rmtree(coverage_dir) -if __name__ == '__main__': + +if __name__ == "__main__": cli() diff --git a/test/core/tests/basic_artifact.py b/test/core/tests/basic_artifact.py index 5db4ee04698..daf5c070df8 100644 --- a/test/core/tests/basic_artifact.py +++ b/test/core/tests/basic_artifact.py @@ -1,27 +1,30 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class BasicArtifactTest(MetaflowTest): """ Test that an artifact defined in the first step is available in all steps downstream. """ + PRIORITY = 0 - @steps(0, ['start']) + @steps(0, ["start"]) def step_start(self): - self.data = 'abc' + self.data = "abc" - @steps(1, ['join']) + @steps(1, ["join"]) def step_join(self): import metaflow_test + inputset = {inp.data for inp in inputs} - assert_equals({'abc'}, inputset) + assert_equals({"abc"}, inputset) self.data = list(inputset)[0] - @steps(2, ['all']) + @steps(2, ["all"]) def step_all(self): pass def check_results(self, flow, checker): for step in flow: - checker.assert_artifact(step.name, 'data', 'abc') + checker.assert_artifact(step.name, "data", "abc") diff --git a/test/core/tests/basic_foreach.py b/test/core/tests/basic_foreach.py index e40c8812e41..e1d75dda36a 100644 --- a/test/core/tests/basic_foreach.py +++ b/test/core/tests/basic_foreach.py @@ -1,14 +1,15 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class BasicForeachTest(MetaflowTest): PRIORITY = 0 - @steps(0, ['foreach-split'], required=True) + @steps(0, ["foreach-split"], required=True) def split(self): self.my_index = None self.arr = range(32) - @steps(0, ['foreach-inner'], required=True) + @steps(0, ["foreach-inner"], required=True) def inner(self): # index must stay constant over multiple steps inside foreach if self.my_index is None: @@ -17,11 +18,11 @@ def inner(self): assert_equals(self.input, self.arr[self.index]) self.my_input = self.input - @steps(0, ['foreach-join'], required=True) + @steps(0, ["foreach-join"], required=True) def join(self, inputs): got = sorted([inp.my_input for inp in inputs]) assert_equals(list(range(32)), got) - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): pass diff --git a/test/core/tests/basic_include.py b/test/core/tests/basic_include.py index fa9f0404b1d..e79bf653f7f 100644 --- a/test/core/tests/basic_include.py +++ b/test/core/tests/basic_include.py @@ -4,11 +4,11 @@ class BasicIncludeTest(MetaflowTest): PRIORITY = 1 INCLUDE_FILES = { - 'myfile_txt': {'default': "'./reg.txt'"}, - 'myfile_utf8': {'default': "'./utf8.txt'", 'encoding': "'utf8'"}, - 'myfile_binary': {'default': "'./utf8.txt'", 'is_text': False}, - 'myfile_overriden': {'default': "'./reg.txt'"}, - 'absent_file': {'required': False} + "myfile_txt": {"default": "'./reg.txt'"}, + "myfile_utf8": {"default": "'./utf8.txt'", "encoding": "'utf8'"}, + "myfile_binary": {"default": "'./utf8.txt'", "is_text": False}, + "myfile_overriden": {"default": "'./reg.txt'"}, + "absent_file": {"required": False}, } HEADER = """ import codecs @@ -23,12 +23,13 @@ class BasicIncludeTest(MetaflowTest): f.write("Override Text File") """ - @steps(0, ['all']) + @steps(0, ["all"]) def step_all(self): assert_equals("Regular Text File", self.myfile_txt) assert_equals(u"UTF Text File \u5e74", self.myfile_utf8) assert_equals( - u"UTF Text File \u5e74".encode(encoding='utf8'), self.myfile_binary) + u"UTF Text File \u5e74".encode(encoding="utf8"), self.myfile_binary + ) assert_equals("Override Text File", self.myfile_overriden) # Check that an absent file does not make things crash @@ -36,7 +37,7 @@ def step_all(self): try: # Include files should be immutable self.myfile_txt = 5 - raise ExpectationFailed(AttributeError, 'nothing') + raise ExpectationFailed(AttributeError, "nothing") except AttributeError: pass @@ -44,30 +45,25 @@ def check_results(self, flow, checker): for step in flow: checker.assert_artifact( step.name, - 'myfile_txt', + "myfile_txt", None, - fields={'type': 'uploader-v1', - 'is_text': True, - 'encoding': None}) + fields={"type": "uploader-v1", "is_text": True, "encoding": None}, + ) checker.assert_artifact( step.name, - 'myfile_utf8', + "myfile_utf8", None, - fields={'type': 'uploader-v1', - 'is_text': True, - 'encoding': 'utf8'}) + fields={"type": "uploader-v1", "is_text": True, "encoding": "utf8"}, + ) checker.assert_artifact( step.name, - 'myfile_binary', + "myfile_binary", None, - fields={'type': 'uploader-v1', - 'is_text': False, - 'encoding': None}) + fields={"type": "uploader-v1", "is_text": False, "encoding": None}, + ) checker.assert_artifact( step.name, - 'myfile_overriden', + "myfile_overriden", None, - fields={'type': 'uploader-v1', - 'is_text': True, - 'encoding': None}) - + fields={"type": "uploader-v1", "is_text": True, "encoding": None}, + ) diff --git a/test/core/tests/basic_log.py b/test/core/tests/basic_log.py index 99b2adea05f..c315ead8b4d 100644 --- a/test/core/tests/basic_log.py +++ b/test/core/tests/basic_log.py @@ -1,45 +1,52 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class BasicLogTest(MetaflowTest): """ Test that log messages emitted in the first step are saved and readable. """ + PRIORITY = 0 - @steps(0, ['singleton'], required=True) + @steps(0, ["singleton"], required=True) def step_single(self): import sys - msg1 = 'stdout: A regular message.\n' - msg2 = u'stdout: A message with unicode: \u5e74\n' + + msg1 = "stdout: A regular message.\n" + msg2 = u"stdout: A message with unicode: \u5e74\n" sys.stdout.write(msg1) if not sys.stdout.encoding: - sys.stdout.write(msg2.encode('utf8')) + sys.stdout.write(msg2.encode("utf8")) else: sys.stdout.write(msg2) - msg3 = 'stderr: A regular message.\n' - msg4 = u'stderr: A message with unicode: \u5e74\n' + msg3 = "stderr: A regular message.\n" + msg4 = u"stderr: A message with unicode: \u5e74\n" sys.stderr.write(msg3) if not sys.stderr.encoding: - sys.stderr.write(msg4.encode('utf8')) + sys.stderr.write(msg4.encode("utf8")) else: sys.stderr.write(msg4) - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): pass def check_results(self, flow, checker): - msg1 = 'stdout: A regular message.\n' - msg2 = u'stdout: A message with unicode: \u5e74\n' - stdout_combined_msg = ''.join([msg1, msg2, '']) + msg1 = "stdout: A regular message.\n" + msg2 = u"stdout: A message with unicode: \u5e74\n" + stdout_combined_msg = "".join([msg1, msg2, ""]) - msg3 = 'stderr: A regular message.\n' - msg4 = u'stderr: A message with unicode: \u5e74\n' - stderr_combined_msg = ''.join([msg3, msg4, '']) + msg3 = "stderr: A regular message.\n" + msg4 = u"stderr: A message with unicode: \u5e74\n" + stderr_combined_msg = "".join([msg3, msg4, ""]) for step in flow: - if step.name not in ['start', 'end']: - checker.assert_log(step.name, 'stdout', stdout_combined_msg, exact_match=False) - checker.assert_log(step.name, 'stderr', stderr_combined_msg, exact_match=False) + if step.name not in ["start", "end"]: + checker.assert_log( + step.name, "stdout", stdout_combined_msg, exact_match=False + ) + checker.assert_log( + step.name, "stderr", stderr_combined_msg, exact_match=False + ) diff --git a/test/core/tests/basic_parameters.py b/test/core/tests/basic_parameters.py index 2fafd89f409..7775dcfdef3 100644 --- a/test/core/tests/basic_parameters.py +++ b/test/core/tests/basic_parameters.py @@ -4,14 +4,14 @@ class BasicParameterTest(MetaflowTest): PRIORITY = 1 PARAMETERS = { - 'no_default_param': {'default': None}, + "no_default_param": {"default": None}, # Note this value is overridden in contexts.json - 'bool_param': {'default': False}, - 'bool_true_param': {'default': True}, - 'int_param': {'default': 123}, - 'str_param': {'default': "'foobar'"}, - 'list_param': {'separator': "','", 'default': '"a,b,c"'}, - 'json_param': {'default': """'{"a": [1,2,3]}'""", 'type': 'JSONType'} + "bool_param": {"default": False}, + "bool_true_param": {"default": True}, + "int_param": {"default": 123}, + "str_param": {"default": "'foobar'"}, + "list_param": {"separator": "','", "default": '"a,b,c"'}, + "json_param": {"default": """'{"a": [1,2,3]}'""", "type": "JSONType"}, } HEADER = """ import os @@ -19,29 +19,28 @@ class BasicParameterTest(MetaflowTest): os.environ['METAFLOW_RUN_BOOL_PARAM'] = 'False' """ - @steps(0, ['all']) + @steps(0, ["all"]) def step_all(self): assert_equals("test_str", self.no_default_param) assert_equals(False, self.bool_param) assert_equals(True, self.bool_true_param) assert_equals(123, self.int_param) - assert_equals('foobar', self.str_param) - assert_equals(['a', 'b', 'c'], self.list_param) - assert_equals({'a': [1, 2, 3]}, self.json_param) + assert_equals("foobar", self.str_param) + assert_equals(["a", "b", "c"], self.list_param) + assert_equals({"a": [1, 2, 3]}, self.json_param) try: # parameters should be immutable self.int_param = 5 - raise ExpectationFailed(AttributeError, 'nothing') + raise ExpectationFailed(AttributeError, "nothing") except AttributeError: pass def check_results(self, flow, checker): for step in flow: - checker.assert_artifact(step.name, 'no_default_param', "test_str") - checker.assert_artifact(step.name, 'bool_param', False) - checker.assert_artifact(step.name, 'bool_true_param', True) - checker.assert_artifact(step.name, 'int_param', 123) - checker.assert_artifact(step.name, 'str_param', 'foobar') - checker.assert_artifact(step.name, 'list_param', ['a', 'b', 'c']) - checker.assert_artifact(step.name, 'json_param', {'a': [1, 2, 3]}) - + checker.assert_artifact(step.name, "no_default_param", "test_str") + checker.assert_artifact(step.name, "bool_param", False) + checker.assert_artifact(step.name, "bool_true_param", True) + checker.assert_artifact(step.name, "int_param", 123) + checker.assert_artifact(step.name, "str_param", "foobar") + checker.assert_artifact(step.name, "list_param", ["a", "b", "c"]) + checker.assert_artifact(step.name, "json_param", {"a": [1, 2, 3]}) diff --git a/test/core/tests/basic_tags.py b/test/core/tests/basic_tags.py index 7cfffb19894..3b1d90bd244 100644 --- a/test/core/tests/basic_tags.py +++ b/test/core/tests/basic_tags.py @@ -1,24 +1,28 @@ # -*- coding: utf-8 -*- from metaflow_test import MetaflowTest, ExpectationFailed, steps + class BasicTagTest(MetaflowTest): """ Test that tags are assigned properly. """ + PRIORITY = 2 HEADER = "@project(name='basic_tag')" - @steps(0, ['all']) + @steps(0, ["all"]) def step_all(self): # TODO we could call self.tag() in some steps, once it is implemented from metaflow import get_namespace import os - user = 'user:%s' % os.environ.get('METAFLOW_USER') + + user = "user:%s" % os.environ.get("METAFLOW_USER") assert_equals(user, get_namespace()) def check_results(self, flow, checker): import os from metaflow import namespace + run = checker.get_run() if run is None: # CliChecker does not return a run object, that's ok @@ -26,11 +30,13 @@ def check_results(self, flow, checker): flow_obj = run.parent # test crazy unicode and spaces in tags # these tags must be set with --tag option in contexts.json - tags = (u'project:basic_tag', - u'project_branch:user.tester', - u'user:%s' % os.environ.get('METAFLOW_USER'), - u'刺身 means sashimi', - u'multiple tags should be ok') + tags = ( + u"project:basic_tag", + u"project_branch:user.tester", + u"user:%s" % os.environ.get("METAFLOW_USER"), + u"刺身 means sashimi", + u"multiple tags should be ok", + ) for tag in tags: # test different namespaces: one is a system-tag, # another is a user tag @@ -41,29 +47,35 @@ def check_results(self, flow, checker): # the run object should have the namespace tags assert_equals([True] * len(tags), [t in run.tags for t in tags]) # filtering by a non-existent tag should return nothing - assert_equals([], list(flow_obj.runs('not_a_tag'))) + assert_equals([], list(flow_obj.runs("not_a_tag"))) # a conjunction of a non-existent tag and an existent tag # should return nothing - assert_equals([], list(flow_obj.runs('not_a_tag', tag))) + assert_equals([], list(flow_obj.runs("not_a_tag", tag))) # all steps should be returned with tag filtering - assert_equals(frozenset(step.name for step in flow), - frozenset(step.id.split('/')[-1] for step in run.steps(tag))) + assert_equals( + frozenset(step.name for step in flow), + frozenset(step.id.split("/")[-1] for step in run.steps(tag)), + ) # a conjunction of two existent tags should return the original list - assert_equals(frozenset(step.name for step in flow), - frozenset(step.id.split('/')[-1] - for step in run.steps(*tags))) + assert_equals( + frozenset(step.name for step in flow), + frozenset(step.id.split("/")[-1] for step in run.steps(*tags)), + ) # all tasks should be returned with tag filtering for step in run: # the run object should have the tags assert_equals([True] * len(tags), [t in step.tags for t in tags]) # filtering by a non-existent tag should return nothing - assert_equals([], list(step.tasks('not_a_tag'))) + assert_equals([], list(step.tasks("not_a_tag"))) # filtering by the tag should not exclude any tasks - assert_equals([task.id for task in step], - [task.id for task in step.tasks(tag)]) + assert_equals( + [task.id for task in step], [task.id for task in step.tasks(tag)] + ) for task in step.tasks(tag): # the task object should have the tags assert_equals([True] * len(tags), [t in task.tags for t in tags]) for data in task: # the data artifact should have the tags - assert_equals([True] * len(tags), [t in data.tags for t in tags]) \ No newline at end of file + assert_equals( + [True] * len(tags), [t in data.tags for t in tags] + ) diff --git a/test/core/tests/basic_unbounded_foreach.py b/test/core/tests/basic_unbounded_foreach.py index 7c6faf6cb71..2c962f85d93 100644 --- a/test/core/tests/basic_unbounded_foreach.py +++ b/test/core/tests/basic_unbounded_foreach.py @@ -1,16 +1,18 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + class BasicUnboundedForeachTest(MetaflowTest): PRIORITY = 1 - @steps(0, ['foreach-split-small'], required=True) + @steps(0, ["foreach-split-small"], required=True) def split(self): self.my_index = None from metaflow.plugins import InternalTestUnboundedForeachInput + self.arr = InternalTestUnboundedForeachInput(range(2)) - @tag('unbounded_test_foreach_internal') - @steps(0, ['foreach-inner-small'], required=True) + @tag("unbounded_test_foreach_internal") + @steps(0, ["foreach-inner-small"], required=True) def inner(self): # index must stay constant over multiple steps inside foreach if self.my_index is None: @@ -19,23 +21,23 @@ def inner(self): assert_equals(self.input, self.arr[self.index]) self.my_input = self.input - @steps(0, ['foreach-join-small'], required=True) + @steps(0, ["foreach-join-small"], required=True) def join(self, inputs): got = sorted([inp.my_input for inp in inputs]) assert_equals(list(range(2)), got) - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): pass def check_results(self, flow, checker): run = checker.get_run() - if type(checker).__name__ == 'CliCheck': + if type(checker).__name__ == "CliCheck": # CliCheck doesn't support enlisting of tasks. - assert(run is None) + assert run is None else: - assert(run is not None) - tasks = run['foreach_inner'].tasks() + assert run is not None + tasks = run["foreach_inner"].tasks() task_list = list(tasks) assert_equals(2, len(task_list)) - assert_equals(1, len(list(run['foreach_inner'].control_tasks()))) + assert_equals(1, len(list(run["foreach_inner"].control_tasks()))) diff --git a/test/core/tests/catch_retry.py b/test/core/tests/catch_retry.py index 031a30600c3..e9eeb2ca24e 100644 --- a/test/core/tests/catch_retry.py +++ b/test/core/tests/catch_retry.py @@ -1,54 +1,58 @@ - from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag from metaflow import current + class CatchRetryTest(MetaflowTest): PRIORITY = 2 - @tag('retry(times=3)') - @steps(0, ['start']) + @tag("retry(times=3)") + @steps(0, ["start"]) def step_start(self): import os, sys + self.test_attempt = current.retry_count - sys.stdout.write('stdout testing logs %d\n' % self.test_attempt) - sys.stderr.write('stderr testing logs %d\n' % self.test_attempt) + sys.stdout.write("stdout testing logs %d\n" % self.test_attempt) + sys.stderr.write("stderr testing logs %d\n" % self.test_attempt) if self.test_attempt < 3: self.invisible = True raise TestRetry() # foreach splits don't support @catch but @retry should work - @tag('retry(times=2)') - @steps(0, ['foreach-split']) + @tag("retry(times=2)") + @steps(0, ["foreach-split"]) def step_split(self): import os + if current.retry_count == 2: self.this_is_split = True else: raise TestRetry() - @tag('retry(times=2)') - @steps(0, ['join']) + @tag("retry(times=2)") + @steps(0, ["join"]) def step_join(self): import os + if current.retry_count == 2: self.test_attempt = inputs[0].test_attempt else: raise TestRetry() @tag('catch(var="end_ex", print_exception=False)') - @steps(0, ['end'], required=True) + @steps(0, ["end"], required=True) def step_end(self): from metaflow.exception import ExternalCommandFailed + # make sure we see the latest attempt version of the artifact assert_equals(3, self.test_attempt) # the test uses a non-trivial derived exception on purpose # which is non-trivial to pickle correctly self.here = True - raise ExternalCommandFailed('catch me!') + raise ExternalCommandFailed("catch me!") @tag('catch(var="ex", print_exception=False)') - @tag('retry(times=2)') - @steps(1, ['all']) + @tag("retry(times=2)") + @steps(1, ["all"]) def step_all(self): # Die a soft death; this should retry and then catch in the end self.retry_with_catch = current.retry_count @@ -56,74 +60,82 @@ def step_all(self): def check_results(self, flow, checker): - checker.assert_log('start', 'stdout', 'stdout testing logs 3\n', exact_match=False) - checker.assert_log('start', 'stderr', 'stderr testing logs 3\n', exact_match=False) + checker.assert_log( + "start", "stdout", "stdout testing logs 3\n", exact_match=False + ) + checker.assert_log( + "start", "stderr", "stderr testing logs 3\n", exact_match=False + ) for step in flow: - if step.name == 'start': - checker.assert_artifact('start', 'test_attempt', 3) + if step.name == "start": + checker.assert_artifact("start", "test_attempt", 3) try: - for task in checker.artifact_dict('start', - 'invisible').values(): + for task in checker.artifact_dict("start", "invisible").values(): if task: - raise Exception("'invisible' should not be visible "\ - "in 'start'") + raise Exception( + "'invisible' should not be visible " "in 'start'" + ) except KeyError: pass - elif step.name == 'end': - checker.assert_artifact('end', 'test_attempt', 3) - for task in checker.artifact_dict(step.name, 'end_ex').values(): - assert_equals('catch me!', str(task['end_ex'].exception)) + elif step.name == "end": + checker.assert_artifact("end", "test_attempt", 3) + for task in checker.artifact_dict(step.name, "end_ex").values(): + assert_equals("catch me!", str(task["end_ex"].exception)) break else: raise Exception("No artifact 'end_ex' in step 'end'") - elif flow._graph[step.name].type == 'foreach': - checker.assert_artifact(step.name, 'this_is_split', True) + elif flow._graph[step.name].type == "foreach": + checker.assert_artifact(step.name, "this_is_split", True) - elif flow._graph[step.name].type == 'join': - checker.assert_artifact('end', 'test_attempt', 3) + elif flow._graph[step.name].type == "join": + checker.assert_artifact("end", "test_attempt", 3) else: - for task in checker.artifact_dict(step.name, 'ex').values(): - extype = 'metaflow_test.TestRetry' - assert_equals(extype, str(task['ex'].type)) + for task in checker.artifact_dict(step.name, "ex").values(): + extype = "metaflow_test.TestRetry" + assert_equals(extype, str(task["ex"].type)) break else: raise Exception("No artifact 'ex' in step '%s'" % step.name) - for task in checker.artifact_dict(step.name, 'retry_with_catch').values(): - assert_equals(task['retry_with_catch'], 2) + for task in checker.artifact_dict( + step.name, "retry_with_catch" + ).values(): + assert_equals(task["retry_with_catch"], 2) break else: - raise Exception("No artifact 'retry_with_catch' in step '%s'" % step.name) + raise Exception( + "No artifact 'retry_with_catch' in step '%s'" % step.name + ) run = checker.get_run() if run: for step in run: - if step.id == 'end': + if step.id == "end": continue - if flow._graph[step.id].type in ('foreach', 'join'): + if flow._graph[step.id].type in ("foreach", "join"): # 1 normal run + 2 retries = 3 attempts attempts = 3 - elif step.id == 'start': - attempts = 4 # 1 normal run + 3 retries = 4 attempts + elif step.id == "start": + attempts = 4 # 1 normal run + 3 retries = 4 attempts else: # 1 normal run + 2 retries = 3 attempts attempts = 3 for task in step: data = task.data - got = sorted(m.value for m in task.metadata - if m.type == 'attempt') + got = sorted(m.value for m in task.metadata if m.type == "attempt") assert_equals(list(map(str, range(attempts))), got) - assert_equals(False, 'invisible' in run['start'].task.data) - assert_equals(3, run['start'].task.data.test_attempt) - end = run['end'].task + assert_equals(False, "invisible" in run["start"].task.data) + assert_equals(3, run["start"].task.data.test_attempt) + end = run["end"].task assert_equals(True, end.data.here) assert_equals(3, end.data.test_attempt) # task.exception is None since the exception was handled assert_equals(None, end.exception) - assert_equals('catch me!', end.data.end_ex.exception) - assert_equals('metaflow.exception.ExternalCommandFailed', - end.data.end_ex.type) + assert_equals("catch me!", end.data.end_ex.exception) + assert_equals( + "metaflow.exception.ExternalCommandFailed", end.data.end_ex.type + ) diff --git a/test/core/tests/current_singleton.py b/test/core/tests/current_singleton.py index 4032332aa8f..723ad49f019 100644 --- a/test/core/tests/current_singleton.py +++ b/test/core/tests/current_singleton.py @@ -1,17 +1,20 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class CurrentSingletonTest(MetaflowTest): """ Test that the current singleton returns the right values """ + PRIORITY = 1 HEADER = "@project(name='current_singleton')" - @steps(0, ['start']) + @steps(0, ["start"]) def step_start(self): from uuid import uuid4 from metaflow import current + self.project_names = {current.project_name} self.branch_names = {current.branch_name} self.project_flow_names = {current.project_flow_name} @@ -26,7 +29,7 @@ def step_start(self): self.uuid = str(uuid4()) self.task_data = {current.pathspec: self.uuid} - @steps(1, ['join']) + @steps(1, ["join"]) def step_join(self): from uuid import uuid4 from metaflow import current @@ -65,7 +68,7 @@ def step_join(self): self.uuid = str(uuid4()) self.task_data[current.pathspec] = self.uuid - @steps(2, ['all']) + @steps(2, ["all"]) def step_all(self): from uuid import uuid4 from metaflow import current @@ -89,27 +92,30 @@ def check_results(self, flow, checker): if run is None: # very basic sanity check for CLI for step in flow: - checker.assert_artifact(step.name, 'step_name', step.name) - checker.assert_artifact(step.name, - 'project_names', - {'current_singleton'}) + checker.assert_artifact(step.name, "step_name", step.name) + checker.assert_artifact( + step.name, "project_names", {"current_singleton"} + ) else: from metaflow import Task + task_data = run.data.task_data for pathspec, uuid in task_data.items(): assert_equals(Task(pathspec).data.uuid, uuid) for step in run: for task in step: assert_equals(task.data.step_name, step.id) - pathspec = '/'.join(task.pathspec.split('/')[-4:]) + pathspec = "/".join(task.pathspec.split("/")[-4:]) assert_equals(task.data.uuid, task_data[pathspec]) - assert_equals(run.data.project_names, {'current_singleton'}) - assert_equals(run.data.branch_names, {'user.tester'}) - assert_equals(run.data.project_flow_names,\ - {'current_singleton.user.tester.CurrentSingletonTestFlow'}) + assert_equals(run.data.project_names, {"current_singleton"}) + assert_equals(run.data.branch_names, {"user.tester"}) + assert_equals( + run.data.project_flow_names, + {"current_singleton.user.tester.CurrentSingletonTestFlow"}, + ) assert_equals(run.data.is_production, {False}) assert_equals(run.data.flow_names, {run.parent.id}) assert_equals(run.data.run_ids, {run.id}) assert_equals(run.data.origin_run_ids, {None}) - assert_equals(run.data.namespaces, {'user:tester'}) - assert_equals(run.data.usernames, {'tester'}) \ No newline at end of file + assert_equals(run.data.namespaces, {"user:tester"}) + assert_equals(run.data.usernames, {"tester"}) diff --git a/test/core/tests/detect_segfault.py b/test/core/tests/detect_segfault.py index 5287f95e555..c865a2daafb 100644 --- a/test/core/tests/detect_segfault.py +++ b/test/core/tests/detect_segfault.py @@ -1,20 +1,23 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class DetectSegFaultTest(MetaflowTest): """ Test that segmentation faults produce a message in the logs """ + PRIORITY = 2 SHOULD_FAIL = True - @steps(0, ['singleton-end'], required=True) + @steps(0, ["singleton-end"], required=True) def step_end(self): # cause a segfault import ctypes + print("Crash and burn!") ctypes.string_at(0) - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): pass @@ -24,12 +27,6 @@ def check_results(self, flow, checker): run = checker.get_run() if run: # loglines prior to the segfault should be persisted - checker.assert_log('end', - 'stdout', - 'Crash and burn!', - exact_match=False) + checker.assert_log("end", "stdout", "Crash and burn!", exact_match=False) # a message should be printed that mentions "segmentation fault" - checker.assert_log('end', - 'stderr', - 'segmentation fault', - exact_match=False) + checker.assert_log("end", "stderr", "segmentation fault", exact_match=False) diff --git a/test/core/tests/dynamic_parameters.py b/test/core/tests/dynamic_parameters.py index d83995929a4..5de80ec782b 100644 --- a/test/core/tests/dynamic_parameters.py +++ b/test/core/tests/dynamic_parameters.py @@ -1,11 +1,12 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class DynamicParameterTest(MetaflowTest): PRIORITY = 3 PARAMETERS = { - 'str_param': {'default': 'str_func'}, - 'json_param': {'default': 'json_func', 'type': 'JSONType'}, - 'nondefault_param': {'default': 'lambda _: True', 'type': 'bool'} + "str_param": {"default": "str_func"}, + "json_param": {"default": "json_func", "type": "JSONType"}, + "nondefault_param": {"default": "lambda _: True", "type": "bool"}, } HEADER = """ import os @@ -34,19 +35,18 @@ def json_func(ctx): @project(name='dynamic_parameters_project') """ - @steps(0, ['singleton'], required=True) + @steps(0, ["singleton"], required=True) def step_single(self): - assert_equals(self.str_param, 'does this work?') + assert_equals(self.str_param, "does this work?") assert_equals(self.nondefault_param, False) - assert_equals(self.json_param, {'a': [8]}) + assert_equals(self.json_param, {"a": [8]}) - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): pass def check_results(self, flow, checker): for step in flow: - checker.assert_artifact(step.name, 'nondefault_param', False) - checker.assert_artifact(step.name, 'str_param', 'does this work?') - checker.assert_artifact(step.name, 'json_param', {'a': [8]}) - + checker.assert_artifact(step.name, "nondefault_param", False) + checker.assert_artifact(step.name, "str_param", "does this work?") + checker.assert_artifact(step.name, "json_param", {"a": [8]}) diff --git a/test/core/tests/extensions.py b/test/core/tests/extensions.py index 6669e07095e..b760d605ea9 100644 --- a/test/core/tests/extensions.py +++ b/test/core/tests/extensions.py @@ -1,13 +1,15 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + class ExtensionsTest(MetaflowTest): """ Test that the metaflow_extensions module is properly loaded """ + PRIORITY = 0 - @tag('test_step_decorator') - @steps(0, ['all']) + @tag("test_step_decorator") + @steps(0, ["all"]) def step_all(self): from metaflow.metaflow_config import METAFLOW_ADDITIONAL_VALUE from metaflow import tl_value @@ -21,7 +23,7 @@ def step_all(self): def check_results(self, flow, checker): for step in flow: - checker.assert_artifact(step.name, 'additional_value', 42) - checker.assert_artifact(step.name, 'tl_value', 42) - checker.assert_artifact(step.name, 'plugin_value', 42) - checker.assert_artifact(step.name, 'plugin_set_value', step.name) + checker.assert_artifact(step.name, "additional_value", 42) + checker.assert_artifact(step.name, "tl_value", 42) + checker.assert_artifact(step.name, "plugin_value", 42) + checker.assert_artifact(step.name, "plugin_set_value", step.name) diff --git a/test/core/tests/flow_options.py b/test/core/tests/flow_options.py index 99e4f96249f..951782e972d 100644 --- a/test/core/tests/flow_options.py +++ b/test/core/tests/flow_options.py @@ -5,6 +5,7 @@ class FlowOptionsTest(MetaflowTest): """ Test that the metaflow_extensions module is properly loaded """ + PRIORITY = 0 HEADER = """ import os @@ -14,7 +15,8 @@ class FlowOptionsTest(MetaflowTest): @test_flow_decorator """ - @steps(0, ['all']) + @steps(0, ["all"]) def step_all(self): from metaflow import current - assert_equals(current.foobar_value, 'this_is_foobar') + + assert_equals(current.foobar_value, "this_is_foobar") diff --git a/test/core/tests/large_artifact.py b/test/core/tests/large_artifact.py index 7a7e88696b5..cbb56f2738d 100644 --- a/test/core/tests/large_artifact.py +++ b/test/core/tests/large_artifact.py @@ -1,5 +1,6 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class LargeArtifactTest(MetaflowTest): """ Test that you can serialize large objects (over 4GB) @@ -7,31 +8,33 @@ class LargeArtifactTest(MetaflowTest): to serialize objects over 2GB - https://bugs.python.org/issue24658 so YMMV. """ + PRIORITY = 2 - @steps(0, ['singleton'], required=True) + @steps(0, ["singleton"], required=True) def step_single(self): import sys + if sys.version_info[0] > 2: - self.large = b'x' * int(4.1 * 1024**3) + self.large = b"x" * int(4.1 * 1024 ** 3) self.noop = False else: self.noop = True - @steps(0, ['end']) + @steps(0, ["end"]) def step_end(self): import sys + if sys.version_info[0] > 2: - assert_equals(self.large, b'x' * int(4.1 * 1024**3)) + assert_equals(self.large, b"x" * int(4.1 * 1024 ** 3)) - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): pass def check_results(self, flow, checker): import sys - noop = next(iter(checker.artifact_dict('end', 'noop').values()))['noop'] + + noop = next(iter(checker.artifact_dict("end", "noop").values()))["noop"] if not noop and sys.version_info[0] > 2: - checker.assert_artifact('end', - 'large', - b'x' * int(4.1 * 1024**3)) + checker.assert_artifact("end", "large", b"x" * int(4.1 * 1024 ** 3)) diff --git a/test/core/tests/large_mflog.py b/test/core/tests/large_mflog.py index 6316d9e1b4a..cb244fe9b03 100644 --- a/test/core/tests/large_mflog.py +++ b/test/core/tests/large_mflog.py @@ -1,45 +1,54 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class LargeMflogTest(MetaflowTest): """ Test that we can capture a large amount of log messages with accurate timings """ + PRIORITY = 2 HEADER = """ NUM_FOREACH = 32 NUM_LINES = 5000 """ - @steps(0, ['foreach-split-small'], required=True) + + @steps(0, ["foreach-split-small"], required=True) def split(self): self.arr = range(NUM_FOREACH) import random, string - self.random_log_prefix = ''.join([random.choice(string.ascii_lowercase) for _ in range(5)]) - @steps(0, ['foreach-inner-small'], required=True) + self.random_log_prefix = "".join( + [random.choice(string.ascii_lowercase) for _ in range(5)] + ) + + @steps(0, ["foreach-inner-small"], required=True) def inner(self): ISOFORMAT = "%Y-%m-%dT%H:%M:%S.%f" from datetime import datetime from metaflow import current import sys + self.log_step = current.step_name task_id = current.task_id for i in range(NUM_LINES): now = datetime.utcnow().strftime(ISOFORMAT) - print('%s %s stdout %d %s' % (self.random_log_prefix, task_id, i, now)) - sys.stderr.write('%s %s stderr %d %s\n' % (self.random_log_prefix, task_id, i, now)) + print("%s %s stdout %d %s" % (self.random_log_prefix, task_id, i, now)) + sys.stderr.write( + "%s %s stderr %d %s\n" % (self.random_log_prefix, task_id, i, now) + ) - @steps(0, ['foreach-join-small'], required=True) + @steps(0, ["foreach-join-small"], required=True) def join(self, inputs): self.log_step = inputs[0].log_step self.random_log_prefix = inputs[0].random_log_prefix - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): pass - @steps(0, ['end']) + @steps(0, ["end"]) def step_end(self): self.num_foreach = NUM_FOREACH self.num_lines = NUM_LINES @@ -50,21 +59,24 @@ def check_results(self, flow, checker): ISOFORMAT = "%Y-%m-%dT%H:%M:%S.%f" - _val = lambda n: list(checker.artifact_dict('end', n).values())[0][n] + _val = lambda n: list(checker.artifact_dict("end", n).values())[0][n] - step_name = _val('log_step') - num_foreach = _val('num_foreach') - num_lines = _val('num_lines') - random_log_prefix = _val('random_log_prefix') + step_name = _val("log_step") + num_foreach = _val("num_foreach") + num_lines = _val("num_lines") + random_log_prefix = _val("random_log_prefix") run = checker.get_run() - for stream in ('stdout', 'stderr'): + for stream in ("stdout", "stderr"): log = checker.get_log(step_name, stream) # ignore event_logger noise and Batch/Lambda noise by only looking at # log lines with the random prefix (generated by the very first step) - lines = [line.split() for line in log.splitlines() - if line.startswith(random_log_prefix)] + lines = [ + line.split() + for line in log.splitlines() + if line.startswith(random_log_prefix) + ] assert_equals(len(lines), num_foreach * num_lines) @@ -81,9 +93,11 @@ def check_results(self, flow, checker): if run is not None: for task in run[step_name]: # test task.loglines - task_lines = [(tstamp, msg) - for tstamp, msg in task.loglines(stream) - if msg.startswith(random_log_prefix)] + task_lines = [ + (tstamp, msg) + for tstamp, msg in task.loglines(stream) + if msg.startswith(random_log_prefix) + ] assert_equals(len(task_lines), num_lines) for i, (mf_tstamp, msg) in enumerate(task_lines): _, task_id, stream_type, idx, tstamp_str = msg.split() @@ -92,8 +106,8 @@ def check_results(self, flow, checker): assert_equals(stream_type, stream) assert_equals(int(idx), i) - # May 13, 2021 - Muting this test for now since the - # GitHub CI runner is constrained on resources causing + # May 13, 2021 - Muting this test for now since the + # GitHub CI runner is constrained on resources causing # this test to flake. TODO: Make this check less flaky. # tstamp = datetime.strptime(tstamp_str, ISOFORMAT) # delta = mf_tstamp - tstamp @@ -101,9 +115,8 @@ def check_results(self, flow, checker): # # delta.seconds can be made smaller, e.g. 5 secs # # enable this line to see a distribution of deltas: # # print("DELTA", delta.seconds) - # if delta.days > 0 or delta.seconds > 60: # raise Exception("Time delta too high. "\ # "Mflog %s, user %s"\ - # % (mf_tstamp, tstamp)) \ No newline at end of file + # % (mf_tstamp, tstamp)) diff --git a/test/core/tests/lineage.py b/test/core/tests/lineage.py index 2e92112af6d..4ed4386c75a 100644 --- a/test/core/tests/lineage.py +++ b/test/core/tests/lineage.py @@ -1,20 +1,20 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class LineageTest(MetaflowTest): PRIORITY = 1 - @steps(0, ['start']) + @steps(0, ["start"]) def step_start(self): self.lineage = (self._current_step,) - @steps(1, ['join']) + @steps(1, ["join"]) def step_join(self): # we can't easily account for the number of foreach splits, # so we only care about unique lineages (hence set()) - self.lineage = (tuple(sorted({x.lineage for x in inputs})), - self._current_step) + self.lineage = (tuple(sorted({x.lineage for x in inputs})), self._current_step) - @steps(2, ['all']) + @steps(2, ["all"]) def step_all(self): self.lineage += (self._current_step,) @@ -29,7 +29,7 @@ def check_results(self, flow, checker): # collect lineages on the way and finally compare them # to the lineages produced by the actual run def traverse(step, lineage): - if graph[step].type == 'join': + if graph[step].type == "join": join_sets[step].add(tuple(lineage)) if len(join_sets[step]) < len(graph[step].in_funcs): return @@ -39,7 +39,7 @@ def traverse(step, lineage): for n in graph[step].out_funcs: traverse(n, lineage + (step,)) - traverse('start', ()) + traverse("start", ()) for step in flow: - checker.assert_artifact(step.name, 'lineage', lineages[step.name]) + checker.assert_artifact(step.name, "lineage", lineages[step.name]) diff --git a/test/core/tests/merge_artifacts.py b/test/core/tests/merge_artifacts.py index 494a3fadb8a..e91030aca1c 100644 --- a/test/core/tests/merge_artifacts.py +++ b/test/core/tests/merge_artifacts.py @@ -1,66 +1,76 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class MergeArtifactsTest(MetaflowTest): PRIORITY = 1 - @steps(0, ['start']) + @steps(0, ["start"]) def start(self): - self.non_modified_passdown = 'a' - self.modified_to_same_value = 'b' - self.manual_merge_required = 'c' - self.ignore_me = 'd' + self.non_modified_passdown = "a" + self.modified_to_same_value = "b" + self.manual_merge_required = "c" + self.ignore_me = "d" - @steps(2, ['linear']) + @steps(2, ["linear"]) def modify_things(self): # Set to different things from metaflow.current import current + self.manual_merge_required = current.task_id self.ignore_me = current.task_id - self.modified_to_same_value = 'e' - assert_equals(self.non_modified_passdown, 'a') + self.modified_to_same_value = "e" + assert_equals(self.non_modified_passdown, "a") - @steps(0, ['join'], required=True) + @steps(0, ["join"], required=True) def merge_things(self, inputs): from metaflow.current import current - from metaflow.exception import UnhandledInMergeArtifactsException, MetaflowException + from metaflow.exception import ( + UnhandledInMergeArtifactsException, + MetaflowException, + ) # Test to make sure non-merged values are reported - assert_exception(lambda: self.merge_artifacts(inputs), UnhandledInMergeArtifactsException) + assert_exception( + lambda: self.merge_artifacts(inputs), UnhandledInMergeArtifactsException + ) # Test to make sure nothing is set if failed merge_artifacts - assert(not hasattr(self, 'non_modified_passdown')) - assert(not hasattr(self, 'manual_merge_required')) + assert not hasattr(self, "non_modified_passdown") + assert not hasattr(self, "manual_merge_required") # Test to make sure that only one of exclude/include is used - assert_exception(lambda: self.merge_artifacts( - inputs, - exclude=['ignore_me'], - include=['non_modified_passdown']), MetaflowException) + assert_exception( + lambda: self.merge_artifacts( + inputs, exclude=["ignore_me"], include=["non_modified_passdown"] + ), + MetaflowException, + ) # Test to make sure nothing is set if failed merge_artifacts - assert(not hasattr(self, 'non_modified_passdown')) - assert(not hasattr(self, 'manual_merge_required')) + assert not hasattr(self, "non_modified_passdown") + assert not hasattr(self, "manual_merge_required") # Test actual merge (ignores set values and excluded names, merges common and non modified) self.manual_merge_required = current.task_id - self.merge_artifacts(inputs, exclude=['ignore_me']) + self.merge_artifacts(inputs, exclude=["ignore_me"]) # Ensure that everything we expect is passed down - assert_equals(self.non_modified_passdown, 'a') - assert_equals(self.modified_to_same_value, 'e') + assert_equals(self.non_modified_passdown, "a") + assert_equals(self.modified_to_same_value, "e") assert_equals(self.manual_merge_required, current.task_id) - assert(not hasattr(self, 'ignore_me')) + assert not hasattr(self, "ignore_me") - @steps(0, ['end']) + @steps(0, ["end"]) def end(self): from metaflow.exception import MetaflowException + # This is not a join so test exception for calling in non-join assert_exception(lambda: self.merge_artifacts([]), MetaflowException) # Check that all values made it through - assert_equals(self.non_modified_passdown, 'a') - assert_equals(self.modified_to_same_value, 'e') - assert(hasattr(self, 'manual_merge_required')) + assert_equals(self.non_modified_passdown, "a") + assert_equals(self.modified_to_same_value, "e") + assert hasattr(self, "manual_merge_required") - @steps(3, ['all']) + @steps(3, ["all"]) def step_all(self): - assert_equals(self.non_modified_passdown, 'a') + assert_equals(self.non_modified_passdown, "a") diff --git a/test/core/tests/merge_artifacts_include.py b/test/core/tests/merge_artifacts_include.py index a30f9303df6..983b7e17e73 100644 --- a/test/core/tests/merge_artifacts_include.py +++ b/test/core/tests/merge_artifacts_include.py @@ -1,52 +1,58 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class MergeArtifactsIncludeTest(MetaflowTest): PRIORITY = 1 - @steps(0, ['start']) + @steps(0, ["start"]) def start(self): - self.non_modified_passdown = 'a' - self.modified_to_same_value = 'b' - self.manual_merge_required = 'c' - self.ignore_me = 'd' + self.non_modified_passdown = "a" + self.modified_to_same_value = "b" + self.manual_merge_required = "c" + self.ignore_me = "d" - @steps(2, ['linear']) + @steps(2, ["linear"]) def modify_things(self): # Set to different things from metaflow.current import current + self.manual_merge_required = current.task_id self.ignore_me = current.task_id - self.modified_to_same_value = 'e' - assert_equals(self.non_modified_passdown, 'a') + self.modified_to_same_value = "e" + assert_equals(self.non_modified_passdown, "a") - @steps(0, ['join'], required=True) + @steps(0, ["join"], required=True) def merge_things(self, inputs): from metaflow.current import current from metaflow.exception import MissingInMergeArtifactsException self.manual_merge_required = current.task_id # Test to see if we raise an exception if include specifies non-merged things - assert_exception(lambda: self.merge_artifacts( - inputs, include=['manual_merge_required', 'foobar']), MissingInMergeArtifactsException) + assert_exception( + lambda: self.merge_artifacts( + inputs, include=["manual_merge_required", "foobar"] + ), + MissingInMergeArtifactsException, + ) # Test to make sure nothing is set if failed merge_artifacts - assert(not hasattr(self, 'non_modified_passdown')) + assert not hasattr(self, "non_modified_passdown") # Merge include non_modified_passdown - self.merge_artifacts(inputs, include=['non_modified_passdown']) + self.merge_artifacts(inputs, include=["non_modified_passdown"]) # Ensure that everything we expect is passed down - assert_equals(self.non_modified_passdown, 'a') + assert_equals(self.non_modified_passdown, "a") assert_equals(self.manual_merge_required, current.task_id) - assert(not hasattr(self, 'ignore_me')) - assert(not hasattr(self, 'modified_to_same_value')) + assert not hasattr(self, "ignore_me") + assert not hasattr(self, "modified_to_same_value") - @steps(0, ['end']) + @steps(0, ["end"]) def end(self): # Check that all values made it through - assert_equals(self.non_modified_passdown, 'a') - assert(hasattr(self, 'manual_merge_required')) + assert_equals(self.non_modified_passdown, "a") + assert hasattr(self, "manual_merge_required") - @steps(3, ['all']) + @steps(3, ["all"]) def step_all(self): - assert_equals(self.non_modified_passdown, 'a') + assert_equals(self.non_modified_passdown, "a") diff --git a/test/core/tests/merge_artifacts_propagation.py b/test/core/tests/merge_artifacts_propagation.py index 8fc3b8f1821..779e0a94c3b 100644 --- a/test/core/tests/merge_artifacts_propagation.py +++ b/test/core/tests/merge_artifacts_propagation.py @@ -1,5 +1,6 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class MergeArtifactsPropagationTest(MetaflowTest): # This test simply tests whether things set on a single branch will # still get propagated down properly. Other merge_artifacts behaviors @@ -8,25 +9,27 @@ class MergeArtifactsPropagationTest(MetaflowTest): # more generic. PRIORITY = 1 - @steps(0, ['start']) + @steps(0, ["start"]) def start(self): - self.non_modified_passdown = 'a' + self.non_modified_passdown = "a" - @steps(0, ['foreach-inner-small'], required=True) + @steps(0, ["foreach-inner-small"], required=True) def modify_things(self): # Set different names to different things val = self.index - setattr(self, 'var%d' % (val), val) + setattr(self, "var%d" % (val), val) - @steps(0, ['foreach-join-small'], required=True) + @steps(0, ["foreach-join-small"], required=True) def merge_things(self, inputs): - self.merge_artifacts(inputs,) + self.merge_artifacts( + inputs, + ) # Ensure that everything we expect is passed down - assert_equals(self.non_modified_passdown, 'a') + assert_equals(self.non_modified_passdown, "a") for i, _ in enumerate(inputs): - assert_equals(getattr(self, 'var%d' % (i)), i) + assert_equals(getattr(self, "var%d" % (i)), i) - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): - assert_equals(self.non_modified_passdown, 'a') + assert_equals(self.non_modified_passdown, "a") diff --git a/test/core/tests/nested_foreach.py b/test/core/tests/nested_foreach.py index 04b706face9..4ad715aa2d0 100644 --- a/test/core/tests/nested_foreach.py +++ b/test/core/tests/nested_foreach.py @@ -1,9 +1,10 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class NestedForeachTest(MetaflowTest): PRIORITY = 1 - @steps(0, ['foreach-nested-inner'], required=True) + @steps(0, ["foreach-nested-inner"], required=True) def inner(self): [x, y, z] = self.foreach_stack() @@ -19,14 +20,14 @@ def inner(self): self.combo = x[2] + y[2] + z[2] - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): pass def check_results(self, flow, checker): from itertools import product - artifacts = checker.artifact_dict('foreach_inner', 'combo') - got = sorted(val['combo'] for val in artifacts.values()) - expected = sorted(''.join(p) for p in product('abc', 'de', 'fghijk')) + artifacts = checker.artifact_dict("foreach_inner", "combo") + got = sorted(val["combo"] for val in artifacts.values()) + expected = sorted("".join(p) for p in product("abc", "de", "fghijk")) assert_equals(expected, got) diff --git a/test/core/tests/nested_unbounded_foreach.py b/test/core/tests/nested_unbounded_foreach.py index 75aee08f779..60c2aee4d1e 100644 --- a/test/core/tests/nested_unbounded_foreach.py +++ b/test/core/tests/nested_unbounded_foreach.py @@ -1,15 +1,17 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + class NestedUnboundedForeachTest(MetaflowTest): PRIORITY = 1 - @steps(0, ['foreach-nested-split'], required=True) + @steps(0, ["foreach-nested-split"], required=True) def split_z(self): from metaflow.plugins import InternalTestUnboundedForeachInput + self.z = InternalTestUnboundedForeachInput(self.z) - @tag('unbounded_test_foreach_internal') - @steps(0, ['foreach-nested-inner'], required=True) + @tag("unbounded_test_foreach_internal") + @steps(0, ["foreach-nested-inner"], required=True) def inner(self): [x, y, z] = self.foreach_stack() @@ -17,7 +19,7 @@ def inner(self): assert_equals(len(self.x), x[1]) assert_equals(len(self.y), y[1]) # Note: We can't assert the actual num_splits for unbounded-foreach. - assert_equals(None, z[1]) # expected=len(self.z) for bounded. + assert_equals(None, z[1]) # expected=len(self.z) for bounded. # assert that variables are correct given their indices assert_equals(x[2], self.x[x[0]]) @@ -27,7 +29,7 @@ def inner(self): assert_equals(self.input, z[2]) self.combo = x[2] + y[2] + z[2] - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): pass @@ -35,21 +37,25 @@ def check_results(self, flow, checker): from itertools import product run = checker.get_run() - if type(checker).__name__ == 'CliCheck': + if type(checker).__name__ == "CliCheck": # CliCheck doesn't support enlisting of tasks nor can disambiguate # control vs ubf tasks while dumping artifacts. - assert(run is None) + assert run is None else: - assert(run is not None) - foreach_inner_tasks = {t.pathspec for t in run['foreach_inner'].tasks()} + assert run is not None + foreach_inner_tasks = {t.pathspec for t in run["foreach_inner"].tasks()} assert_equals(36, len(foreach_inner_tasks)) - assert_equals(6, len(list(run['foreach_inner'].control_tasks()))) + assert_equals(6, len(list(run["foreach_inner"].control_tasks()))) - artifacts = checker.artifact_dict('foreach_inner', 'combo') + artifacts = checker.artifact_dict("foreach_inner", "combo") # Explicitly only consider UBF tasks since the CLIChecker isn't aware of them. - step_prefix = run['foreach_inner'].pathspec + step_prefix = run["foreach_inner"].pathspec import os - got = sorted(val['combo'] for task, val in artifacts.items() - if os.path.join(step_prefix, task) in foreach_inner_tasks) - expected = sorted(''.join(p) for p in product('abc', 'de', 'fghijk')) + + got = sorted( + val["combo"] + for task, val in artifacts.items() + if os.path.join(step_prefix, task) in foreach_inner_tasks + ) + expected = sorted("".join(p) for p in product("abc", "de", "fghijk")) assert_equals(expected, got) diff --git a/test/core/tests/param_names.py b/test/core/tests/param_names.py index 9dc1cd5b14c..c1ac9cd8b92 100644 --- a/test/core/tests/param_names.py +++ b/test/core/tests/param_names.py @@ -1,13 +1,13 @@ from metaflow_test import MetaflowTest, steps + class ParameterNameTest(MetaflowTest): PRIORITY = 1 - PARAMETERS = { - "foo":{"default":1} - } + PARAMETERS = {"foo": {"default": 1}} - @steps(0, ['all']) + @steps(0, ["all"]) def step_all(self): from metaflow import current - assert_equals(len(current.parameter_names),1) - assert_equals(current.parameter_names[0],'foo') \ No newline at end of file + + assert_equals(len(current.parameter_names), 1) + assert_equals(current.parameter_names[0], "foo") diff --git a/test/core/tests/project_branch.py b/test/core/tests/project_branch.py index 0fb6355db19..a4a24ebe73e 100644 --- a/test/core/tests/project_branch.py +++ b/test/core/tests/project_branch.py @@ -1,5 +1,6 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + class ProjectBranchTest(MetaflowTest): PRIORITY = 1 @@ -10,13 +11,16 @@ class ProjectBranchTest(MetaflowTest): @project(name='project_branch') """ - @steps(0, ['singleton'], required=True) + @steps(0, ["singleton"], required=True) def step_single(self): pass - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): from metaflow import current - assert_equals(current.branch_name, 'test.this_is_a_test_branch') - assert_equals(current.project_flow_name, - 'project_branch.test.this_is_a_test_branch.ProjectBranchTestFlow') \ No newline at end of file + + assert_equals(current.branch_name, "test.this_is_a_test_branch") + assert_equals( + current.project_flow_name, + "project_branch.test.this_is_a_test_branch.ProjectBranchTestFlow", + ) diff --git a/test/core/tests/project_production.py b/test/core/tests/project_production.py index cec4546e839..730bb771cec 100644 --- a/test/core/tests/project_production.py +++ b/test/core/tests/project_production.py @@ -1,5 +1,6 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + class ProjectProductionTest(MetaflowTest): PRIORITY = 1 @@ -10,13 +11,15 @@ class ProjectProductionTest(MetaflowTest): @project(name='project_prod') """ - @steps(0, ['singleton'], required=True) + @steps(0, ["singleton"], required=True) def step_single(self): pass - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): from metaflow import current - assert_equals(current.branch_name, 'prod') - assert_equals(current.project_flow_name, - 'project_prod.prod.ProjectProductionTestFlow') \ No newline at end of file + + assert_equals(current.branch_name, "prod") + assert_equals( + current.project_flow_name, "project_prod.prod.ProjectProductionTestFlow" + ) diff --git a/test/core/tests/resume_end_step.py b/test/core/tests/resume_end_step.py index b494c7ee784..c184d5335c8 100644 --- a/test/core/tests/resume_end_step.py +++ b/test/core/tests/resume_end_step.py @@ -1,35 +1,34 @@ - from metaflow_test import MetaflowTest, ExpectationFailed, steps + class ResumeEndStepTest(MetaflowTest): """ Resuming from the end step should work """ + RESUME = True PRIORITY = 3 - PARAMETERS = { - 'int_param': {'default': 123} - } + PARAMETERS = {"int_param": {"default": 123}} - @steps(0, ['start']) + @steps(0, ["start"]) def step_start(self): - self.data = 'start' + self.data = "start" - @steps(0, ['singleton-end'], required=True) + @steps(0, ["singleton-end"], required=True) def step_end(self): if is_resumed(): - self.data = 'foo' + self.data = "foo" else: - self.data = 'bar' + self.data = "bar" raise ResumeFromHere() - @steps(2, ['all']) + @steps(2, ["all"]) def step_all(self): pass def check_results(self, flow, checker): for step in flow: - if step.name == 'end': - checker.assert_artifact(step.name, 'data', 'foo') + if step.name == "end": + checker.assert_artifact(step.name, "data", "foo") else: - checker.assert_artifact(step.name, 'data', 'start') + checker.assert_artifact(step.name, "data", "start") diff --git a/test/core/tests/resume_foreach_inner.py b/test/core/tests/resume_foreach_inner.py index 5fdcc2dba9b..bdd8a18bdf6 100644 --- a/test/core/tests/resume_foreach_inner.py +++ b/test/core/tests/resume_foreach_inner.py @@ -1,64 +1,65 @@ - from metaflow_test import MetaflowTest, ExpectationFailed, steps + class ResumeForeachInnerTest(MetaflowTest): """ Resuming from a foreach inner should work. Check that data changes in all downstream steps after resume. """ + RESUME = True PRIORITY = 3 - @steps(0, ['start']) + @steps(0, ["start"]) def step_start(self): - self.data = 'start' + self.data = "start" self.after = False - @steps(0, ['foreach-nested-split', 'foreach-split'], required=True) + @steps(0, ["foreach-nested-split", "foreach-split"], required=True) def step_split(self): if self.after: - assert_equals('resume', self.data) + assert_equals("resume", self.data) else: - assert_equals('start', self.data) + assert_equals("start", self.data) - @steps(0, ['foreach-inner'], required=True) + @steps(0, ["foreach-inner"], required=True) def inner(self): self.after = True if is_resumed(): - self.data = 'resume' + self.data = "resume" else: - self.data = 'run' + self.data = "run" raise ResumeFromHere() - self.stack = [list(map(str, getattr(self, frame.var))) - for frame in self._foreach_stack] - self.var = [''.join(str(x[2]) for x in self.foreach_stack())] + self.stack = [ + list(map(str, getattr(self, frame.var))) for frame in self._foreach_stack + ] + self.var = ["".join(str(x[2]) for x in self.foreach_stack())] - @steps(0, ['join'], required=True) + @steps(0, ["join"], required=True) def step_join(self, inputs): from itertools import chain + self.var = list(sorted(chain.from_iterable(i.var for i in inputs))) self.data = inputs[0].data self.after = inputs[0].after self.stack = inputs[0].stack if self.after: - assert_equals('resume', self.data) + assert_equals("resume", self.data) else: - assert_equals('start', self.data) + assert_equals("start", self.data) - @steps(2, ['all']) + @steps(2, ["all"]) def step_all(self): if self.after: - assert_equals('resume', self.data) + assert_equals("resume", self.data) else: - assert_equals('start', self.data) + assert_equals("start", self.data) def check_results(self, flow, checker): from itertools import product - checker.assert_artifact('start', 'data', 'start') - checker.assert_artifact('end', 'data', 'resume') - stack = next(iter(checker.artifact_dict('end', - 'stack').values()))['stack'] - expected = sorted(''.join(p) for p in product(*stack)) - checker.assert_artifact('end', 'var', expected) - + checker.assert_artifact("start", "data", "start") + checker.assert_artifact("end", "data", "resume") + stack = next(iter(checker.artifact_dict("end", "stack").values()))["stack"] + expected = sorted("".join(p) for p in product(*stack)) + checker.assert_artifact("end", "var", expected) diff --git a/test/core/tests/resume_foreach_join.py b/test/core/tests/resume_foreach_join.py index 8b59215c7e9..9bbea168073 100644 --- a/test/core/tests/resume_foreach_join.py +++ b/test/core/tests/resume_foreach_join.py @@ -1,58 +1,59 @@ - from metaflow_test import MetaflowTest, ExpectationFailed, steps + class ResumeForeachJoinTest(MetaflowTest): """ Resuming from a foreach join should work. Check that data changes in all downstream steps after resume. """ + RESUME = True PRIORITY = 3 - @steps(0, ['start']) + @steps(0, ["start"]) def step_start(self): - self.data = 'start' + self.data = "start" self.after = False - @steps(0, ['foreach-nested-split', 'foreach-split'], required=True) + @steps(0, ["foreach-nested-split", "foreach-split"], required=True) def step_split(self): if self.after: - assert_equals('resume', self.data) + assert_equals("resume", self.data) else: - assert_equals('start', self.data) + assert_equals("start", self.data) - @steps(0, ['foreach-inner'], required=True) + @steps(0, ["foreach-inner"], required=True) def inner(self): - self.stack = [list(map(str, getattr(self, frame.var))) - for frame in self._foreach_stack] - self.var = [''.join(str(x[2]) for x in self.foreach_stack())] + self.stack = [ + list(map(str, getattr(self, frame.var))) for frame in self._foreach_stack + ] + self.var = ["".join(str(x[2]) for x in self.foreach_stack())] - @steps(0, ['join'], required=True) + @steps(0, ["join"], required=True) def step_join(self, inputs): self.after = True if is_resumed(): - self.data = 'resume' + self.data = "resume" else: - self.data = 'run' + self.data = "run" raise ResumeFromHere() from itertools import chain + self.var = list(sorted(chain.from_iterable(i.var for i in inputs))) self.stack = inputs[0].stack - @steps(2, ['all']) + @steps(2, ["all"]) def step_all(self): if self.after: - assert_equals('resume', self.data) + assert_equals("resume", self.data) else: - assert_equals('start', self.data) + assert_equals("start", self.data) def check_results(self, flow, checker): from itertools import product - checker.assert_artifact('start', 'data', 'start') - checker.assert_artifact('end', 'data', 'resume') - stack = next(iter(checker.artifact_dict('end', - 'stack').values()))['stack'] - expected = sorted(''.join(p) for p in product(*stack)) - checker.assert_artifact('end', 'var', expected) - + checker.assert_artifact("start", "data", "start") + checker.assert_artifact("end", "data", "resume") + stack = next(iter(checker.artifact_dict("end", "stack").values()))["stack"] + expected = sorted("".join(p) for p in product(*stack)) + checker.assert_artifact("end", "var", expected) diff --git a/test/core/tests/resume_foreach_split.py b/test/core/tests/resume_foreach_split.py index 29fe67eff13..4b979000629 100644 --- a/test/core/tests/resume_foreach_split.py +++ b/test/core/tests/resume_foreach_split.py @@ -1,64 +1,65 @@ - from metaflow_test import MetaflowTest, ExpectationFailed, steps + class ResumeForeachSplitTest(MetaflowTest): """ Resuming from a foreach split should work. Check that data changes in all downstream steps after resume. """ + RESUME = True PRIORITY = 3 - @steps(0, ['start']) + @steps(0, ["start"]) def step_start(self): - self.data = 'start' + self.data = "start" self.after = False - @steps(0, ['foreach-nested-split', 'foreach-split'], required=True) + @steps(0, ["foreach-nested-split", "foreach-split"], required=True) def step_split(self): self.after = True if is_resumed(): - self.data = 'resume' + self.data = "resume" else: - self.data = 'run' + self.data = "run" raise ResumeFromHere() - @steps(0, ['foreach-inner'], required=True) + @steps(0, ["foreach-inner"], required=True) def inner(self): - self.stack = [list(map(str, getattr(self, frame.var))) - for frame in self._foreach_stack] - self.var = [''.join(str(x[2]) for x in self.foreach_stack())] + self.stack = [ + list(map(str, getattr(self, frame.var))) for frame in self._foreach_stack + ] + self.var = ["".join(str(x[2]) for x in self.foreach_stack())] if self.after: - assert_equals('resume', self.data) + assert_equals("resume", self.data) else: - assert_equals('start', self.data) + assert_equals("start", self.data) - @steps(0, ['join'], required=True) + @steps(0, ["join"], required=True) def step_join(self, inputs): from itertools import chain + self.var = list(sorted(chain.from_iterable(i.var for i in inputs))) self.data = inputs[0].data self.after = inputs[0].after self.stack = inputs[0].stack if self.after: - assert_equals('resume', self.data) + assert_equals("resume", self.data) else: - assert_equals('start', self.data) + assert_equals("start", self.data) - @steps(2, ['all']) + @steps(2, ["all"]) def step_all(self): if self.after: - assert_equals('resume', self.data) + assert_equals("resume", self.data) else: - assert_equals('start', self.data) + assert_equals("start", self.data) def check_results(self, flow, checker): from itertools import product - checker.assert_artifact('start', 'data', 'start') - checker.assert_artifact('end', 'data', 'resume') - stack = next(iter(checker.artifact_dict('end', - 'stack').values()))['stack'] - expected = sorted(''.join(p) for p in product(*stack)) - checker.assert_artifact('end', 'var', expected) - + checker.assert_artifact("start", "data", "start") + checker.assert_artifact("end", "data", "resume") + stack = next(iter(checker.artifact_dict("end", "stack").values()))["stack"] + expected = sorted("".join(p) for p in product(*stack)) + checker.assert_artifact("end", "var", expected) diff --git a/test/core/tests/resume_start_step.py b/test/core/tests/resume_start_step.py index b17e927b130..681d18ba1ca 100644 --- a/test/core/tests/resume_start_step.py +++ b/test/core/tests/resume_start_step.py @@ -1,32 +1,34 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class ResumeStartStepTest(MetaflowTest): """ Resuming from the start step should work """ + RESUME = True PRIORITY = 3 - PARAMETERS = { - 'int_param': {'default': 123} - } + PARAMETERS = {"int_param": {"default": 123}} - @steps(0, ['singleton-start'], required=True) + @steps(0, ["singleton-start"], required=True) def step_start(self): from metaflow import current + if is_resumed(): - self.data = 'foo' + self.data = "foo" # Verify that the `current` singleton contains the correct origin # run_id by double checking with the environment variables used # for tests. self.actual_origin_run_id = current.origin_run_id from metaflow_test import origin_run_id_for_resume + self.expected_origin_run_id = origin_run_id_for_resume() assert len(self.expected_origin_run_id) > 0 else: - self.data = 'bar' + self.data = "bar" raise ResumeFromHere() - @steps(2, ['all']) + @steps(2, ["all"]) def step_all(self): pass @@ -34,8 +36,9 @@ def check_results(self, flow, checker): run = checker.get_run() if run is None: for step in flow: - checker.assert_artifact(step.name, 'data', 'foo') - checker.assert_artifact(step.name, 'int_param', 123) + checker.assert_artifact(step.name, "data", "foo") + checker.assert_artifact(step.name, "int_param", 123) else: - assert_equals(run.data.expected_origin_run_id, - run.data.actual_origin_run_id) + assert_equals( + run.data.expected_origin_run_id, run.data.actual_origin_run_id + ) diff --git a/test/core/tests/s3_failure.py b/test/core/tests/s3_failure.py index 88afc422b11..2b609b5aa4a 100644 --- a/test/core/tests/s3_failure.py +++ b/test/core/tests/s3_failure.py @@ -1,9 +1,11 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class S3FailureTest(MetaflowTest): """ Test that S3 failures are handled correctly. """ + PRIORITY = 1 HEADER = """ @@ -12,20 +14,22 @@ class S3FailureTest(MetaflowTest): os.environ['TEST_S3_RETRY'] = '1' """ - @steps(0, ['singleton-start'], required=True) + @steps(0, ["singleton-start"], required=True) def step_start(self): # we need a unique artifact for every run which we can reconstruct # independently in the start and end tasks from metaflow import current - self.x = '%s/%s' % (current.flow_name, current.run_id) - @steps(0, ['end']) + self.x = "%s/%s" % (current.flow_name, current.run_id) + + @steps(0, ["end"]) def step_end(self): from metaflow import current - run_id = '%s/%s' % (current.flow_name, current.run_id) + + run_id = "%s/%s" % (current.flow_name, current.run_id) assert_equals(self.x, run_id) - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): pass @@ -34,10 +38,7 @@ def check_results(self, flow, checker): if run: # we should see TEST_S3_RETRY error in the logs # when --datastore=s3 - checker.assert_log('start', - 'stderr', - 'TEST_S3_RETRY', - exact_match=False) - run_id = 'S3FailureTestFlow/%s' % checker.run_id - checker.assert_artifact('start', 'x', run_id) - checker.assert_artifact('end', 'x', run_id) + checker.assert_log("start", "stderr", "TEST_S3_RETRY", exact_match=False) + run_id = "S3FailureTestFlow/%s" % checker.run_id + checker.assert_artifact("start", "x", run_id) + checker.assert_artifact("end", "x", run_id) diff --git a/test/core/tests/tag_catch.py b/test/core/tests/tag_catch.py index c347c1eb822..7ff3972d5e3 100644 --- a/test/core/tests/tag_catch.py +++ b/test/core/tests/tag_catch.py @@ -1,4 +1,3 @@ - from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag from metaflow import current @@ -6,91 +5,99 @@ class TagCatchTest(MetaflowTest): PRIORITY = 2 - @tag('retry(times=3)') - @steps(0, ['start']) + @tag("retry(times=3)") + @steps(0, ["start"]) def step_start(self): import os, sys + self.test_attempt = current.retry_count - sys.stdout.write('stdout testing logs %d\n' % self.test_attempt) - sys.stderr.write('stderr testing logs %d\n' % self.test_attempt) + sys.stdout.write("stdout testing logs %d\n" % self.test_attempt) + sys.stderr.write("stderr testing logs %d\n" % self.test_attempt) if self.test_attempt < 3: self.invisible = True raise TestRetry() # foreach splits don't support @catch but @retry should work - @tag('retry(times=2)') - @steps(0, ['foreach-split']) + @tag("retry(times=2)") + @steps(0, ["foreach-split"]) def step_split(self): import os + if current.retry_count == 2: self.this_is_split = True else: raise TestRetry() - @tag('retry(times=2)') - @steps(0, ['join']) + @tag("retry(times=2)") + @steps(0, ["join"]) def step_join(self): import os + if current.retry_count == 2: self.test_attempt = inputs[0].test_attempt else: raise TestRetry() @tag('catch(var="end_ex", print_exception=False)') - @steps(0, ['end'], required=True) + @steps(0, ["end"], required=True) def step_end(self): from metaflow.exception import ExternalCommandFailed + # make sure we see the latest attempt version of the artifact assert_equals(3, self.test_attempt) # the test uses a non-trivial derived exception on purpose # which is non-trivial to pickle correctly self.here = True - raise ExternalCommandFailed('catch me!') + raise ExternalCommandFailed("catch me!") @tag('catch(var="ex", print_exception=False)') - @tag('retry(times=2)') - @steps(1, ['all']) + @tag("retry(times=2)") + @steps(1, ["all"]) def step_all(self): import signal, os + # die an ugly death os.kill(os.getpid(), signal.SIGKILL) def check_results(self, flow, checker): - checker.assert_log('start', 'stdout', 'stdout testing logs 3\n', exact_match=False) - checker.assert_log('start', 'stderr', 'stderr testing logs 3\n', exact_match=False) + checker.assert_log( + "start", "stdout", "stdout testing logs 3\n", exact_match=False + ) + checker.assert_log( + "start", "stderr", "stderr testing logs 3\n", exact_match=False + ) for step in flow: - if step.name == 'start': - checker.assert_artifact('start', 'test_attempt', 3) + if step.name == "start": + checker.assert_artifact("start", "test_attempt", 3) try: - for task in checker.artifact_dict('start', - 'invisible').values(): + for task in checker.artifact_dict("start", "invisible").values(): if task: - raise Exception("'invisible' should not be visible "\ - "in 'start'") + raise Exception( + "'invisible' should not be visible " "in 'start'" + ) except KeyError: pass - elif step.name == 'end': - checker.assert_artifact('end', 'test_attempt', 3) - for task in checker.artifact_dict(step.name, 'end_ex').values(): - assert_equals('catch me!', str(task['end_ex'].exception)) + elif step.name == "end": + checker.assert_artifact("end", "test_attempt", 3) + for task in checker.artifact_dict(step.name, "end_ex").values(): + assert_equals("catch me!", str(task["end_ex"].exception)) break else: raise Exception("No artifact 'end_ex' in step 'end'") - elif flow._graph[step.name].type == 'foreach': - checker.assert_artifact(step.name, 'this_is_split', True) + elif flow._graph[step.name].type == "foreach": + checker.assert_artifact(step.name, "this_is_split", True) - elif flow._graph[step.name].type == 'join': - checker.assert_artifact('end', 'test_attempt', 3) + elif flow._graph[step.name].type == "join": + checker.assert_artifact("end", "test_attempt", 3) else: - for task in checker.artifact_dict(step.name, 'ex').values(): - extype = 'metaflow.plugins.catch_decorator.'\ - 'FailureHandledByCatch' - assert_equals(extype, str(task['ex'].type)) + for task in checker.artifact_dict(step.name, "ex").values(): + extype = "metaflow.plugins.catch_decorator." "FailureHandledByCatch" + assert_equals(extype, str(task["ex"].type)) break else: raise Exception("No artifact 'ex' in step '%s'" % step.name) @@ -98,9 +105,9 @@ def check_results(self, flow, checker): run = checker.get_run() if run: for step in run: - if step.id == 'end': + if step.id == "end": continue - if flow._graph[step.id].type in ('foreach', 'join'): + if flow._graph[step.id].type in ("foreach", "join"): # 1 normal run + 2 retries = 3 attempts attempts = 3 else: @@ -108,17 +115,17 @@ def check_results(self, flow, checker): attempts = 4 for task in step: data = task.data - got = sorted(m.value for m in task.metadata - if m.type == 'attempt') + got = sorted(m.value for m in task.metadata if m.type == "attempt") assert_equals(list(map(str, range(attempts))), got) - assert_equals(False, 'invisible' in run['start'].task.data) - assert_equals(3, run['start'].task.data.test_attempt) - end = run['end'].task + assert_equals(False, "invisible" in run["start"].task.data) + assert_equals(3, run["start"].task.data.test_attempt) + end = run["end"].task assert_equals(True, end.data.here) assert_equals(3, end.data.test_attempt) # task.exception is None since the exception was handled assert_equals(None, end.exception) - assert_equals('catch me!', end.data.end_ex.exception) - assert_equals('metaflow.exception.ExternalCommandFailed', - end.data.end_ex.type) + assert_equals("catch me!", end.data.end_ex.exception) + assert_equals( + "metaflow.exception.ExternalCommandFailed", end.data.end_ex.type + ) diff --git a/test/core/tests/task_exception.py b/test/core/tests/task_exception.py index d50c8c998fd..73b87cf68fd 100644 --- a/test/core/tests/task_exception.py +++ b/test/core/tests/task_exception.py @@ -1,25 +1,25 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps - class TaskExceptionTest(MetaflowTest): """ A test to validate if exceptions are stored and retrieved correctly """ + PRIORITY = 1 SHOULD_FAIL = True - @steps(0, ['singleton-end'], required=True) + @steps(0, ["singleton-end"], required=True) def step_start(self): - raise KeyError('Something has gone wrong') + raise KeyError("Something has gone wrong") - @steps(2, ['all']) + @steps(2, ["all"]) def step_all(self): pass def check_results(self, flow, checker): run = checker.get_run() if run is not None: - for task in run['end']: - assert_equals('KeyError' in str(task.exception), True) - assert_equals(task.exception.exception,"'Something has gone wrong'") + for task in run["end"]: + assert_equals("KeyError" in str(task.exception), True) + assert_equals(task.exception.exception, "'Something has gone wrong'") diff --git a/test/core/tests/timeout_decorator.py b/test/core/tests/timeout_decorator.py index f4b749b3dca..111878b6486 100644 --- a/test/core/tests/timeout_decorator.py +++ b/test/core/tests/timeout_decorator.py @@ -1,20 +1,23 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + class TimeoutDecoratorTest(MetaflowTest): """ Test that checks that the timeout decorator works as intended. """ + PRIORITY = 2 @tag('catch(var="ex", print_exception=False)') - @tag('timeout(seconds=1)') - @steps(0, ['singleton-start', 'foreach-inner'], required=True) + @tag("timeout(seconds=1)") + @steps(0, ["singleton-start", "foreach-inner"], required=True) def step_sleep(self): self.check = True import time + time.sleep(5) - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): pass @@ -24,9 +27,10 @@ def check_results(self, flow, checker): timeout_raised = False for step in run: for task in step: - if 'check' in task.data: - extype = 'metaflow.plugins.timeout_decorator.'\ - 'TimeoutException' + if "check" in task.data: + extype = ( + "metaflow.plugins.timeout_decorator." "TimeoutException" + ) assert_equals(extype, str(task.data.ex.type)) timeout_raised = True assert_equals(True, timeout_raised) diff --git a/test/core/tests/wide_foreach.py b/test/core/tests/wide_foreach.py index 8ad87f3c7b6..337cf37b69a 100644 --- a/test/core/tests/wide_foreach.py +++ b/test/core/tests/wide_foreach.py @@ -1,23 +1,24 @@ from metaflow_test import MetaflowTest, ExpectationFailed, steps + class WideForeachTest(MetaflowTest): PRIORITY = 3 - @steps(0, ['foreach-split-small'], required=True) + @steps(0, ["foreach-split-small"], required=True) def split(self): self.my_index = None self.arr = range(1200) - @steps(0, ['foreach-inner-small'], required=True) + @steps(0, ["foreach-inner-small"], required=True) def inner(self): self.my_input = self.input - @steps(0, ['foreach-join-small'], required=True) + @steps(0, ["foreach-join-small"], required=True) def join(self, inputs): got = sorted([inp.my_input for inp in inputs]) assert_equals(list(range(1200)), got) - @steps(1, ['all']) + @steps(1, ["all"]) def step_all(self): pass @@ -25,10 +26,5 @@ def check_results(self, flow, checker): run = checker.get_run() if run: # The client API shouldn't choke on many tasks - res = sorted(task.data.my_input - for task in run['foreach_inner']) + res = sorted(task.data.my_input for task in run["foreach_inner"]) assert_equals(list(range(1200)), res) - - - - diff --git a/test/data/__init__.py b/test/data/__init__.py index e4f0ca86928..ba908e93ac0 100644 --- a/test/data/__init__.py +++ b/test/data/__init__.py @@ -2,9 +2,10 @@ # Can set a default path here. Note that you can update the path # if you want a fresh set of data -S3ROOT = os.environ.get('METAFLOW_S3_TEST_ROOT') +S3ROOT = os.environ.get("METAFLOW_S3_TEST_ROOT") from metaflow.datatools.s3util import get_s3_client + s3client, _ = get_s3_client() from metaflow import FlowSpec @@ -13,5 +14,3 @@ # to be defined in test_s3.py. Defining it here works. class FakeFlow(FlowSpec): pass - - diff --git a/test/data/s3/__init__.py b/test/data/s3/__init__.py index b2b70f45f2b..b2a4ba59104 100644 --- a/test/data/s3/__init__.py +++ b/test/data/s3/__init__.py @@ -1,2 +1 @@ # nothing here - diff --git a/test/data/s3/s3_data.py b/test/data/s3/s3_data.py index 9fee0314e4f..8062dd38318 100644 --- a/test/data/s3/s3_data.py +++ b/test/data/s3/s3_data.py @@ -22,23 +22,32 @@ from .. import s3client, S3ROOT BASIC_METADATA = { - 'no_meta': (None, None), # No metadata at all but going through the calls - 'content_no_meta': ('text/plain', None), # Content-type but no metadata - 'no_content_meta': (None, {'userkey': 'UserValue'}), # No content-type but metadata - 'isolation': ('text/plain', {'content-type': 'text/css'}), # Check isolation of user metadata - 'multiple': ('text/plain', { - 'userkey1': 'UserValue1', 'userkey2': 'UserValue2'}), # Multiple metadata - 'complex': ('text/plain', { - 'utf8-data': u'\u523a\u8eab/means sashimi', - 'with-weird-chars': 'Space and !@#<>:/-+=&%'}), + "no_meta": (None, None), # No metadata at all but going through the calls + "content_no_meta": ("text/plain", None), # Content-type but no metadata + "no_content_meta": (None, {"userkey": "UserValue"}), # No content-type but metadata + "isolation": ( + "text/plain", + {"content-type": "text/css"}, + ), # Check isolation of user metadata + "multiple": ( + "text/plain", + {"userkey1": "UserValue1", "userkey2": "UserValue2"}, + ), # Multiple metadata + "complex": ( + "text/plain", + { + "utf8-data": u"\u523a\u8eab/means sashimi", + "with-weird-chars": "Space and !@#<>:/-+=&%", + }, + ), } BASIC_RANGE_INFO = { - 'from_beg': (0, 16), # From beginning - 'exceed_end': (0, 10 * 1024**3), # From beginning, should fetch full file - 'middle': (5, 10), # From middle - 'end': (None, -5), # Fetch from end - 'till_end': (5, None), # Fetch till end + "from_beg": (0, 16), # From beginning + "exceed_end": (0, 10 * 1024 ** 3), # From beginning, should fetch full file + "middle": (5, 10), # From middle + "end": (None, -5), # Fetch from end + "till_end": (5, None), # Fetch till end } # None for file size denotes missing keys @@ -46,47 +55,62 @@ # long BASIC_DATA = [ # empty prefixes should not be a problem - ('empty_prefix', {}), + ("empty_prefix", {}), # requesting non-existent data should be handled ok - ('missing_files', {'missing': None}), + ("missing_files", {"missing": None}), # a basic sanity check - ('3_small_files', {'empty_file': 0, - 'kb_file': 1024, - 'mb_file': 1024**2, - 'missing_file': None}), + ( + "3_small_files", + {"empty_file": 0, "kb_file": 1024, "mb_file": 1024 ** 2, "missing_file": None}, + ), # S3 paths can be longer than the max allowed filename on Linux - ('long_path', {'/'.join('x' * 300): 1024, - # one medium-size path for list_path test - '/'.join('y' * 10): 32, - 'x/x/x': None}), + ( + "long_path", + { + "/".join("x" * 300): 1024, + # one medium-size path for list_path test + "/".join("y" * 10): 32, + "x/x/x": None, + }, + ), # test that nested prefixes work correctly - ('prefix', {'prefix': 32, - 'prefixprefix': 33, - # note that prefix/prefix is both an object and a prefix - 'prefix/prefix': 34, - 'prefix/prefix/prefix': None}), + ( + "prefix", + { + "prefix": 32, + "prefixprefix": 33, + # note that prefix/prefix is both an object and a prefix + "prefix/prefix": 34, + "prefix/prefix/prefix": None, + }, + ), # same filename as above but a different prefix - ('samefile', {'prefix': 42, 'x': 43, 'empty_file': 1, 'xx': None}), + ("samefile", {"prefix": 42, "x": 43, "empty_file": 1, "xx": None}), # crazy file names (it seems '#' characters don't work with boto) - ('crazypath', {u'crazy spaces': 34, - u'\x01\xff': 64, - u'\u523a\u8eab/means sashimi': 33, - u'crazy-!.$%@2_()"\'': 100, - u' /cra._:zy/\x01\x02/p a t h/$this/!!is()': 1000, - u'crazy missing :(': None}) + ( + "crazypath", + { + u"crazy spaces": 34, + u"\x01\xff": 64, + u"\u523a\u8eab/means sashimi": 33, + u"crazy-!.$%@2_()\"'": 100, + u" /cra._:zy/\x01\x02/p a t h/$this/!!is()": 1000, + u"crazy missing :(": None, + }, + ), ] BIG_DATA = [ # test a file > 4GB - ('5gb_file', {'5gb_file': 5 * 1024**3}), + ("5gb_file", {"5gb_file": 5 * 1024 ** 3}), # ensure that e.g. paged listings work correctly with many keys - ('3000_files', {str(i): i for i in range(3000)}) + ("3000_files", {str(i): i for i in range(3000)}), ] # Large file to use for benchmark, must be in BASIC_DATA or BIG_DATA -BENCHMARK_SMALL_FILE = ('3000_files', {'1': 1}) -BENCHMARK_MEDIUM_FILE = ('3_small_files', {'mb_file': 1024**2}) -BENCHMARK_LARGE_FILE = ('5gb_file', {'5gb_file': 5 * 1024**3}) +BENCHMARK_SMALL_FILE = ("3000_files", {"1": 1}) +BENCHMARK_MEDIUM_FILE = ("3_small_files", {"mb_file": 1024 ** 2}) +BENCHMARK_LARGE_FILE = ("5gb_file", {"5gb_file": 5 * 1024 ** 3}) BENCHMARK_SMALL_ITER_MAX = 10001 BENCHMARK_MEDIUM_ITER_MAX = 501 @@ -94,13 +118,15 @@ FAKE_RUN_DATA = [ # test a run id - just a random run id - ('HelloFlow/56', {'one_a': 512, 'one_b': 1024, 'two_c': 8192}) + ("HelloFlow/56", {"one_a": 512, "one_b": 1024, "two_c": 8192}) ] -PUT_PREFIX = 'put_tests' +PUT_PREFIX = "put_tests" ExpectedResult = namedtuple( - 'ExpectedResult', 'size checksum content_type metadata range') + "ExpectedResult", "size checksum content_type metadata range" +) + class RandomFile(object): @@ -115,7 +141,7 @@ def __init__(self, prefix, fname, size): self._data = None def _make_data(self): - numpy.random.seed(zlib.adler32(self.key.encode('utf-8')) & 0xffffffff) + numpy.random.seed(zlib.adler32(self.key.encode("utf-8")) & 0xFFFFFFFF) self._data = numpy.random.bytes(self.size) def checksum(self, start=None, length=None): @@ -127,11 +153,13 @@ def checksum(self, start=None, length=None): if self._data is None: self._make_data() if length < 0: - self.cached_digests[lookup_key] =\ - sha1(self._data[length:]).hexdigest() + self.cached_digests[lookup_key] = sha1( + self._data[length:] + ).hexdigest() else: - self.cached_digests[lookup_key] =\ - sha1(self._data[start:start+length]).hexdigest() + self.cached_digests[lookup_key] = sha1( + self._data[start : start + length] + ).hexdigest() return self.cached_digests[lookup_key] def size_from_range(self, start, length): @@ -165,25 +193,34 @@ def data(self): def url(self): return os.path.join(S3ROOT, self.key) + def _format_test_cases(dataset, meta=None, ranges=None): cases = [] ids = [] for prefix, filespecs in dataset: - objs = [RandomFile(prefix, fname, size) - for fname, size in filespecs.items()] + objs = [RandomFile(prefix, fname, size) for fname, size in filespecs.items()] objs = {obj.url: (obj, None, None) for obj in objs} if meta: # We generate one per meta info for metaname, (content_type, usermeta) in meta.items(): - objs.update({"%s_%s" % (obj.url, metaname): \ - (obj, content_type, usermeta) for (obj, _, _) in objs.values()}) + objs.update( + { + "%s_%s" % (obj.url, metaname): (obj, content_type, usermeta) + for (obj, _, _) in objs.values() + } + ) files = { - k: {None: ExpectedResult( - size=obj.size, - checksum=obj.checksum(), - content_type=content_type, - metadata=usermeta, - range=None)} for k, (obj, content_type, usermeta) in objs.items()} + k: { + None: ExpectedResult( + size=obj.size, + checksum=obj.checksum(), + content_type=content_type, + metadata=usermeta, + range=None, + ) + } + for k, (obj, content_type, usermeta) in objs.items() + } if ranges: # For every file we have in files, we calculate the proper # checksum and create a new dictionary @@ -194,41 +231,48 @@ def _format_test_cases(dataset, meta=None, ranges=None): checksum=obj.checksum(offset, length), content_type=content_type, metadata=usermeta, - range=(offset, length)) + range=(offset, length), + ) ids.append(prefix) cases.append((S3ROOT, [prefix], files)) return cases, ids + def pytest_fakerun_cases(): cases, ids = _format_test_cases(FAKE_RUN_DATA) - return {'argvalues': cases, 'ids': ids} + return {"argvalues": cases, "ids": ids} + def pytest_basic_case(): cases, ids = _format_test_cases( - BASIC_DATA, ranges=BASIC_RANGE_INFO, meta=BASIC_METADATA) - return {'argvalues': cases, 'ids': ids} + BASIC_DATA, ranges=BASIC_RANGE_INFO, meta=BASIC_METADATA + ) + return {"argvalues": cases, "ids": ids} + def pytest_large_case(): cases, ids = _format_test_cases(BASIC_DATA, meta=BASIC_METADATA) cases_big, ids_big = _format_test_cases(BIG_DATA) cases.extend(cases_big) ids.extend(ids_big) - return {'argvalues': cases, 'ids': ids} + return {"argvalues": cases, "ids": ids} + def pytest_benchmark_case(): cases, _ = _format_test_cases([BENCHMARK_LARGE_FILE]) - ids = ['5gb'] + ids = ["5gb"] new_cases, _ = _format_test_cases([BENCHMARK_MEDIUM_FILE]) cases.extend(new_cases) - ids.append('1mb') + ids.append("1mb") new_cases, _ = _format_test_cases([BENCHMARK_SMALL_FILE]) cases.extend(new_cases) - ids.append('1b') + ids.append("1b") + + return {"argvalues": cases, "ids": ids} - return {'argvalues': cases, 'ids': ids} def pytest_benchmark_many_case(): large_case = _format_test_cases([BENCHMARK_LARGE_FILE])[0][0] @@ -254,30 +298,41 @@ def pytest_benchmark_many_case(): continue # At this point, form the test id_name = "%ds_%dm_%dl" % (small_count, medium_count, large_count) - cases.append(( - S3ROOT, [], - [(small_count, small_case[2]), - (medium_count, medium_case[2]), - (large_count, large_case[2])])) + cases.append( + ( + S3ROOT, + [], + [ + (small_count, small_case[2]), + (medium_count, medium_case[2]), + (large_count, large_case[2]), + ], + ) + ) ids.append(id_name) - return {'argvalues': cases, 'ids': ids} + return {"argvalues": cases, "ids": ids} + def pytest_benchmark_put_case(): put_prefix = os.path.join(S3ROOT, PUT_PREFIX) cases = [] ids = [] - for prefix, filespecs in \ - [BENCHMARK_LARGE_FILE, BENCHMARK_MEDIUM_FILE, BENCHMARK_SMALL_FILE]: + for prefix, filespecs in [ + BENCHMARK_LARGE_FILE, + BENCHMARK_MEDIUM_FILE, + BENCHMARK_SMALL_FILE, + ]: blobs = [] for fname, size in filespecs.items(): blobs.append((prefix, fname, size)) cases.append((put_prefix, blobs, None)) - ids = ['5gb', '1mb', '1b'] - return {'argvalues': cases, 'ids': ids} + ids = ["5gb", "1mb", "1b"] + return {"argvalues": cases, "ids": ids} + def pytest_benchmark_put_many_case(): single_cases_and_ids = pytest_benchmark_put_case() - single_cases = single_cases_and_ids['argvalues'] + single_cases = single_cases_and_ids["argvalues"] large_blob = single_cases[0][1][0] medium_blob = single_cases[1][1][0] small_blob = single_cases[2][1][0] @@ -302,11 +357,14 @@ def pytest_benchmark_put_many_case(): # At this point, form the test id_name = "%ds_%dm_%dl" % (small_count, medium_count, large_count) blobs = [ - (small_count, small_blob), (medium_count, medium_blob), - (large_count, large_blob)] + (small_count, small_blob), + (medium_count, medium_blob), + (large_count, large_blob), + ] cases.append((put_prefix, blobs, None)) ids.append(id_name) - return {'argvalues': cases, 'ids': ids} + return {"argvalues": cases, "ids": ids} + def pytest_many_prefixes_case(): cases, ids = _format_test_cases(BASIC_DATA, meta=BASIC_METADATA) @@ -316,44 +374,56 @@ def pytest_many_prefixes_case(): many_prefixes.append(prefix) many_prefixes_expected.update(files) # add many prefixes cases - ids.append('many_prefixes') + ids.append("many_prefixes") cases.append((S3ROOT, many_prefixes, many_prefixes_expected)) - return {'argvalues': cases, 'ids': ids} + return {"argvalues": cases, "ids": ids} + def pytest_put_strings_case(meta=None): put_prefix = os.path.join(S3ROOT, PUT_PREFIX) - data = [u"unicode: \u523a\u8eab means sashimi", - b"bytes: \x00\x01\x02", - "just a string"] + data = [ + u"unicode: \u523a\u8eab means sashimi", + b"bytes: \x00\x01\x02", + "just a string", + ] expected = {} objs = [] for text in data: blob = to_bytes(text) checksum = sha1(blob).hexdigest() key = str(uuid4()) - expected[os.path.join(put_prefix, key)] = {None: ExpectedResult( - size=len(blob), - checksum=checksum, - content_type=None, - metadata=None, - range=None)} + expected[os.path.join(put_prefix, key)] = { + None: ExpectedResult( + size=len(blob), + checksum=checksum, + content_type=None, + metadata=None, + range=None, + ) + } objs.append((key, text)) if meta is not None: for content_type, usermeta in meta.values(): key = str(uuid4()) - expected[os.path.join(put_prefix, key)] = {None: ExpectedResult( - size=len(blob), - checksum=checksum, - content_type=content_type, - metadata=usermeta, - range=None)} - objs.append(S3PutObject( - key=key, - value=text, - content_type=content_type, - metadata=usermeta)) - return {'argvalues': [(put_prefix, objs, expected)], - 'ids': ['put_strings']} + expected[os.path.join(put_prefix, key)] = { + None: ExpectedResult( + size=len(blob), + checksum=checksum, + content_type=content_type, + metadata=usermeta, + range=None, + ) + } + objs.append( + S3PutObject( + key=key, + value=text, + content_type=content_type, + metadata=usermeta, + ) + ) + return {"argvalues": [(put_prefix, objs, expected)], "ids": ["put_strings"]} + def pytest_put_blobs_case(meta=None): put_prefix = os.path.join(S3ROOT, PUT_PREFIX) @@ -366,47 +436,62 @@ def pytest_put_blobs_case(meta=None): blob = RandomFile(prefix, fname, size) checksum = blob.checksum() key = str(uuid4()) - expected[os.path.join(put_prefix, key)] = {None: ExpectedResult( - size=blob.size, - checksum=checksum, - content_type=None, - metadata=None, - range=None)} + expected[os.path.join(put_prefix, key)] = { + None: ExpectedResult( + size=blob.size, + checksum=checksum, + content_type=None, + metadata=None, + range=None, + ) + } blobs.append((key, blob.data)) if meta is not None: for content_type, usermeta in meta.values(): key = str(uuid4()) - expected[os.path.join(put_prefix, key)] = {None: ExpectedResult( - size=len(blob), - checksum=checksum, - content_type=content_type, - metadata=usermeta, - range=None)} - blobs.append(S3PutObject( - key=key, - value=blob.data, - content_type=content_type, - metadata=usermeta)) + expected[os.path.join(put_prefix, key)] = { + None: ExpectedResult( + size=len(blob), + checksum=checksum, + content_type=content_type, + metadata=usermeta, + range=None, + ) + } + blobs.append( + S3PutObject( + key=key, + value=blob.data, + content_type=content_type, + metadata=usermeta, + ) + ) ids.append(prefix) cases.append((put_prefix, blobs, expected)) - return {'argvalues': cases, 'ids': ids} + return {"argvalues": cases, "ids": ids} + def ensure_test_data(): # update S3ROOT in __init__.py to get a fresh set of data - print('Ensuring that test data exists at %s' % S3ROOT) - mark = urlparse(os.path.join(S3ROOT, 'ALL_OK')) + print("Ensuring that test data exists at %s" % S3ROOT) + mark = urlparse(os.path.join(S3ROOT, "ALL_OK")) try: # Check if the data exists and has been modified in the last # 29 days (this should be lower than the TTL for your bucket to ensure # the data is available for the test) import datetime + today = datetime.date.today() delta = datetime.timedelta(days=29) - s3client.head_object(Bucket=mark.netloc, Key=mark.path.lstrip('/'), - IfModifiedSince=str(today - delta)) - print('All data ok.') + s3client.head_object( + Bucket=mark.netloc, + Key=mark.path.lstrip("/"), + IfModifiedSince=str(today - delta), + ) + print("All data ok.") except: - print('Uploading test data') + print("Uploading test data") + def _do_upload(prefix, filespecs, meta=None): for fname, size in filespecs.items(): if size is not None: @@ -415,38 +500,47 @@ def _do_upload(prefix, filespecs, meta=None): # For metadata, we don't actually touch RandomFile # (since it is the same) but we modify the path to post-pend # the name - print('Test data case %s: upload to %s started' % (prefix, f.url)) - s3client.upload_fileobj(f.fileobj(), - url.netloc, - url.path.lstrip('/')) - print('Test data case %s: uploaded to %s' % (prefix, f.url)) + print("Test data case %s: upload to %s started" % (prefix, f.url)) + s3client.upload_fileobj( + f.fileobj(), url.netloc, url.path.lstrip("/") + ) + print("Test data case %s: uploaded to %s" % (prefix, f.url)) if meta is not None: for metaname, metainfo in meta.items(): new_url = "%s_%s" % (f.url, metaname) url = urlparse(new_url) - print('Test data case %s: upload to %s started' % (prefix, new_url)) + print( + "Test data case %s: upload to %s started" + % (prefix, new_url) + ) extra = {} content_type, user_meta = metainfo if content_type: - extra['ContentType'] = content_type + extra["ContentType"] = content_type if user_meta: new_meta = { - 'metaflow-user-attributes': json.dumps(user_meta)} - extra['Metadata'] = new_meta - s3client.upload_fileobj(f.fileobj(), - url.netloc, - url.path.lstrip('/'), - ExtraArgs=extra) - print('Test data case %s: uploaded to %s' % (prefix, new_url)) + "metaflow-user-attributes": json.dumps(user_meta) + } + extra["Metadata"] = new_meta + s3client.upload_fileobj( + f.fileobj(), + url.netloc, + url.path.lstrip("/"), + ExtraArgs=extra, + ) + print( + "Test data case %s: uploaded to %s" % (prefix, new_url) + ) for prefix, filespecs in BIG_DATA + FAKE_RUN_DATA: _do_upload(prefix, filespecs) for prefix, filespecs in BASIC_DATA: _do_upload(prefix, filespecs, meta=BASIC_METADATA) - s3client.upload_fileobj(to_fileobj('ok'), - Bucket=mark.netloc, - Key=mark.path.lstrip('/')) - print('Test data uploaded ok') + s3client.upload_fileobj( + to_fileobj("ok"), Bucket=mark.netloc, Key=mark.path.lstrip("/") + ) + print("Test data uploaded ok") + ensure_test_data() diff --git a/test/data/s3/test_s3.py b/test/data/s3/test_s3.py index 0284c7d3f91..30f6a72c030 100644 --- a/test/data/s3/test_s3.py +++ b/test/data/s3/test_s3.py @@ -10,12 +10,14 @@ import pytest from metaflow import current, namespace, Run -from metaflow.datatools.s3 import S3,\ - MetaflowS3AccessDenied,\ - MetaflowS3NotFound,\ - MetaflowS3URLException,\ - MetaflowS3InvalidObject,\ - S3PutObject +from metaflow.datatools.s3 import ( + S3, + MetaflowS3AccessDenied, + MetaflowS3NotFound, + MetaflowS3URLException, + MetaflowS3InvalidObject, + S3PutObject, +) from metaflow.util import to_bytes, unicode_type @@ -29,6 +31,7 @@ # python3 from urllib.parse import urlparse + def assert_results(s3objs, expected, info_should_be_empty=False, info_only=False): # did we receive all expected objects and nothing else? if info_only: @@ -75,7 +78,7 @@ def assert_results(s3objs, expected, info_should_be_empty=False, info_only=False # blob is ok? blob = s3obj.blob assert len(blob) == size - assert type(blob) == type(b'') + assert type(blob) == type(b"") assert sha1(blob).hexdigest() == checksum # size is ok? assert s3obj.size == size @@ -85,7 +88,7 @@ def assert_results(s3objs, expected, info_should_be_empty=False, info_only=False # Content_type is OK if content_type is None: # Default content-type when nothing is supplied - assert s3obj.content_type == 'binary/octet-stream' + assert s3obj.content_type == "binary/octet-stream" else: assert s3obj.content_type == content_type # metadata is OK @@ -100,8 +103,10 @@ def assert_results(s3objs, expected, info_should_be_empty=False, info_only=False assert v1 == v, "Metadata %s mismatch" % k found.add(k) extra_keys = set(s3objmetadata.keys()) - found - assert not extra_keys, \ - "Additional metadata present %s" % str(extra_keys) + assert not extra_keys, "Additional metadata present %s" % str( + extra_keys + ) + def shuffle(objs): for i, (key, value) in enumerate(objs): @@ -109,6 +114,7 @@ def shuffle(objs): key_t, value_t = objs[t] objs[i], objs[t] = (key, value_t), (key_t, value) + def deranged_shuffle(objs): shuffled_objs = objs[:] while True: @@ -119,15 +125,16 @@ def deranged_shuffle(objs): else: return shuffled_objs + @pytest.fixture def tempdir(): - tmpdir = mkdtemp(dir='.', prefix='metaflow.test.tmp') + tmpdir = mkdtemp(dir=".", prefix="metaflow.test.tmp") yield tmpdir shutil.rmtree(tmpdir) + @pytest.mark.parametrize( - argnames=['s3root', 'pathspecs', 'expected'], - **s3_data.pytest_benchmark_case() + argnames=["s3root", "pathspecs", "expected"], **s3_data.pytest_benchmark_case() ) @pytest.mark.benchmark(max_time=30) def test_info_one_benchmark(benchmark, s3root, pathspecs, expected): @@ -137,32 +144,35 @@ def _do(): for url in expected: res.append(s3.info(url)) return res + res = benchmark(_do) assert_results(res, expected, info_only=True) + @pytest.mark.parametrize( - argnames=['s3root', 'pathspecs', 'expected'], - **s3_data.pytest_benchmark_many_case() + argnames=["s3root", "pathspecs", "expected"], **s3_data.pytest_benchmark_many_case() ) @pytest.mark.benchmark(max_time=30) def test_info_many_benchmark(benchmark, s3root, pathspecs, expected): urls = [] check_expected = {} for count, v in expected: - urls.extend(list(v)*count) + urls.extend(list(v) * count) if count > 0: check_expected.update(v) random.shuffle(urls) + def _do(): with S3() as s3: res = s3.info_many(urls) return res + res = benchmark(_do) assert_results(res, check_expected, info_only=True) + @pytest.mark.parametrize( - argnames=['s3root', 'pathspecs', 'expected'], - **s3_data.pytest_benchmark_case() + argnames=["s3root", "pathspecs", "expected"], **s3_data.pytest_benchmark_case() ) @pytest.mark.benchmark(max_time=60) def test_get_one_benchmark(benchmark, s3root, pathspecs, expected): @@ -173,35 +183,38 @@ def _do(): # Use return_missing as this is the most expensive path res.append(s3.get(url, return_missing=True)) return res + res = benchmark(_do) # We do not actually check results because the files will be cleared # Could be improved if we want to be real precise # assert_results(res, expected, info_should_be_empty=True) + @pytest.mark.parametrize( - argnames=['s3root', 'pathspecs', 'expected'], - **s3_data.pytest_benchmark_many_case() + argnames=["s3root", "pathspecs", "expected"], **s3_data.pytest_benchmark_many_case() ) @pytest.mark.benchmark(max_time=60) def test_get_many_benchmark(benchmark, s3root, pathspecs, expected): urls = [] check_expected = {} for count, v in expected: - urls.extend(list(v)*count) + urls.extend(list(v) * count) if count > 0: check_expected.update(v) random.shuffle(urls) + def _do(): with S3() as s3: # Use return_missing as this is the most expensive path res = s3.get_many(urls, return_missing=True) return res + res = benchmark(_do) # assert_results(res, check_expected, info_should_be_empty=True) + @pytest.mark.parametrize( - argnames=['s3root', 'blobs', 'expected'], - **s3_data.pytest_benchmark_put_case() + argnames=["s3root", "blobs", "expected"], **s3_data.pytest_benchmark_put_case() ) @pytest.mark.benchmark(max_time=60) def test_put_one_benchmark(benchmark, tempdir, s3root, blobs, expected): @@ -213,23 +226,26 @@ def _generate_files(blobs): data = s3_data.RandomFile(prefix, fname, size) key = str(uuid4()) path = os.path.join(tempdir, key) - with open(path, 'wb') as f: + with open(path, "wb") as f: f.write(data.data) yield key, path + # Generate all files before the test so we don't time this all_files = list(_generate_files(blobs)) + def _do(): with S3(s3root=s3root) as s3: res = [] for key, obj in all_files: - key = str(uuid4()) # New "name" every time + key = str(uuid4()) # New "name" every time res.append(s3.put(key, obj, overwrite=False)) return res + res = benchmark(_do) + @pytest.mark.parametrize( - argnames=['s3root', 'blobs', 'expected'], - **s3_data.pytest_benchmark_put_many_case() + argnames=["s3root", "blobs", "expected"], **s3_data.pytest_benchmark_put_many_case() ) @pytest.mark.benchmark(max_time=60) def test_put_many_benchmark(benchmark, tempdir, s3root, blobs, expected): @@ -245,26 +261,29 @@ def _generate_files(blobs): data = s3_data.RandomFile(prefix, fname, size) key = str(uuid4()) path = os.path.join(tempdir, key) - with open(path, 'wb') as f: + with open(path, "wb") as f: f.write(data.data) generated_paths[blob_info] = path for _ in range(count): yield str(uuid4()), path + all_files = list(_generate_files(blobs)) + def _do(): new_files = [(str(uuid4()), path) for _, path in all_files] with S3(s3root=s3root) as s3: s3urls = s3.put_files(new_files, overwrite=False) return s3urls + res = benchmark(_do) + @pytest.mark.parametrize( - argnames=['s3root', 'pathspecs', 'expected'], - **s3_data.pytest_fakerun_cases() + argnames=["s3root", "pathspecs", "expected"], **s3_data.pytest_fakerun_cases() ) def test_init_options(s3root, pathspecs, expected): [pathspec] = pathspecs - flow_name, run_id = pathspec.split('/') + flow_name, run_id = pathspec.split("/") plen = len(s3root) # option 1) s3root as prefix @@ -275,7 +294,7 @@ def test_init_options(s3root, pathspecs, expected): assert s3obj.key == url[plen:] assert_results([s3obj], {url: exp}) with pytest.raises(MetaflowS3URLException): - s3.get('s3://some/fake/address') + s3.get("s3://some/fake/address") # option 2) full url as s3root for url, exp in expected.items(): @@ -291,13 +310,13 @@ def test_init_options(s3root, pathspecs, expected): assert s3obj.key == url assert_results([s3obj], {url: exp}) with pytest.raises(MetaflowS3URLException): - s3.get('suffix') + s3.get("suffix") with pytest.raises(MetaflowS3URLException): - s3.get('s3://nopath') + s3.get("s3://nopath") with pytest.raises(MetaflowS3URLException): - s3.get_many(['suffixes']) + s3.get_many(["suffixes"]) with pytest.raises(MetaflowS3URLException): - s3.get_recursive(['suffixes']) + s3.get_recursive(["suffixes"]) with pytest.raises(MetaflowS3URLException): s3.get_all() @@ -310,21 +329,17 @@ def test_init_options(s3root, pathspecs, expected): with S3(run=flow): pass - current._set_env(flow_name, - run_id, - 'no_step', - 'no_task', - 'no_origin_run_id', - 'no_ns', - 'no_user') + current._set_env( + flow_name, run_id, "no_step", "no_task", "no_origin_run_id", "no_ns", "no_user" + ) with S3(bucket=parsed.netloc, prefix=parsed.path, run=flow) as s3: for url, exp in expected.items(): - name = url.split('/')[-1] + name = url.split("/")[-1] s3obj = s3.get(name) assert s3obj.key == name assert_results([s3obj], {url: exp}) - names = [url.split('/')[-1] for url in expected] + names = [url.split("/")[-1] for url in expected] s3objs = s3.get_many(names) assert {e.key for e in s3objs} == set(names) assert_results(s3objs, expected) @@ -332,8 +347,7 @@ def test_init_options(s3root, pathspecs, expected): @pytest.mark.parametrize( - argnames=['s3root', 'prefixes', 'expected'], - **s3_data.pytest_basic_case() + argnames=["s3root", "prefixes", "expected"], **s3_data.pytest_basic_case() ) def test_info_one(s3root, prefixes, expected): with S3() as s3: @@ -344,16 +358,14 @@ def test_info_one(s3root, prefixes, expected): s3obj = s3.info(url) # test return_missing=True s3obj = s3.info(url, return_missing=True) - assert_results( - [s3obj], {url: expected[url]}, info_only=True) + assert_results([s3obj], {url: expected[url]}, info_only=True) else: s3obj = s3.info(url) - assert_results( - [s3obj], {url: expected[url]}, info_only=True) + assert_results([s3obj], {url: expected[url]}, info_only=True) + @pytest.mark.parametrize( - argnames=['s3root', 'prefixes', 'expected'], - **s3_data.pytest_basic_case() + argnames=["s3root", "prefixes", "expected"], **s3_data.pytest_basic_case() ) def test_info_many(s3root, prefixes, expected): with S3() as s3: @@ -361,15 +373,13 @@ def test_info_many(s3root, prefixes, expected): # to test result ordering, make sure we are requesting # keys in a non-lexicographic order - not_missing = [url for url, v in expected.items() - if v[None].size is not None] + not_missing = [url for url, v in expected.items() if v[None].size is not None] urls = list(sorted(not_missing, reverse=True)) s3objs = s3.info_many(urls) # results should come out in the order of keys requested assert urls == [e.url for e in s3objs] - assert_results( - s3objs, {k: expected[k] for k in not_missing}, info_only=True) + assert_results(s3objs, {k: expected[k] for k in not_missing}, info_only=True) # 2) test with missing items, default case if not_missing != list(expected): @@ -386,26 +396,26 @@ def test_info_many(s3root, prefixes, expected): assert urls == [e.url for e in s3objs] assert_results(s3objs, expected, info_only=True) + @pytest.mark.parametrize( - argnames=['s3root', 'prefixes', 'expected'], - **s3_data.pytest_fakerun_cases() + argnames=["s3root", "prefixes", "expected"], **s3_data.pytest_fakerun_cases() ) def test_get_exceptions(s3root, prefixes, expected): # get_many() goes via s3op, get() is a method - test both the code paths with S3() as s3: with pytest.raises(MetaflowS3AccessDenied): - s3.get_many(['s3://foobar/foo']) + s3.get_many(["s3://foobar/foo"]) with pytest.raises(MetaflowS3AccessDenied): - s3.get('s3://foobar/foo') + s3.get("s3://foobar/foo") with S3(s3root=s3root) as s3: with pytest.raises(MetaflowS3NotFound): - s3.get_many(['this_file_does_not_exist']) + s3.get_many(["this_file_does_not_exist"]) with pytest.raises(MetaflowS3NotFound): - s3.get('this_file_does_not_exist') + s3.get("this_file_does_not_exist") + @pytest.mark.parametrize( - argnames=['s3root', 'prefixes', 'expected'], - **s3_data.pytest_basic_case() + argnames=["s3root", "prefixes", "expected"], **s3_data.pytest_basic_case() ) def test_get_one(s3root, prefixes, expected): with S3() as s3: @@ -416,16 +426,14 @@ def test_get_one(s3root, prefixes, expected): s3obj = s3.get(url) # test return_missing=True s3obj = s3.get(url, return_missing=True) - assert_results( - [s3obj], {url: expected[url]}) + assert_results([s3obj], {url: expected[url]}) else: s3obj = s3.get(url, return_info=True) - assert_results( - [s3obj], {url: expected[url]}) + assert_results([s3obj], {url: expected[url]}) + @pytest.mark.parametrize( - argnames=['s3root', 'prefixes', 'expected'], - **s3_data.pytest_basic_case() + argnames=["s3root", "prefixes", "expected"], **s3_data.pytest_basic_case() ) def test_get_one_wo_meta(s3root, prefixes, expected): with S3() as s3: @@ -435,48 +443,44 @@ def test_get_one_wo_meta(s3root, prefixes, expected): with pytest.raises(MetaflowS3NotFound): s3obj = s3.get(url) s3obj = s3.get(url, return_missing=True, return_info=False) - assert_results( - [s3obj], {url: expected[url]}, info_should_be_empty=True) + assert_results([s3obj], {url: expected[url]}, info_should_be_empty=True) else: s3obj = s3.get(url, return_info=False) - assert_results( - [s3obj], {url: expected[url]}, info_should_be_empty=True) + assert_results([s3obj], {url: expected[url]}, info_should_be_empty=True) + @pytest.mark.parametrize( - argnames=['s3root', 'prefixes', 'expected'], - **s3_data.pytest_large_case() + argnames=["s3root", "prefixes", "expected"], **s3_data.pytest_large_case() ) def test_get_all(s3root, prefixes, expected): - expected_exists = {url: v - for url, v in expected.items() - if v[None].size is not None} + expected_exists = { + url: v for url, v in expected.items() if v[None].size is not None + } for prefix in prefixes: with S3(s3root=os.path.join(s3root, prefix)) as s3: s3objs = s3.get_all() # results should be in lexicographic order - assert list(sorted(e.url for e in s3objs))\ - == [e.url for e in s3objs] + assert list(sorted(e.url for e in s3objs)) == [e.url for e in s3objs] assert_results(s3objs, expected_exists, info_should_be_empty=True) + @pytest.mark.parametrize( - argnames=['s3root', 'prefixes', 'expected'], - **s3_data.pytest_basic_case() + argnames=["s3root", "prefixes", "expected"], **s3_data.pytest_basic_case() ) def test_get_all_with_meta(s3root, prefixes, expected): - expected_exists = {url: v - for url, v in expected.items() - if v[None].size is not None} + expected_exists = { + url: v for url, v in expected.items() if v[None].size is not None + } for prefix in prefixes: with S3(s3root=os.path.join(s3root, prefix)) as s3: s3objs = s3.get_all(return_info=True) # results should be in lexicographic order - assert list(sorted(e.url for e in s3objs))\ - == [e.url for e in s3objs] + assert list(sorted(e.url for e in s3objs)) == [e.url for e in s3objs] assert_results(s3objs, expected_exists) + @pytest.mark.parametrize( - argnames=['s3root', 'prefixes', 'expected'], - **s3_data.pytest_basic_case() + argnames=["s3root", "prefixes", "expected"], **s3_data.pytest_basic_case() ) def test_get_many(s3root, prefixes, expected): with S3() as s3: @@ -484,15 +488,13 @@ def test_get_many(s3root, prefixes, expected): # to test result ordering, make sure we are requesting # keys in a non-lexicographic order - not_missing = [url for url, v in expected.items() - if v[None].size is not None] + not_missing = [url for url, v in expected.items() if v[None].size is not None] urls = list(sorted(not_missing, reverse=True)) s3objs = s3.get_many(urls, return_info=True) # results should come out in the order of keys requested assert urls == [e.url for e in s3objs] - assert_results( - s3objs, {k: expected[k] for k in not_missing}) + assert_results(s3objs, {k: expected[k] for k in not_missing}) # 2) test with missing items, default case if not_missing != list(expected): @@ -509,9 +511,9 @@ def test_get_many(s3root, prefixes, expected): assert urls == [e.url for e in s3objs] assert_results(s3objs, expected) + @pytest.mark.parametrize( - argnames=['s3root', 'prefixes', 'expected'], - **s3_data.pytest_many_prefixes_case() + argnames=["s3root", "prefixes", "expected"], **s3_data.pytest_many_prefixes_case() ) def test_list_paths(s3root, prefixes, expected): def urls_by_prefix(prefix): @@ -538,14 +540,15 @@ def urls_by_prefix(prefix): # 2) test querying by many prefixes with S3(s3root=s3root) as s3: s3objs = s3.list_paths(prefixes) - assert frozenset(e.prefix.rstrip('/').split('/')[-1] - for e in s3objs) == non_empty + assert ( + frozenset(e.prefix.rstrip("/").split("/")[-1] for e in s3objs) == non_empty + ) for prefix, exp in matches.items(): - exists = frozenset(e.url for e in s3objs - if e.prefix == prefix and e.exists) - not_exists = frozenset(e.url for e in s3objs - if e.prefix == prefix and not e.exists) + exists = frozenset(e.url for e in s3objs if e.prefix == prefix and e.exists) + not_exists = frozenset( + e.url for e in s3objs if e.prefix == prefix and not e.exists + ) # every object should be expected assert all(e in exp for e in exists) # not existing ones are prefixes, they shouldn't match @@ -560,8 +563,8 @@ def urls_by_prefix(prefix): s3objs = s3.list_paths([url]) assert [e for e in s3objs if e.exists] == [] else: - suffix = url[len(s3root):] - expected_keys = suffix.split('/') + suffix = url[len(s3root) :] + expected_keys = suffix.split("/") if len(expected_keys) > 20: # speed optimization: exclude crazy long paths continue @@ -572,27 +575,29 @@ def urls_by_prefix(prefix): # are we at the leaf? if idx == len(expected_keys) - 1: # a leaf object should always exist - [match] = [e for e in s3objs - if e.key == expected_key and e.exists] + [match] = [ + e for e in s3objs if e.key == expected_key and e.exists + ] else: # a non-leaf may match objects that are also prefixes - [match] = [e for e in s3objs - if e.key == expected_key and not e.exists] + [match] = [ + e for e in s3objs if e.key == expected_key and not e.exists + ] # prefix + key == url - assert os.path.join(match.prefix, match.key) ==\ - match.url.rstrip('/') + assert os.path.join(match.prefix, match.key) == match.url.rstrip( + "/" + ) got_url = match.url # the leaf should be the object itself assert match.url == url + @pytest.mark.parametrize( - argnames=['s3root', 'prefixes', 'expected'], - **s3_data.pytest_many_prefixes_case() + argnames=["s3root", "prefixes", "expected"], **s3_data.pytest_many_prefixes_case() ) def test_list_recursive(s3root, prefixes, expected): - not_missing = [url for url, v in expected.items() - if v[None].size is not None] + not_missing = [url for url, v in expected.items() if v[None].size is not None] with S3(s3root=s3root) as s3: s3objs = s3.list_recursive(prefixes) assert frozenset(e.url for e in s3objs) == frozenset(not_missing) @@ -601,23 +606,26 @@ def test_list_recursive(s3root, prefixes, expected): # list_recursive returns leaves only assert all(e.exists for e in s3objs) + @pytest.mark.parametrize( - argnames=['s3root', 'prefixes', 'expected'], - **s3_data.pytest_many_prefixes_case() + argnames=["s3root", "prefixes", "expected"], **s3_data.pytest_many_prefixes_case() ) def test_get_recursive(s3root, prefixes, expected): - expected_exists = {url: v - for url, v in expected.items() - if v[None].size is not None} + expected_exists = { + url: v for url, v in expected.items() if v[None].size is not None + } local_files = [] with S3(s3root=s3root) as s3: s3objs = s3.get_recursive(prefixes) # we need to deduce which prefixes actually produce results nonempty_prefixes = list( - filter(lambda p: any(url.startswith(os.path.join(s3root, p)) - for url in expected_exists), - prefixes) + filter( + lambda p: any( + url.startswith(os.path.join(s3root, p)) for url in expected_exists + ), + prefixes, + ) ) # prefixes must be returned in the order of prefixes requested @@ -638,7 +646,7 @@ def test_get_recursive(s3root, prefixes, expected): if len(prefixes) == 1: [prefix] = prefixes s3root = os.path.join(s3root, prefix) - keys = {url[len(s3root) + 1:] for url in expected_exists} + keys = {url[len(s3root) + 1 :] for url in expected_exists} assert {e.key for e in s3objs} == keys local_files = [s3obj.path for s3obj in s3objs] @@ -646,20 +654,21 @@ def test_get_recursive(s3root, prefixes, expected): for path in local_files: assert not os.path.exists(path) + def test_put_exceptions(): with S3() as s3: with pytest.raises(MetaflowS3InvalidObject): - s3.put_many([('a', 1)]) + s3.put_many([("a", 1)]) with pytest.raises(MetaflowS3InvalidObject): - s3.put('a', 1) + s3.put("a", 1) with pytest.raises(MetaflowS3NotFound): - s3.put_files([('a', '/non-existent/local-file')]) + s3.put_files([("a", "/non-existent/local-file")]) with pytest.raises(MetaflowS3URLException): - s3.put_many([('foo', 'bar')]) + s3.put_many([("foo", "bar")]) + @pytest.mark.parametrize( - argnames=['s3root', 'objs', 'expected'], - **s3_data.pytest_put_strings_case() + argnames=["s3root", "objs", "expected"], **s3_data.pytest_put_strings_case() ) def test_put_many(s3root, objs, expected): with S3(s3root=s3root) as s3: @@ -686,8 +695,7 @@ def test_put_many(s3root, objs, expected): @pytest.mark.parametrize( - argnames=['s3root', 'objs', 'expected'], - **s3_data.pytest_put_strings_case() + argnames=["s3root", "objs", "expected"], **s3_data.pytest_put_strings_case() ) def test_put_one(s3root, objs, expected): with S3(s3root=s3root) as s3: @@ -706,25 +714,24 @@ def test_put_one(s3root, objs, expected): assert_results([s3obj], {s3url: expected[s3url]}) assert s3obj.blob == to_bytes(obj) + @pytest.mark.parametrize( - argnames=['s3root', 'blobs', 'expected'], - **s3_data.pytest_put_blobs_case() + argnames=["s3root", "blobs", "expected"], **s3_data.pytest_put_blobs_case() ) def test_put_files(tempdir, s3root, blobs, expected): def _files(blobs): for blob in blobs: - key = getattr(blob, 'key', blob[0]) - data = getattr(blob, 'value', blob[1]) - content_type = getattr(blob, 'content_type', None) - metadata = getattr(blob, 'metadata', None) + key = getattr(blob, "key", blob[0]) + data = getattr(blob, "value", blob[1]) + content_type = getattr(blob, "content_type", None) + metadata = getattr(blob, "metadata", None) path = os.path.join(tempdir, key) - with open(path, 'wb') as f: + with open(path, "wb") as f: f.write(data) yield S3PutObject( - key=key, - value=path, - content_type=content_type, - metadata=metadata) + key=key, value=path, content_type=content_type, metadata=metadata + ) + with S3(s3root=s3root) as s3: s3urls = s3.put_files(_files(blobs)) assert list(dict(s3urls)) == list(dict(blobs)) @@ -743,7 +750,9 @@ def _files(blobs): shuffled_blobs = blobs[:] shuffle(shuffled_blobs) with S3(s3root=s3root) as s3: - overwrite_disabled_s3urls = s3.put_files(_files(shuffled_blobs), overwrite=False) + overwrite_disabled_s3urls = s3.put_files( + _files(shuffled_blobs), overwrite=False + ) assert len(overwrite_disabled_s3urls) == 0 with S3() as s3: diff --git a/test/env_escape/example.py b/test/env_escape/example.py index 9691c2143a0..bef9d0871f7 100644 --- a/test/env_escape/example.py +++ b/test/env_escape/example.py @@ -3,18 +3,31 @@ from metaflow import FlowSpec, step, conda + def run_test(through_escape=False): # NOTE: This will be the same for both escaped path and non-escaped path # if the library test_lib is installed. For the unescaped path, we pretend # we installed the library by modifying the path if not through_escape: # HACK to pretend that we installed test_lib - sys.path.append(os.path.realpath( - os.path.join(os.path.dirname(__file__), '..', '..', 'metaflow', 'plugins', - 'env_escape', 'configurations', 'test_lib_impl'))) + sys.path.append( + os.path.realpath( + os.path.join( + os.path.dirname(__file__), + "..", + "..", + "metaflow", + "plugins", + "env_escape", + "configurations", + "test_lib_impl", + ) + ) + ) print("Path is %s" % str(sys.path)) import test_lib as test + if through_escape: # This tests package aliasing from test_lib.package import TestClass3 @@ -79,7 +92,6 @@ def run_test(through_escape=False): class EscapeTest(FlowSpec): - @conda(disabled=True) @step def start(self): @@ -111,5 +123,6 @@ def join(self, inputs): def end(self): pass -if __name__ == '__main__': + +if __name__ == "__main__": EscapeTest() diff --git a/test/unit/test_k8s_job_name_sanitizer.py b/test/unit/test_k8s_job_name_sanitizer.py index e019a47bea3..0b82893d765 100644 --- a/test/unit/test_k8s_job_name_sanitizer.py +++ b/test/unit/test_k8s_job_name_sanitizer.py @@ -1,26 +1,33 @@ import re from metaflow.plugins.aws.eks.kubernetes import generate_rfc1123_name -rfc1123 = re.compile(r'^[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?$') +rfc1123 = re.compile(r"^[a-zA-Z0-9]([a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])?$") + def test_job_name_santitizer(): # Basic name - assert rfc1123.match(generate_rfc1123_name('HelloFlow', '1', 'end', '321', '1')) + assert rfc1123.match(generate_rfc1123_name("HelloFlow", "1", "end", "321", "1")) # Step name ends with _ - assert rfc1123.match(generate_rfc1123_name('HelloFlow', '1', '_end', '321', '1')) + assert rfc1123.match(generate_rfc1123_name("HelloFlow", "1", "_end", "321", "1")) # Step name starts and ends with _ - assert rfc1123.match(generate_rfc1123_name('HelloFlow', '1', '_end_', '321', '1')) + assert rfc1123.match(generate_rfc1123_name("HelloFlow", "1", "_end_", "321", "1")) # Flow name ends with _ - assert rfc1123.match(generate_rfc1123_name('HelloFlow_', '1', 'end', '321', '1')) + assert rfc1123.match(generate_rfc1123_name("HelloFlow_", "1", "end", "321", "1")) # Same flow name, different case must produce different job names - assert generate_rfc1123_name('Helloflow', '1', 'end', '321', '1') != generate_rfc1123_name('HelloFlow', '1', 'end', '321', '1') + assert generate_rfc1123_name( + "Helloflow", "1", "end", "321", "1" + ) != generate_rfc1123_name("HelloFlow", "1", "end", "321", "1") # Very long step name should be fine - assert rfc1123.match(generate_rfc1123_name('Helloflow', '1', 'end'*50, '321', '1')) + assert rfc1123.match( + generate_rfc1123_name("Helloflow", "1", "end" * 50, "321", "1") + ) # Very long run id should be fine too - assert rfc1123.match(generate_rfc1123_name('Helloflow', '1'*100, 'end', '321', '1')) \ No newline at end of file + assert rfc1123.match( + generate_rfc1123_name("Helloflow", "1" * 100, "end", "321", "1") + ) diff --git a/test/unit/test_k8s_label_sanitizer.py b/test/unit/test_k8s_label_sanitizer.py index 6fcfbd5553f..c053996af16 100644 --- a/test/unit/test_k8s_label_sanitizer.py +++ b/test/unit/test_k8s_label_sanitizer.py @@ -3,26 +3,26 @@ def test_label_value_santitizer(): - assert LABEL_VALUE_REGEX.match(sanitize_label_value('HelloFlow')) + assert LABEL_VALUE_REGEX.match(sanitize_label_value("HelloFlow")) # The value is too long - assert LABEL_VALUE_REGEX.match(sanitize_label_value('a' * 1000)) + assert LABEL_VALUE_REGEX.match(sanitize_label_value("a" * 1000)) # Different long values should still not be equal after sanitization - assert sanitize_label_value('a' * 1000) != sanitize_label_value('a' * 1001) - assert sanitize_label_value('-' * 1000) != sanitize_label_value('-' * 1001) + assert sanitize_label_value("a" * 1000) != sanitize_label_value("a" * 1001) + assert sanitize_label_value("-" * 1000) != sanitize_label_value("-" * 1001) # Different long values should still not be equal after sanitization - assert sanitize_label_value('alice!') != sanitize_label_value('alice?') + assert sanitize_label_value("alice!") != sanitize_label_value("alice?") # ends with dash - assert LABEL_VALUE_REGEX.match(sanitize_label_value('HelloFlow-')) + assert LABEL_VALUE_REGEX.match(sanitize_label_value("HelloFlow-")) # non-ascii - assert LABEL_VALUE_REGEX.match(sanitize_label_value('метафлоу')) + assert LABEL_VALUE_REGEX.match(sanitize_label_value("метафлоу")) # different only in case - assert sanitize_label_value('Alice') != sanitize_label_value('alice') + assert sanitize_label_value("Alice") != sanitize_label_value("alice") # spaces - assert LABEL_VALUE_REGEX.match(sanitize_label_value('Meta flow')) \ No newline at end of file + assert LABEL_VALUE_REGEX.match(sanitize_label_value("Meta flow")) From db20d4e5282918e00f0ae59d32c6a71650d67962 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 29 Oct 2021 10:20:30 +0200 Subject: [PATCH 084/176] Fix Python version logic that would fail with Py 4 (#792) Fix error discovered by https://github.com/asottile/flake8-2020 % `flake8 . --count --select=E9,F63,F7,F82,Y --show-source --statistics` ``` ./metaflow/metaflow/util.py:53:36: YTT204 `sys.version_info.minor` compared to integer (python4), compare `sys.version_info` to tuple if sys.version_info.major >= 3 and sys.version_info.minor >= 7: ^ ``` --- metaflow/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/util.py b/metaflow/util.py index e57f646391b..14483206bd9 100644 --- a/metaflow/util.py +++ b/metaflow/util.py @@ -50,7 +50,7 @@ def unquote_bytes(x): from shlex import quote as _quote -if sys.version_info.major >= 3 and sys.version_info.minor >= 7: +if sys.version_info >= (3, 7): from collections import namedtuple namedtuple_with_defaults = namedtuple From d68ee36751c0b6ac64622c7991664f94e7f6a50a Mon Sep 17 00:00:00 2001 From: Romain Date: Fri, 29 Oct 2021 02:32:36 -0700 Subject: [PATCH 085/176] Fix improper version display on SFN with extensions (#794) --- metaflow/metaflow_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metaflow/metaflow_version.py b/metaflow/metaflow_version.py index d83193e50cd..fde8c8695e1 100644 --- a/metaflow/metaflow_version.py +++ b/metaflow/metaflow_version.py @@ -138,7 +138,7 @@ def get_version(pep440=False): version = metaflow.__version__ version_addl = metaflow.__version_addl__ if version is None: # not a proper python package - version = read_info_version() - if version and version_addl: + return read_info_version() + if version_addl: return "+".join([version, version_addl]) return version From 53789f5a1971cc4b1782eac1885c6db36407ddbf Mon Sep 17 00:00:00 2001 From: Romain Date: Fri, 29 Oct 2021 02:32:50 -0700 Subject: [PATCH 086/176] Patch release for Metaflow (#795) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7ed8f1c21c7..fdf93968226 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = "2.4.2" +version = "2.4.3" setup( name="metaflow", From 2a46edc6796de6252eca4fd3ef9eab1438ea7b17 Mon Sep 17 00:00:00 2001 From: Savin Date: Mon, 1 Nov 2021 10:59:33 -0700 Subject: [PATCH 087/176] Add head_ref to branch name for black branch (#797) * Add head_ref to branch name for black branch * test * remove test --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 80b759816cc..8bfabf85c80 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,7 +29,7 @@ jobs: This pull request uses the [psf/black](https://github.com/psf/black) formatter to fix these issues. `black .` base: ${{ github.head_ref }} - branch: actions/black + branch: actions/black-${{ github.head_ref }} test: name: tests / ${{ matrix.lang }} tests on ${{ matrix.os }} runs-on: ${{ matrix.os }} From fe60f52792926247acfb5b0cad5d106090f71700 Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Mon, 1 Nov 2021 10:59:55 -0700 Subject: [PATCH 088/176] add pre-commit (#793) --- .github/workflows/test.yml | 6 ++++++ .pre-commit-config.yaml | 11 +++++++++++ 2 files changed, 17 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8bfabf85c80..69618c007a5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,12 @@ on: branches: - master jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: pre-commit/action@v2.0.3 formatter: name: runner / black runs-on: ubuntu-latest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000..c611e7a09fb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.4.0 + hooks: + - id: check-yaml + - id: check-json + - repo: https://github.com/ambv/black + rev: 21.9b0 + hooks: + - id: black + language_version: python3 From bf59093332f4cc738e0756e37d042bb5a48398cc Mon Sep 17 00:00:00 2001 From: Savin Date: Mon, 1 Nov 2021 11:18:22 -0700 Subject: [PATCH 089/176] Remove auto-formatter for black (#800) * Add head_ref to branch name for black branch * test * remove test * remove autoformatter --- .github/workflows/test.yml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 69618c007a5..16884420296 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,29 +13,6 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - uses: pre-commit/action@v2.0.3 - formatter: - name: runner / black - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Check files using the black formatter - uses: rickstaa/action-black@v1 - id: action_black - with: - black_args: "." - - name: Create Pull Request - if: steps.action_black.outputs.is_formatted == 'true' - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - title: "Format Python code with psf/black push" - commit-message: ":art: Format Python code with psf/black" - body: | - There appear to be some python formatting errors in #${{ github.event.number }} - ${{ github.sha }}. - This pull request uses the [psf/black](https://github.com/psf/black) formatter to fix these issues. - `black .` - base: ${{ github.head_ref }} - branch: actions/black-${{ github.head_ref }} test: name: tests / ${{ matrix.lang }} tests on ${{ matrix.os }} runs-on: ${{ matrix.os }} From dc74dde2b6d9ebca9cd5a3d3dfa6d39eb8eaee39 Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Mon, 1 Nov 2021 15:18:25 -0700 Subject: [PATCH 090/176] fix reading METAFLOW_SERVICE_RETRY_COUNT from env (#802) --- metaflow/metaflow_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py index 05325aa092c..de194903f8e 100644 --- a/metaflow/metaflow_config.py +++ b/metaflow/metaflow_config.py @@ -109,7 +109,7 @@ def from_conf(name, default=None): # Metadata configuration ### METADATA_SERVICE_URL = from_conf("METAFLOW_SERVICE_URL") -METADATA_SERVICE_NUM_RETRIES = from_conf("METAFLOW_SERVICE_RETRY_COUNT", 5) +METADATA_SERVICE_NUM_RETRIES = int(from_conf("METAFLOW_SERVICE_RETRY_COUNT", 5)) METADATA_SERVICE_AUTH_KEY = from_conf("METAFLOW_SERVICE_AUTH_KEY") METADATA_SERVICE_HEADERS = json.loads(from_conf("METAFLOW_SERVICE_HEADERS", "{}")) if METADATA_SERVICE_AUTH_KEY is not None: From d8646b97469a63c73c61edd889c29f04b7c53fe4 Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Tue, 2 Nov 2021 07:29:39 -0700 Subject: [PATCH 091/176] make catch/retry test not wait to retry (otherwise takes too long on batch/k8s where default is 2 min) (#803) --- test/core/tests/catch_retry.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/core/tests/catch_retry.py b/test/core/tests/catch_retry.py index e9eeb2ca24e..9a672e7374b 100644 --- a/test/core/tests/catch_retry.py +++ b/test/core/tests/catch_retry.py @@ -5,7 +5,7 @@ class CatchRetryTest(MetaflowTest): PRIORITY = 2 - @tag("retry(times=3)") + @tag("retry(times=3,minutes_between_retries=0)") @steps(0, ["start"]) def step_start(self): import os, sys @@ -18,7 +18,7 @@ def step_start(self): raise TestRetry() # foreach splits don't support @catch but @retry should work - @tag("retry(times=2)") + @tag("retry(times=2,minutes_between_retries=0)") @steps(0, ["foreach-split"]) def step_split(self): import os @@ -28,7 +28,7 @@ def step_split(self): else: raise TestRetry() - @tag("retry(times=2)") + @tag("retry(times=2,minutes_between_retries=0)") @steps(0, ["join"]) def step_join(self): import os @@ -51,7 +51,7 @@ def step_end(self): raise ExternalCommandFailed("catch me!") @tag('catch(var="ex", print_exception=False)') - @tag("retry(times=2)") + @tag("retry(times=2,minutes_between_retries=0)") @steps(1, ["all"]) def step_all(self): # Die a soft death; this should retry and then catch in the end @@ -75,7 +75,7 @@ def check_results(self, flow, checker): for task in checker.artifact_dict("start", "invisible").values(): if task: raise Exception( - "'invisible' should not be visible " "in 'start'" + "'invisible' should not be visible in 'start'" ) except KeyError: pass From 29c09a9a1539628eae4c51073fa100858a8bd42c Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Wed, 3 Nov 2021 11:16:16 +0200 Subject: [PATCH 092/176] Multinode UBF support for Metaflow core --- metaflow/flowspec.py | 23 +++++++++++++++++++++++ metaflow/graph.py | 8 +++++++- metaflow/plugins/__init__.py | 2 ++ metaflow/plugins/multinode_decorator.py | 13 +++++++++++++ metaflow/runtime.py | 6 +++++- metaflow/task.py | 8 -------- 6 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 metaflow/plugins/multinode_decorator.py diff --git a/metaflow/flowspec.py b/metaflow/flowspec.py index 775cfee424a..b102bd1a649 100644 --- a/metaflow/flowspec.py +++ b/metaflow/flowspec.py @@ -32,6 +32,18 @@ def __init__(self, msg): super(InvalidNextException, self).__init__(msg, line_no) +class Multinode(UnboundedForeachInput): + """ + Unbounded-for-each placeholder for supporting multinode steps. + """ + + def __init__(self): + pass + + def __getitem__(self, item): + return item + + class FlowSpec(object): """ Main class from which all Flows should inherit. @@ -132,6 +144,10 @@ def __iter__(self): return iter(self._steps) def __getattr__(self, name): + if name == "_multinode_ubf_iter": + # Special multinode UBF iterator object + return Multinode() + if self._datastore and name in self._datastore: # load the attribute from the datastore... x = self._datastore[name] @@ -465,6 +481,13 @@ def next(self, *dsts, **kwargs): raise InvalidNextException(msg) funcs.append(name) + # if only one dst, check if it is a multinode one. If it is, then create a synthetic + # UBF construct. + if len(dsts) == 1: + dst = dsts[0] + if any(deco.name == "multinode" for deco in dst.__func__.decorators): + foreach = "_multinode_ubf_iter" + # check: foreach and condition are mutually exclusive if not (foreach is None or condition is None): msg = ( diff --git a/metaflow/graph.py b/metaflow/graph.py index ecf83a9ddd5..1b036213dbd 100644 --- a/metaflow/graph.py +++ b/metaflow/graph.py @@ -63,6 +63,7 @@ def __init__(self, func_ast, decos, doc): self.in_funcs = set() self.split_parents = [] self.matching_join = None + self.is_multinode_step = any(deco.name == "multinode" for deco in decos) # these attributes are populated by _postprocess self.is_inside_foreach = False @@ -192,7 +193,12 @@ def _postprocess(self): def _traverse_graph(self): def traverse(node, seen, split_parents): - + # check multinode: if the next step is a multinode-step, then + # this step is of type "foreach" + if len(node.out_funcs) == 1: + child = self[node.out_funcs[0]] + if child.is_multinode_step: + node.type = "foreach" if node.type in ("split-or", "split-and", "foreach"): node.split_parents = split_parents split_parents = split_parents + [node.name] diff --git a/metaflow/plugins/__init__.py b/metaflow/plugins/__init__.py index 086a0c57dae..3d1ceddcdaa 100644 --- a/metaflow/plugins/__init__.py +++ b/metaflow/plugins/__init__.py @@ -131,6 +131,7 @@ def _merge_lists(base, overrides, attr): InternalTestUnboundedForeachDecorator, InternalTestUnboundedForeachInput, ) +from .multinode_decorator import MultinodeDecorator from .conda.conda_step_decorator import CondaStepDecorator STEP_DECORATORS = _merge_lists( @@ -145,6 +146,7 @@ def _merge_lists(base, overrides, attr): StepFunctionsInternalDecorator, CondaStepDecorator, InternalTestUnboundedForeachDecorator, + MultinodeDecorator, ], _ext_plugins.STEP_DECORATORS, "name", diff --git a/metaflow/plugins/multinode_decorator.py b/metaflow/plugins/multinode_decorator.py new file mode 100644 index 00000000000..ca0ce7a054f --- /dev/null +++ b/metaflow/plugins/multinode_decorator.py @@ -0,0 +1,13 @@ +from metaflow.decorators import StepDecorator +from metaflow.unbounded_foreach import UnboundedForeachInput + + +class MultinodeDecorator(StepDecorator): + name = "multinode" + defaults = { + "nodes": 2, + } + + def __init__(self, attributes=None, statically_defined=False): + self.nodes = attributes["nodes"] + super(MultinodeDecorator, self).__init__(attributes, statically_defined) diff --git a/metaflow/runtime.py b/metaflow/runtime.py index a2cdc0e121e..0107a669acb 100644 --- a/metaflow/runtime.py +++ b/metaflow/runtime.py @@ -364,6 +364,11 @@ def _queue_task_join(self, task, next_steps): top = foreach_stack[-1] bottom = list(foreach_stack[:-1]) s = tuple(bottom + [top._replace(index=None)]) + + # UBF control can also be the first task of the list. Then + # it will have index=0 instead of index=None. + if task.results.get("_control_task_is_mapper_zero", False): + s = tuple(bottom + [top._replace(index=0)]) control_path = self._finished.get((task.step, s)) if control_path: # Control task was successful. @@ -374,7 +379,6 @@ def _queue_task_join(self, task, next_steps): for i in range(num_splits): s = tuple(bottom + [top._replace(index=i)]) required_tasks.append(self._finished.get((task.step, s))) - required_tasks.append(control_path) if all(required_tasks): # all tasks to be joined are ready. Schedule the next join step. diff --git a/metaflow/task.py b/metaflow/task.py index 9a97c596d10..dcff980e3d6 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -345,14 +345,6 @@ def run_step( output.init_task() if input_paths: - control_paths = [ - path - for path in input_paths - if path.split("/")[-1].startswith("control-") - ] - if control_paths: - [control_path] = control_paths - input_paths.remove(control_path) # 2. initialize input datastores inputs = self._init_data(run_id, join_type, input_paths) From 7e3611fdcae7e6b3e1e1757f73eeee4fe96688a5 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Wed, 3 Nov 2021 12:00:16 +0200 Subject: [PATCH 093/176] batch plugin multinode support --- metaflow/plugins/aws/batch/batch.py | 40 ++++++++- metaflow/plugins/aws/batch/batch_cli.py | 8 +- metaflow/plugins/aws/batch/batch_client.py | 86 +++++++++++++++++- metaflow/plugins/aws/batch/batch_decorator.py | 87 ++++++++++++++++++- 4 files changed, 213 insertions(+), 8 deletions(-) diff --git a/metaflow/plugins/aws/batch/batch.py b/metaflow/plugins/aws/batch/batch.py index 833e97801a7..a8e548f2016 100644 --- a/metaflow/plugins/aws/batch/batch.py +++ b/metaflow/plugins/aws/batch/batch.py @@ -1,4 +1,5 @@ import atexit +import copy import json import os import select @@ -174,6 +175,7 @@ def create_job( env={}, attrs={}, host_volumes=None, + nodes=1, ): job_name = self._job_name( attrs.get("metaflow.user"), @@ -204,6 +206,7 @@ def create_job( max_swap, swappiness, host_volumes=host_volumes, + nodes=nodes, ) .cpu(cpu) .gpu(gpu) @@ -212,6 +215,7 @@ def create_job( .max_swap(max_swap) .swappiness(swappiness) .timeout_in_secs(run_time_limit) + .task_id(attrs.get("metaflow.task_id")) .environment_variable("AWS_DEFAULT_REGION", self._client.region()) .environment_variable("METAFLOW_CODE_SHA", code_package_sha) .environment_variable("METAFLOW_CODE_URL", code_package_url) @@ -271,6 +275,7 @@ def launch_job( max_swap=None, swappiness=None, host_volumes=None, + nodes=1, env={}, attrs={}, ): @@ -302,11 +307,13 @@ def launch_job( env=env, attrs=attrs, host_volumes=host_volumes, + nodes=nodes, ) + self.nodes = nodes self.job = job.execute() def wait(self, stdout_location, stderr_location, echo=None): - def wait_for_launch(job): + def wait_for_launch(job, child_jobs): status = job.status echo( "Task is starting (status %s)..." % status, @@ -316,9 +323,15 @@ def wait_for_launch(job): t = time.time() while True: if status != job.status or (time.time() - t) > 30: + if not child_jobs: + child_statuses = "" + else: + child_statuses = " (child nodes: [{}])".format( + ", ".join([child_job.status for child_job in child_jobs]) + ) status = job.status echo( - "Task is starting (status %s)..." % status, + "Task is starting (status %s)... %s" % (status, child_statuses), "stderr", batch_id=job.id, ) @@ -348,16 +361,37 @@ def _print_available(tail, stream, should_persist=False): stdout_tail = S3Tail(stdout_location) stderr_tail = S3Tail(stderr_location) + child_jobs = [] + if self.nodes > 1: + for node in range(1, self.nodes): + child_job = copy.copy(self.job) + child_job._id = child_job._id + "#{}".format(node) + child_jobs.append(child_job) + # 1) Loop until the job has started - wait_for_launch(self.job) + wait_for_launch(self.job, child_jobs) # 2) Loop until the job has finished start_time = time.time() is_running = True next_log_update = start_time log_update_delay = 1 + next_child_job_update = start_time + child_job_update_delay = 20 while is_running: + if child_jobs and not all(child_job.is_running for child_job in child_jobs): + if time.time() > next_child_job_update: + next_child_job_update = time.time() + child_job_update_delay + print( + "Child job status: {}".format( + [ + "{}:{}".format(child_job.id, child_job.status) + for child_job in child_jobs + ] + ) + ) + if time.time() > next_log_update: _print_available(stdout_tail, "stdout") _print_available(stderr_tail, "stderr") diff --git a/metaflow/plugins/aws/batch/batch_cli.py b/metaflow/plugins/aws/batch/batch_cli.py index a838472e2a2..04db8f17784 100644 --- a/metaflow/plugins/aws/batch/batch_cli.py +++ b/metaflow/plugins/aws/batch/batch_cli.py @@ -141,8 +141,9 @@ def kill(ctx, run_id, user, my_runs): @click.option("--max-swap", help="Max Swap requirement for AWS Batch.") @click.option("--swappiness", help="Swappiness requirement for AWS Batch.") # TODO: Maybe remove it altogether since it's not used here -@click.option("--ubf-context", default=None, type=click.Choice([None])) +@click.option("--ubf-context", default=None, type=click.Choice([None, "ubf_control"])) @click.option("--host-volumes", multiple=True) +@click.option("--nodes", type=int) @click.pass_context def step( ctx, @@ -162,6 +163,7 @@ def step( max_swap=None, swappiness=None, host_volumes=None, + nodes=None, **kwargs ): def echo(msg, stream="stderr", batch_id=None): @@ -190,6 +192,9 @@ def echo(msg, stream="stderr", batch_id=None): kwargs["input_paths"] = "".join("${%s}" % s for s in split_vars.keys()) step_args = " ".join(util.dict_to_cli_options(kwargs)) + if nodes > 1: + # For multinode, we need to add a placeholder that can be mutated by the caller + step_args += " [multinode-args]" step_cli = u"{entrypoint} {top_args} step {step} {step_args}".format( entrypoint=entrypoint, top_args=top_args, @@ -282,6 +287,7 @@ def _sync_metadata(): env=env, attrs=attrs, host_volumes=host_volumes, + nodes=nodes, ) except Exception as e: traceback.print_exc() diff --git a/metaflow/plugins/aws/batch/batch_client.py b/metaflow/plugins/aws/batch/batch_client.py index dc2faf06c23..523bc4d4943 100644 --- a/metaflow/plugins/aws/batch/batch_client.py +++ b/metaflow/plugins/aws/batch/batch_client.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- from collections import defaultdict, deque +import copy import random import select import sys @@ -94,6 +95,50 @@ def execute(self): self._max_swap, self._swappiness, ) + + # Multinode + if getattr(self, "_nodes", 0) > 1: + num_nodes = self._nodes + main_task_override = copy.deepcopy(self.payload["containerOverrides"]) + # TERRIBLE HACK JUST TO TRY THIS WORKS + + # main + commands = self.payload["containerOverrides"]["command"][-1] + # add split-index as this worker is also an ubf_task + commands = commands.replace("[multinode-args]", "--split-index 0") + main_task_override["command"][-1] = commands + + # secondary tasks + secondary_task_container_override = copy.deepcopy( + self.payload["containerOverrides"] + ) + secondary_commands = self.payload["containerOverrides"]["command"][-1] + # other tasks do not have control- prefix, and have the split id appended to the task -id + secondary_commands = secondary_commands.replace( + self._task_id, + self._task_id.replace("control-", "") + + "-node-$AWS_BATCH_JOB_NODE_INDEX", + ) + secondary_commands = secondary_commands.replace( + "ubf_control", + "ubf_task", + ) + secondary_commands = secondary_commands.replace( + "[multinode-args]", "--split-index $AWS_BATCH_JOB_NODE_INDEX" + ) + + secondary_task_container_override["command"][-1] = secondary_commands + self.payload["nodeOverrides"] = { + "nodePropertyOverrides": [ + {"targetNodes": "0:0", "containerOverrides": main_task_override}, + { + "targetNodes": "1:{}".format(num_nodes - 1), + "containerOverrides": secondary_task_container_override, + }, + ], + } + del self.payload["containerOverrides"] + response = self._client.submit_job(**self.payload) job = RunningJob(response["jobId"], self._client) return job.update() @@ -108,6 +153,7 @@ def _register_job_definition( max_swap, swappiness, host_volumes, + nodes, ): # identify platform from any compute environment associated with the # queue @@ -146,6 +192,8 @@ def _register_job_definition( } if platform == "FARGATE" or platform == "FARGATE_SPOT": + if nodes > 1: + raise BatchJobException("Fargate does not support multinode jobs.") if execution_role is None: raise BatchJobException( "No AWS Fargate task execution IAM role found. Please see " @@ -215,6 +263,25 @@ def _register_job_definition( {"sourceVolume": name, "containerPath": host_path} ) + nodes = int(nodes) + if nodes > 1: + job_definition["type"] = "multinode" + job_definition["nodeProperties"] = {"numNodes": nodes, "mainNode": 0} + job_definition["nodeProperties"]["nodeRangeProperties"] = [ + { + "targetNodes": "0:0", # The properties are same for main node and others, + # but as we use nodeOverrides later for main and others + # differently, also the job definition must match those patterns + "container": job_definition["containerProperties"], + }, + { + "targetNodes": "1:{}".format(nodes - 1), + "container": job_definition["containerProperties"], + }, + ] + del job_definition["containerProperties"] # not used for multi-node + self._nodes = nodes + # check if job definition already exists def_name = ( "metaflow_%s" @@ -252,6 +319,7 @@ def job_def( max_swap, swappiness, host_volumes, + nodes, ): self.payload["jobDefinition"] = self._register_job_definition( image, @@ -262,6 +330,7 @@ def job_def( max_swap, swappiness, host_volumes, + nodes, ) return self @@ -277,6 +346,10 @@ def image(self, image): self._image = image return self + def task_id(self, task_id): + self._task_id = task_id + return self + def iam_role(self, iam_role): self._iam_role = iam_role return self @@ -505,13 +578,22 @@ def is_crashed(self): @property def reason(self): - return self.info["container"].get("reason") + if "container" in self.info: + # single-node job + return self.info["container"].get("reason") + else: + # multinode + return self.info["statusReason"] @property def status_code(self): if not self.is_done: self.update() - return self.info["container"].get("exitCode") + if "container" in self.info: + return self.info["container"].get("exitCode") + else: + # multinode + return self.info["attempts"][-1]["container"].get("exitCode") def kill(self): if not self.is_done: diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index 3e69bd50e9d..018cc331b3e 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -2,9 +2,10 @@ import sys import platform import requests +import time from metaflow import util -from metaflow import R +from metaflow import R, current from metaflow.decorators import StepDecorator from metaflow.plugins import ResourcesDecorator @@ -20,6 +21,7 @@ DATASTORE_LOCAL_DIR, ) from metaflow.sidecar import SidecarSubProcess +from metaflow.unbounded_foreach import UBF_CONTROL from .batch import BatchException from ..aws_utils import get_docker_registry @@ -96,6 +98,7 @@ def my_step(self): "max_swap": None, "swappiness": None, "host_volumes": None, + "nodes": 1, } package_url = None package_sha = None @@ -150,7 +153,10 @@ def step_init(self, flow, graph, step, decos, environment, flow_datastore, logge my_val = self.attributes.get(k) if not (my_val is None and v is None): self.attributes[k] = str(max(int(my_val or 0), int(v or 0))) - + elif ( + deco.__class__.__name__ == "MultinodeDecorator" + ): # avoid circular dependency + self.attributes["nodes"] = deco.nodes # Set run time limit for the AWS Batch job. self.run_time_limit = get_run_time_limit_for_task(decos) if self.run_time_limit < 60: @@ -249,6 +255,24 @@ def task_pre_step( self._save_logs_sidecar = SidecarSubProcess("save_logs_periodically") + nodes = self.attributes["nodes"] + + if nodes > 1 and ubf_context == UBF_CONTROL: + # UBF handling for multinode case + control_task_id = current.task_id + top_task_id = control_task_id.replace("control-", "") # chop "-0" + mapper_task_ids = [control_task_id] + [ + "%s-node-%d" % (top_task_id, node_idx) for node_idx in range(1, nodes) + ] + flow._control_mapper_tasks = [ + "%s/%s/%s" % (run_id, step_name, mapper_task_id) + for mapper_task_id in mapper_task_ids + ] + flow._control_task_is_mapper_zero = True + + if nodes > 1: + _set_multinode_environment() + def task_post_step( self, step_name, flow, graph, retry_count, max_user_code_retries ): @@ -292,9 +316,68 @@ def task_finished( # Best effort kill pass + if is_task_ok and getattr(flow, "_control_mapper_tasks", []): + self._wait_for_mapper_tasks(flow, step_name) + + def _wait_for_mapper_tasks(self, flow, step_name): + """ + When lauching multinode task with UBF, need to wait for the secondary + tasks to finish cleanly and produce their output before exiting the + main task. Otherwise main task finishing will cause secondary nodes + to terminate immediately, and possibly prematurely. + """ + from metaflow import Step # avoid circular dependency + + t = time.time() + TIMEOUT = 600 + print("Waiting for batch secondary tasks to finish") + while t + TIMEOUT > time.time(): + time.sleep(2) + try: + step_path = "%s/%s/%s" % (flow.name, current.run_id, step_name) + tasks = [task for task in Step(step_path)] + if len(tasks) == len(flow._control_mapper_tasks) - 1: + if all( + task.finished_at is not None for task in tasks + ): # for some reason task.finished fails + return True + else: + print( + "Not sufficient number of tasks:", + len(tasks), + len(flow._control_mapper_tasks), + ) + except Exception as e: + print(e) + pass + raise Exception( + "Batch secondary workers did not finish in %s seconds" % TIMEOUT + ) + @classmethod def _save_package_once(cls, flow_datastore, package): if cls.package_url is None: cls.package_url, cls.package_sha = flow_datastore.save_data( [package.blob], len_hint=1 )[0] + + +def _set_multinode_environment(): + # setup the multinode environment + import socket + + if "AWS_BATCH_JOB_MAIN_NODE_PRIVATE_IPV4_ADDRESS" not in os.environ: + # we are the main node + local_ips = socket.gethostbyname_ex(socket.gethostname())[-1] + assert local_ips, "Could not find local ip address" + os.environ["MF_MULTINODE_MAIN_IP"] = local_ips[0] + else: + os.environ["MF_MULTINODE_MAIN_IP"] = os.environ[ + "AWS_BATCH_JOB_MAIN_NODE_PRIVATE_IPV4_ADDRESS" + ] + os.environ["MF_MULTINODE_NUM_NODES"] = os.environ["AWS_BATCH_JOB_NUM_NODES"] + os.environ["MF_MULTINODE_NODE_INDEX"] = os.environ["AWS_BATCH_JOB_NODE_INDEX"] + print( + "Multinode environment:", + {k: v for k, v in os.environ.items() if k.startswith("MF_MULTINODE")}, + ) From ac7d7df6ce4b89523c8ffc36cdc6373099bd5cd0 Mon Sep 17 00:00:00 2001 From: Romain Date: Wed, 3 Nov 2021 18:28:27 -0700 Subject: [PATCH 094/176] Addressed an issue with class members (#804) * Addressed an issue with class members Class members that were not parameters caused an error (regression in 2.4.0). This addresses this issue. This patch also clarifies and makes explicit that modifying class variables is not safe; an error message is now raised. * Avoid persisting class variables multiple times. This treats class variables more like parameters (constants) and persists them once in the renamed persist_constants (used to be persist_parameters). * add constants test * Make test pass (hopefully) * add CLASS_VARS to tests Co-authored-by: Ville Tuulos --- docs/lifecycle.dot | 4 +-- metaflow/cli.py | 10 +++--- metaflow/datastore/task_datastore.py | 2 +- metaflow/flowspec.py | 39 ++++++++++++++++++++ metaflow/includefile.py | 2 +- metaflow/parameters.py | 23 ------------ metaflow/runtime.py | 4 +-- metaflow/task.py | 27 ++++++++++++-- test/core/metaflow_test/__init__.py | 1 + test/core/metaflow_test/formatter.py | 3 ++ test/core/tests/constants.py | 54 ++++++++++++++++++++++++++++ 11 files changed, 132 insertions(+), 37 deletions(-) create mode 100644 test/core/tests/constants.py diff --git a/docs/lifecycle.dot b/docs/lifecycle.dot index d6cac9a57b5..c499dce21f8 100644 --- a/docs/lifecycle.dot +++ b/docs/lifecycle.dot @@ -53,7 +53,7 @@ digraph Metaflow { command_run [label="{command|run}", fillcolor=tan] new_run_id [label="{metadata|new_run_id}", fillcolor=lightgoldenrod1] runtime_init [label="{decorator|runtime_init}", fillcolor=lightblue2] - local_params [label="{runtime|persist_parameters}", fillcolor=lightpink2] + local_params [label="{runtime|persist_constants}", fillcolor=lightpink2] start_run_heartbeat [label="{metadata|start_run_heartbeat}", fillcolor=lightgoldenrod1] schedule_local_task [shape="circle", label="Schedule\nTask", width=1, fillcolor=grey78] runtime_finished [label="{decorator|runtime_finished}", fillcolor=lightblue2] @@ -99,7 +99,7 @@ digraph Metaflow { stepfunctions_run [label="{AWS Step Functions|start_execution}", fillcolor=lightpink2] stepfunctions_bootstrap_batch [shape="circle", label="Bootstrap\nAWS Batch", width=1, fillcolor=grey78] stepfunctions_init [label="{command|init}" fillcolor=tan] - stepfunctions_params [label="{runtime|persist_parameters}", fillcolor=lightpink2] + stepfunctions_params [label="{runtime|persist_constants}", fillcolor=lightpink2] stepfunctions_task [shape="circle", label="Execute\nTask", width=1, fillcolor=grey78] } diff --git a/metaflow/cli.py b/metaflow/cli.py index cac4c75737c..356c7ecc5b9 100644 --- a/metaflow/cli.py +++ b/metaflow/cli.py @@ -592,8 +592,8 @@ def init(obj, run_id=None, task_id=None, tags=None, **kwargs): obj.monitor, run_id=run_id, ) - parameters.set_parameters(obj.flow, kwargs) - runtime.persist_parameters(task_id=task_id) + obj.flow._set_constants(kwargs) + runtime.persist_constants(task_id=task_id) def common_run_options(func): @@ -711,7 +711,7 @@ def resume( max_num_splits=max_num_splits, max_log_size=max_log_size * 1024 * 1024, ) - runtime.persist_parameters() + runtime.persist_constants() runtime.execute() write_run_id(run_id_file, runtime.run_id) @@ -766,8 +766,8 @@ def run( write_latest_run_id(obj, runtime.run_id) write_run_id(run_id_file, runtime.run_id) - parameters.set_parameters(obj.flow, kwargs) - runtime.persist_parameters() + obj.flow._set_constants(kwargs) + runtime.persist_constants() runtime.execute() diff --git a/metaflow/datastore/task_datastore.py b/metaflow/datastore/task_datastore.py index f04b129ebf2..418b253fa7a 100644 --- a/metaflow/datastore/task_datastore.py +++ b/metaflow/datastore/task_datastore.py @@ -643,7 +643,7 @@ def persist(self, flow): for var in dir(flow): if var.startswith("__") or var in flow._EPHEMERAL: continue - # Skip over properties of the class (Parameters) + # Skip over properties of the class (Parameters or class variables) if hasattr(flow.__class__, var) and isinstance( getattr(flow.__class__, var), property ): diff --git a/metaflow/flowspec.py b/metaflow/flowspec.py index 775cfee424a..4b122550267 100644 --- a/metaflow/flowspec.py +++ b/metaflow/flowspec.py @@ -3,6 +3,7 @@ import sys import inspect import traceback +from types import FunctionType, MethodType from . import cmd_with_io from .parameters import Parameter @@ -106,6 +107,44 @@ def script_name(self): fname = fname[:-1] return os.path.basename(fname) + def _set_constants(self, kwargs): + # Persist values for parameters and other constants (class level variables) + # only once. This method is called before persist_constants is called to + # persist all values set using setattr + seen = set() + for var, param in self._get_parameters(): + norm = param.name.lower() + if norm in seen: + raise MetaflowException( + "Parameter *%s* is specified twice. " + "Note that parameter names are " + "case-insensitive." % param.name + ) + seen.add(norm) + seen.clear() + self._success = True + + for var, param in self._get_parameters(): + seen.add(var) + val = kwargs[param.name.replace("-", "_").lower()] + # Support for delayed evaluation of parameters. This is used for + # includefile in particular + if callable(val): + val = val() + val = val.split(param.separator) if val and param.separator else val + setattr(self, var, val) + + # Do the same for class variables which will be forced constant as modifications + # to them don't propagate well since we create a new process for each step and + # re-read the flow file + for var in dir(self.__class__): + if var[0] == "_" or var in self._NON_PARAMETERS or var in seen: + continue + val = getattr(self.__class__, var) + if isinstance(val, (MethodType, FunctionType, property, type)): + continue + setattr(self, var, val) + def _get_parameters(self): for var in dir(self): if var[0] == "_" or var in self._NON_PARAMETERS: diff --git a/metaflow/includefile.py b/metaflow/includefile.py index 6b4f7b717bf..ffa5a40bb07 100644 --- a/metaflow/includefile.py +++ b/metaflow/includefile.py @@ -232,7 +232,7 @@ class FilePathClass(click.ParamType): # + If the value is already such a string, nothing happens and it returns that same value # + If the value is a LocalFile, it will persist the local file and return the path # of the persisted file - # - The artifact will be persisted prior to any run (for non-scheduled runs through persist_parameters) + # - The artifact will be persisted prior to any run (for non-scheduled runs through persist_constants) # + This will therefore persist a simple string # - When the parameter is loaded again, the load_parameter in the IncludeFile class will get called # which will download and return the bytes of the persisted file. diff --git a/metaflow/parameters.py b/metaflow/parameters.py index d84e362c3d7..7ff505ad86d 100644 --- a/metaflow/parameters.py +++ b/metaflow/parameters.py @@ -250,26 +250,3 @@ def wrapper(cmd): return cmd return wrapper - - -def set_parameters(flow, kwargs): - seen = set() - for var, param in flow._get_parameters(): - norm = param.name.lower() - if norm in seen: - raise MetaflowException( - "Parameter *%s* is specified twice. " - "Note that parameter names are " - "case-insensitive." % param.name - ) - seen.add(norm) - - flow._success = True - for var, param in flow._get_parameters(): - val = kwargs[param.name.replace("-", "_").lower()] - # Support for delayed evaluation of parameters. This is used for - # includefile in particular - if callable(val): - val = val() - val = val.split(param.separator) if val and param.separator else val - setattr(flow, var, val) diff --git a/metaflow/runtime.py b/metaflow/runtime.py index a2cdc0e121e..fad0746f2af 100644 --- a/metaflow/runtime.py +++ b/metaflow/runtime.py @@ -176,7 +176,7 @@ def _new_task(self, step, input_paths=None, **kwargs): def run_id(self): return self._run_id - def persist_parameters(self, task_id=None): + def persist_constants(self, task_id=None): task = self._new_task("_parameters", task_id=task_id) if not task.is_cloned: task.persist(self._flow) @@ -614,7 +614,7 @@ def __init__( if task_id is None: task_id = str(metadata.new_task_id(run_id, step)) else: - # task_id is preset only by persist_parameters() or control tasks. + # task_id is preset only by persist_constants() or control tasks. if ubf_context == UBF_CONTROL: metadata.register_task_id( run_id, step, task_id, 0, sys_tags=[CONTROL_TASK_TAG] diff --git a/metaflow/task.py b/metaflow/task.py index 9a97c596d10..8925c0cc84b 100644 --- a/metaflow/task.py +++ b/metaflow/task.py @@ -3,6 +3,8 @@ import os import time +from types import MethodType, FunctionType + from .metaflow_config import MAX_ATTEMPTS from .metadata import MetaDatum from .datastore import Inputs, TaskDataStoreSet @@ -53,6 +55,10 @@ def _exec_step_function(self, step_function, input_obj=None): step_function(input_obj) def _init_parameters(self, parameter_ds, passdown=True): + def set_cls_var(_, __): + raise AttributeError("Flow level attributes are not modifiable") + + cls = self.flow.__class__ # overwrite Parameters in the flow object vars = [] for var, param in self.flow._get_parameters(): @@ -60,7 +66,7 @@ def _init_parameters(self, parameter_ds, passdown=True): # note x=x binds the current value of x to the closure def property_setter( _, - cls=self.flow.__class__, + cls=cls, param=param, var=var, parameter_ds=parameter_ds, @@ -69,11 +75,26 @@ def property_setter( setattr(cls, var, property(fget=lambda _, val=v: val)) return v - setattr(self.flow.__class__, var, property(fget=property_setter)) + setattr(cls, var, property(fget=property_setter)) vars.append(var) + + param_only_vars = list(vars) + # make class-level values read-only to be more consistent across steps in a flow + # they are also only persisted once and so we similarly pass them down if + # required + for var in dir(cls): + if var[0] == "_" or var in cls._NON_PARAMETERS or var in vars: + continue + val = getattr(cls, var) + # Exclude methods, properties and other classes + if isinstance(val, (MethodType, FunctionType, property, type)): + continue + setattr(cls, var, property(fget=lambda _, val=val: val, fset=set_cls_var)) + vars.append(var) + if passdown: self.flow._datastore.passdown_partial(parameter_ds, vars) - return vars + return param_only_vars def _init_data(self, run_id, join_type, input_paths): # We prefer to use the parallelized version to initialize datastores diff --git a/test/core/metaflow_test/__init__.py b/test/core/metaflow_test/__init__.py index 02ddacd6465..9f4456c3959 100644 --- a/test/core/metaflow_test/__init__.py +++ b/test/core/metaflow_test/__init__.py @@ -90,6 +90,7 @@ class MetaflowTest(object): PRIORITY = 999999999 PARAMETERS = {} INCLUDE_FILES = {} + CLASS_VARS = {} HEADER = "" def check_results(self, flow, checker): diff --git a/test/core/metaflow_test/formatter.py b/test/core/metaflow_test/formatter.py index 2c56f306d19..b349c9015c7 100644 --- a/test/core/metaflow_test/formatter.py +++ b/test/core/metaflow_test/formatter.py @@ -88,6 +88,9 @@ def _flow_lines(self): yield 0, self.test.HEADER yield 0, "class %s(FlowSpec):" % self.flow_name + for var, val in self.test.CLASS_VARS.items(): + yield 1, '%s = %s' % (var, val) + for var, parameter in self.test.PARAMETERS.items(): kwargs = ["%s=%s" % (k, v) for k, v in parameter.items()] yield 1, '%s = Parameter("%s", %s)' % (var, var, ",".join(kwargs)) diff --git a/test/core/tests/constants.py b/test/core/tests/constants.py new file mode 100644 index 00000000000..59b70598fc2 --- /dev/null +++ b/test/core/tests/constants.py @@ -0,0 +1,54 @@ +from metaflow_test import MetaflowTest, ExpectationFailed, steps + + +class ConstantsTest(MetaflowTest): + """ + Test that an artifact defined in the first step + is available in all steps downstream. + """ + + PRIORITY = 0 + CLASS_VARS = {'str_const': '"this is a constant"', + 'int_const': 123, + 'obj_const': '[]'} + + PARAMETERS = { + "int_param": {"default": 456}, + "str_param": {"default": "'foobar'"}, + } + + @steps(0, ["all"]) + def step_all(self): + # make sure class attributes are available in all steps + # through joins etc + assert_equals('this is a constant', self.str_const) + assert_equals(123, self.int_const) + # obj_const is mutable. Not much that can be done about it + assert_equals([], self.obj_const) + + assert_equals(456, self.int_param) + assert_equals('foobar', self.str_param) + + # make sure class variables are not listed as parameters + from metaflow import current + assert_equals({'int_param', 'str_param'}, + set(current.parameter_names)) + + try: + self.int_param = 5 + except AttributeError: + pass + else: + raise Exception("It shouldn't be possible to modify parameters") + + try: + self.int_const = 122 + except AttributeError: + pass + else: + raise Exception("It shouldn't be possible to modify constants") + + def check_results(self, flow, checker): + for step in flow: + checker.assert_artifact(step.name, 'int_param', 456) + checker.assert_artifact(step.name, 'int_const', 123) From e9158ba9bd6a7550cc74445e2de9c9cb38e98b87 Mon Sep 17 00:00:00 2001 From: Savin Date: Wed, 3 Nov 2021 19:39:55 -0700 Subject: [PATCH 095/176] Pipe logs to $cwd/.logs instead of /logs for @batch & @kubernetes (#807) --- metaflow/plugins/aws/batch/batch.py | 7 ++++--- metaflow/plugins/aws/eks/kubernetes.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/metaflow/plugins/aws/batch/batch.py b/metaflow/plugins/aws/batch/batch.py index 833e97801a7..7e19c422cfb 100644 --- a/metaflow/plugins/aws/batch/batch.py +++ b/metaflow/plugins/aws/batch/batch.py @@ -27,8 +27,8 @@ from .batch_client import BatchClient -# Redirect structured logs to /logs/ -LOGS_DIR = "/logs" +# Redirect structured logs to $PWD/.logs/ +LOGS_DIR = "$PWD/.logs" STDOUT_FILE = "mflog_stdout" STDERR_FILE = "mflog_stderr" STDOUT_PATH = os.path.join(LOGS_DIR, STDOUT_FILE) @@ -71,7 +71,8 @@ def _command(self, environment, code_package_url, step_name, step_cmds, task_spe # the `true` command is to make sure that the generated command # plays well with docker containers which have entrypoint set as # eval $@ - cmd_str = "true && mkdir -p /logs && %s && %s && %s; " % ( + cmd_str = "true && mkdir -p %s && %s && %s && %s; " % ( + LOGS_DIR, mflog_expr, init_expr, step_expr, diff --git a/metaflow/plugins/aws/eks/kubernetes.py b/metaflow/plugins/aws/eks/kubernetes.py index eb61c006fae..c908020e6a1 100644 --- a/metaflow/plugins/aws/eks/kubernetes.py +++ b/metaflow/plugins/aws/eks/kubernetes.py @@ -28,8 +28,8 @@ from .kubernetes_client import KubernetesClient -# Redirect structured logs to /logs/ -LOGS_DIR = "/logs" +# Redirect structured logs to $PWD/.logs/ +LOGS_DIR = "$PWD/.logs" STDOUT_FILE = "mflog_stdout" STDERR_FILE = "mflog_stderr" STDOUT_PATH = os.path.join(LOGS_DIR, STDOUT_FILE) @@ -148,7 +148,8 @@ def _command( # The `true` command is to make sure that the generated command # plays well with docker containers which have entrypoint set as # eval $@ - cmd_str = "true && mkdir -p /logs && %s && %s && %s; " % ( + cmd_str = "true && mkdir -p %s && %s && %s && %s; " % ( + LOGS_DIR, mflog_expr, init_expr, step_expr, From e778f5dd7ef173b8d63f89d4ad6e78869298765f Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Thu, 4 Nov 2021 10:05:23 -0700 Subject: [PATCH 096/176] fix formatting from #804 (#811) --- test/core/metaflow_test/formatter.py | 2 +- test/core/tests/constants.py | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/test/core/metaflow_test/formatter.py b/test/core/metaflow_test/formatter.py index b349c9015c7..e421f8ed59a 100644 --- a/test/core/metaflow_test/formatter.py +++ b/test/core/metaflow_test/formatter.py @@ -89,7 +89,7 @@ def _flow_lines(self): yield 0, "class %s(FlowSpec):" % self.flow_name for var, val in self.test.CLASS_VARS.items(): - yield 1, '%s = %s' % (var, val) + yield 1, "%s = %s" % (var, val) for var, parameter in self.test.PARAMETERS.items(): kwargs = ["%s=%s" % (k, v) for k, v in parameter.items()] diff --git a/test/core/tests/constants.py b/test/core/tests/constants.py index 59b70598fc2..23093eedd42 100644 --- a/test/core/tests/constants.py +++ b/test/core/tests/constants.py @@ -8,9 +8,11 @@ class ConstantsTest(MetaflowTest): """ PRIORITY = 0 - CLASS_VARS = {'str_const': '"this is a constant"', - 'int_const': 123, - 'obj_const': '[]'} + CLASS_VARS = { + "str_const": '"this is a constant"', + "int_const": 123, + "obj_const": "[]", + } PARAMETERS = { "int_param": {"default": 456}, @@ -21,18 +23,18 @@ class ConstantsTest(MetaflowTest): def step_all(self): # make sure class attributes are available in all steps # through joins etc - assert_equals('this is a constant', self.str_const) + assert_equals("this is a constant", self.str_const) assert_equals(123, self.int_const) # obj_const is mutable. Not much that can be done about it assert_equals([], self.obj_const) assert_equals(456, self.int_param) - assert_equals('foobar', self.str_param) + assert_equals("foobar", self.str_param) # make sure class variables are not listed as parameters from metaflow import current - assert_equals({'int_param', 'str_param'}, - set(current.parameter_names)) + + assert_equals({"int_param", "str_param"}, set(current.parameter_names)) try: self.int_param = 5 @@ -50,5 +52,5 @@ def step_all(self): def check_results(self, flow, checker): for step in flow: - checker.assert_artifact(step.name, 'int_param', 456) - checker.assert_artifact(step.name, 'int_const', 123) + checker.assert_artifact(step.name, "int_param", 456) + checker.assert_artifact(step.name, "int_const", 123) From ea74c4683501d01b4ba3e91ada157034d990d575 Mon Sep 17 00:00:00 2001 From: Savin Date: Fri, 5 Nov 2021 14:12:51 -0700 Subject: [PATCH 097/176] mflog changes for supporting AWS Lambda (#801) * mflog changes for supporting AWS Lambda * updates to mflog * updates to mflog * fix mflog capture for multiple commands (#805) Co-authored-by: Oleg Avdeev --- metaflow/mflog/__init__.py | 62 +++++++++++++++++++---- metaflow/mflog/redirect_streams.py | 54 ++++++++++++++++++++ metaflow/plugins/aws/batch/batch.py | 70 +++++++------------------- metaflow/plugins/aws/eks/kubernetes.py | 62 +++++++---------------- 4 files changed, 142 insertions(+), 106 deletions(-) create mode 100644 metaflow/mflog/redirect_streams.py diff --git a/metaflow/mflog/__init__.py b/metaflow/mflog/__init__.py index 5a9076320b4..f6fd4688785 100644 --- a/metaflow/mflog/__init__.py +++ b/metaflow/mflog/__init__.py @@ -1,4 +1,9 @@ import math +import time + +from .mflog import refine, set_should_persist + +from metaflow.util import to_unicode # Log source indicates the system that *minted the timestamp* # for the logline. This means that for a single task we can @@ -39,17 +44,17 @@ BASH_SAVE_LOGS = " ".join(BASH_SAVE_LOGS_ARGS) # this function returns a bash expression that redirects stdout -# and stderr of the given bash expression to mflog.tee -def bash_capture_logs(bash_expr, var_transform=None): +# and stderr of the given command to mflog +def capture_output_to_mflog(command_and_args, var_transform=None): if var_transform is None: var_transform = lambda s: "$%s" % s - cmd = "python -m metaflow.mflog.tee %s %s" - parts = ( - bash_expr, - cmd % (TASK_LOG_SOURCE, var_transform("MFLOG_STDOUT")), - cmd % (TASK_LOG_SOURCE, var_transform("MFLOG_STDERR")), + + return "python -m metaflow.mflog.redirect_streams %s %s %s %s" % ( + TASK_LOG_SOURCE, + var_transform("MFLOG_STDOUT"), + var_transform("MFLOG_STDERR"), + command_and_args, ) - return "(%s) 1>> >(%s) 2>> >(%s >&2)" % parts # update_delay determines how often logs should be uploaded to S3 @@ -71,7 +76,8 @@ def update_delay(secs_since_start): # this function is used to generate a Bash 'export' expression that -# sets environment variables that are used by 'tee' and 'save_logs'. +# sets environment variables that are used by 'redirect_streams' and +# 'save_logs'. # Note that we can't set the env vars statically, as some of them # may need to be evaluated during runtime def export_mflog_env_vars( @@ -99,3 +105,41 @@ def export_mflog_env_vars( env_vars["MF_DATASTORE_ROOT"] = datastore_root return "export " + " ".join("%s=%s" % kv for kv in env_vars.items()) + + +def tail_logs(prefix, stdout_tail, stderr_tail, echo, has_log_updates): + def _available_logs(tail, stream, echo, should_persist=False): + # print the latest batch of lines + try: + for line in tail: + if should_persist: + line = set_should_persist(line) + else: + line = refine(line, prefix=prefix) + echo(line.strip().decode("utf-8", errors="replace"), stream) + except Exception as ex: + echo( + "%s[ temporary error in fetching logs: %s ]" % to_unicode(prefix), + ex, + "stderr", + ) + + start_time = time.time() + next_log_update = start_time + log_update_delay = 1 + while has_log_updates(): + if time.time() > next_log_update: + _available_logs(stdout_tail, "stdout", echo) + _available_logs(stderr_tail, "stderr", echo) + now = time.time() + log_update_delay = update_delay(now - start_time) + next_log_update = now + log_update_delay + + # This sleep should never delay log updates. On the other hand, + # we should exit this loop when the task has finished without + # a long delay, regardless of the log tailing schedule + time.sleep(min(log_update_delay, 5.0)) + # It is possible that we exit the loop above before all logs have been + # tailed. + _available_logs(stdout_tail, "stdout", echo) + _available_logs(stderr_tail, "stderr", echo) diff --git a/metaflow/mflog/redirect_streams.py b/metaflow/mflog/redirect_streams.py new file mode 100644 index 00000000000..36aac03342c --- /dev/null +++ b/metaflow/mflog/redirect_streams.py @@ -0,0 +1,54 @@ +import os +import sys +import subprocess +import threading + +from .mflog import decorate + +# This script runs another process and captures stderr and stdout to a file, decorating +# lines with mflog metadata. +# +# Usage: redirect_streams SOURCE STDOUT_FILE STDERR_FILE PROGRAM ARG1 ARG2 ... + + +def reader_thread(SOURCE, dest_file, dest_stream, src): + with open(dest_file, mode="ab", buffering=0) as f: + if sys.version_info < (3, 0): + # Python 2 + for line in iter(sys.stdin.readline, ""): + # https://bugs.python.org/issue3907 + decorated = decorate(SOURCE, line) + f.write(decorated) + sys.stdout.write(line) + else: + # Python 3 + for line in src: + decorated = decorate(SOURCE, line) + f.write(decorated) + dest_stream.buffer.write(line) + + +if __name__ == "__main__": + SOURCE = sys.argv[1].encode("utf-8") + stdout_dest = sys.argv[2] + stderr_dest = sys.argv[3] + + p = subprocess.Popen( + sys.argv[4:], + env=os.environ, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + stdout_reader = threading.Thread( + target=reader_thread, args=(SOURCE, stdout_dest, sys.stdout, p.stdout) + ) + stdout_reader.start() + stderr_reader = threading.Thread( + target=reader_thread, args=(SOURCE, stderr_dest, sys.stderr, p.stderr) + ) + stderr_reader.start() + rc = p.wait() + stdout_reader.join() + stderr_reader.join() + sys.exit(rc) diff --git a/metaflow/plugins/aws/batch/batch.py b/metaflow/plugins/aws/batch/batch.py index 7e19c422cfb..13c0d9fd9d8 100644 --- a/metaflow/plugins/aws/batch/batch.py +++ b/metaflow/plugins/aws/batch/batch.py @@ -20,8 +20,8 @@ from metaflow.mflog.mflog import refine, set_should_persist from metaflow.mflog import ( export_mflog_env_vars, - bash_capture_logs, - update_delay, + capture_output_to_mflog, + tail_logs, BASH_SAVE_LOGS, ) @@ -59,8 +59,11 @@ def _command(self, environment, code_package_url, step_name, step_cmds, task_spe ) init_cmds = environment.get_package_commands(code_package_url) init_expr = " && ".join(init_cmds) - step_expr = bash_capture_logs( - " && ".join(environment.bootstrap_commands(step_name) + step_cmds) + step_expr = " && ".join( + [ + capture_output_to_mflog(a) + for a in (environment.bootstrap_commands(step_name) + step_cmds) + ] ) # construct an entry point that @@ -329,63 +332,24 @@ def wait_for_launch(job): select.poll().poll(200) prefix = b"[%s] " % util.to_bytes(self.job.id) - - def _print_available(tail, stream, should_persist=False): - # print the latest batch of lines from S3Tail - try: - for line in tail: - if should_persist: - line = set_should_persist(line) - else: - line = refine(line, prefix=prefix) - echo(line.strip().decode("utf-8", errors="replace"), stream) - except Exception as ex: - echo( - "[ temporary error in fetching logs: %s ]" % ex, - "stderr", - batch_id=self.job.id, - ) - stdout_tail = S3Tail(stdout_location) stderr_tail = S3Tail(stderr_location) # 1) Loop until the job has started wait_for_launch(self.job) - # 2) Loop until the job has finished - start_time = time.time() - is_running = True - next_log_update = start_time - log_update_delay = 1 - - while is_running: - if time.time() > next_log_update: - _print_available(stdout_tail, "stdout") - _print_available(stderr_tail, "stderr") - now = time.time() - log_update_delay = update_delay(now - start_time) - next_log_update = now + log_update_delay - is_running = self.job.is_running - - # This sleep should never delay log updates. On the other hand, - # we should exit this loop when the task has finished without - # a long delay, regardless of the log tailing schedule - d = min(log_update_delay, 5.0) - select.poll().poll(d * 1000) + # 2) Tail logs until the job has finished + tail_logs( + prefix=prefix, + stdout_tail=stdout_tail, + stderr_tail=stderr_tail, + echo=echo, + has_log_updates=lambda: self.job.is_running, + ) - # 3) Fetch remaining logs - # - # It is possible that we exit the loop above before all logs have been - # shown. - # - # TODO if we notice AWS Batch failing to upload logs to S3, we can add a - # HEAD request here to ensure that the file exists prior to calling - # S3Tail and note the user about truncated logs if it doesn't - _print_available(stdout_tail, "stdout") - _print_available(stderr_tail, "stderr") # In case of hard crashes (OOM), the final save_logs won't happen. - # We fetch the remaining logs from AWS CloudWatch and persist them to - # Amazon S3. + # We can fetch the remaining logs from AWS CloudWatch and persist them + # to Amazon S3. if self.job.is_crashed: msg = next( diff --git a/metaflow/plugins/aws/eks/kubernetes.py b/metaflow/plugins/aws/eks/kubernetes.py index c908020e6a1..652f5dcc7f1 100644 --- a/metaflow/plugins/aws/eks/kubernetes.py +++ b/metaflow/plugins/aws/eks/kubernetes.py @@ -20,8 +20,8 @@ ) from metaflow.mflog import ( export_mflog_env_vars, - bash_capture_logs, - update_delay, + capture_output_to_mflog, + tail_logs, BASH_SAVE_LOGS, ) from metaflow.mflog.mflog import refine, set_should_persist @@ -134,10 +134,13 @@ def _command( ) init_cmds = self._environment.get_package_commands(code_package_url) init_expr = " && ".join(init_cmds) - step_expr = bash_capture_logs( - " && ".join( - self._environment.bootstrap_commands(self._step_name) + step_cmds - ) + step_expr = " && ".join( + [ + capture_output_to_mflog(a) + for a in ( + self._environment.bootstrap_commands(self._step_name) + step_cmds + ) + ] ) # Construct an entry point that @@ -302,48 +305,21 @@ def wait_for_launch(job): break time.sleep(1) - def _print_available(tail, stream, should_persist=False): - # print the latest batch of lines from S3Tail - prefix = b"[%s] " % util.to_bytes(self._job.id) - try: - for line in tail: - if should_persist: - line = set_should_persist(line) - else: - line = refine(line, prefix=prefix) - echo(line.strip().decode("utf-8", errors="replace"), stream) - except Exception as ex: - echo( - "[ temporary error in fetching logs: %s ]" % ex, - "stderr", - job_id=self._job.id, - ) - + prefix = b"[%s] " % util.to_bytes(self._job.id) stdout_tail = S3Tail(stdout_location) stderr_tail = S3Tail(stderr_location) # 1) Loop until the job has started wait_for_launch(self._job) - # 2) Loop until the job has finished - start_time = time.time() - is_running = True - next_log_update = start_time - log_update_delay = 1 - - while is_running: - if time.time() > next_log_update: - _print_available(stdout_tail, "stdout") - _print_available(stderr_tail, "stderr") - now = time.time() - log_update_delay = update_delay(now - start_time) - next_log_update = now + log_update_delay - is_running = self._job.is_running - - # This sleep should never delay log updates. On the other hand, - # we should exit this loop when the task has finished without - # a long delay, regardless of the log tailing schedule - time.sleep(min(log_update_delay, 5.0)) + # 2) Tail logs until the job has finished + tail_logs( + prefix=prefix, + stdout_tail=stdout_tail, + stderr_tail=stderr_tail, + echo=echo, + has_log_updates=lambda: self._job.is_running, + ) # 3) Fetch remaining logs # @@ -355,8 +331,6 @@ def _print_available(tail, stream, should_persist=False): # exists prior to calling S3Tail and note the user about # truncated logs if it doesn't. # TODO (savin): For hard crashes, we can fetch logs from the pod. - _print_available(stdout_tail, "stdout") - _print_available(stderr_tail, "stderr") if self._job.has_failed: exit_code, reason = self._job.reason From a668785b89608f7020eb8c83e14bc3331bc03d1b Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Fri, 5 Nov 2021 14:22:19 -0700 Subject: [PATCH 098/176] add a blurb about using black to the README (#812) --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index b78374541b4..edae62e93da 100644 --- a/README.md +++ b/README.md @@ -46,4 +46,9 @@ There are several ways to get in touch with us: * Chat with us on: http://chat.metaflow.org ## Contributing + We welcome contributions to Metaflow. Please see our [contribution guide](https://docs.metaflow.org/introduction/contributing-to-metaflow) for more details. + +### Code style + +We use [black](https://black.readthedocs.io/en/stable/) as a code formatter. The easiest way to ensure your commits are always formatted with the correct version of `black` it is to use [pre-commit](https://pre-commit.com/): install it and then run `pre-commit install` once in your local copy of the repo. \ No newline at end of file From e679b715c2ee856943805d07a467c6d7c23b1b66 Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Fri, 5 Nov 2021 14:27:41 -0700 Subject: [PATCH 099/176] add default image config option as described in #489 (#813) --- metaflow/metaflow_config.py | 24 +++++++++++++++---- .../plugins/aws/eks/kubernetes_decorator.py | 15 +++++------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py index de194903f8e..2dabb3126cd 100644 --- a/metaflow/metaflow_config.py +++ b/metaflow/metaflow_config.py @@ -115,6 +115,11 @@ def from_conf(name, default=None): if METADATA_SERVICE_AUTH_KEY is not None: METADATA_SERVICE_HEADERS["x-api-key"] = METADATA_SERVICE_AUTH_KEY +# Default container image +DEFAULT_CONTAINER_IMAGE = from_conf("METAFLOW_DEFAULT_CONTAINER_IMAGE") +# Default container registry +DEFAULT_CONTAINER_REGISTRY = from_conf("METAFLOW_DEFAULT_CONTAINER_REGISTRY") + ### # AWS Batch configuration ### @@ -126,9 +131,13 @@ def from_conf(name, default=None): # Job queue for AWS Batch BATCH_JOB_QUEUE = from_conf("METAFLOW_BATCH_JOB_QUEUE") # Default container image for AWS Batch -BATCH_CONTAINER_IMAGE = from_conf("METAFLOW_BATCH_CONTAINER_IMAGE") +BATCH_CONTAINER_IMAGE = ( + from_conf("METAFLOW_BATCH_CONTAINER_IMAGE") or DEFAULT_CONTAINER_IMAGE +) # Default container registry for AWS Batch -BATCH_CONTAINER_REGISTRY = from_conf("METAFLOW_BATCH_CONTAINER_REGISTRY") +BATCH_CONTAINER_REGISTRY = ( + from_conf("METAFLOW_BATCH_CONTAINER_REGISTRY") or DEFAULT_CONTAINER_REGISTRY +) # Metadata service URL for AWS Batch BATCH_METADATA_SERVICE_URL = from_conf( "METAFLOW_SERVICE_INTERNAL_URL", METADATA_SERVICE_URL @@ -167,10 +176,15 @@ def from_conf(name, default=None): KUBERNETES_NAMESPACE = from_conf("METAFLOW_KUBERNETES_NAMESPACE") # Service account to use by K8S jobs created by Metaflow KUBERNETES_SERVICE_ACCOUNT = from_conf("METAFLOW_KUBERNETES_SERVICE_ACCOUNT") +# Default container image for K8S +KUBERNETES_CONTAINER_IMAGE = ( + from_conf("METAFLOW_KUBERNETES_CONTAINER_IMAGE") or DEFAULT_CONTAINER_IMAGE +) +# Default container registry for K8S +KUBERNETES_CONTAINER_REGISTRY = ( + from_conf("METAFLOW_KUBERNETES_CONTAINER_REGISTRY") or DEFAULT_CONTAINER_REGISTRY +) # -KUBERNETES_CONTAINER_REGISTRY = from_conf("METAFLOW_KUBERNETES_CONTAINER_REGISTRY") -# -KUBERNETES_CONTAINER_IMAGE = from_conf("METAFLOW_KUBERNETES_CONTAINER_IMAGE") ### # Conda configuration diff --git a/metaflow/plugins/aws/eks/kubernetes_decorator.py b/metaflow/plugins/aws/eks/kubernetes_decorator.py index 9f28ff60e98..60e2371f5aa 100644 --- a/metaflow/plugins/aws/eks/kubernetes_decorator.py +++ b/metaflow/plugins/aws/eks/kubernetes_decorator.py @@ -8,11 +8,8 @@ from metaflow.metadata import MetaDatum from metaflow.metadata.util import sync_local_metadata_to_datastore from metaflow.metaflow_config import ( - ECS_S3_ACCESS_IAM_ROLE, - BATCH_JOB_QUEUE, - BATCH_CONTAINER_IMAGE, - BATCH_CONTAINER_REGISTRY, - ECS_FARGATE_EXECUTION_ROLE, + KUBERNETES_CONTAINER_IMAGE, + KUBERNETES_CONTAINER_REGISTRY, DATASTORE_LOCAL_DIR, ) from metaflow.plugins import ResourcesDecorator @@ -85,8 +82,8 @@ def __init__(self, attributes=None, statically_defined=False): # If no docker image is explicitly specified, impute a default image. if not self.attributes["image"]: # If metaflow-config specifies a docker image, just use that. - if BATCH_CONTAINER_IMAGE: - self.attributes["image"] = BATCH_CONTAINER_IMAGE + if KUBERNETES_CONTAINER_IMAGE: + self.attributes["image"] = KUBERNETES_CONTAINER_IMAGE # If metaflow-config doesn't specify a docker image, assign a # default docker image. else: @@ -98,9 +95,9 @@ def __init__(self, attributes=None, statically_defined=False): ) # Assign docker registry URL for the image. if not get_docker_registry(self.attributes["image"]): - if BATCH_CONTAINER_REGISTRY: + if KUBERNETES_CONTAINER_REGISTRY: self.attributes["image"] = "%s/%s" % ( - BATCH_CONTAINER_REGISTRY.rstrip("/"), + KUBERNETES_CONTAINER_REGISTRY.rstrip("/"), self.attributes["image"], ) From ddc2009bfa559a8aeb0ea1a85043d055f23e50fc Mon Sep 17 00:00:00 2001 From: Romain Date: Tue, 9 Nov 2021 10:29:45 -0800 Subject: [PATCH 100/176] =?UTF-8?q?Fix=20an=20issue=20with=20load=5Fartifa?= =?UTF-8?q?cts=20when=20several=20artifacts=20have=20the=20same=E2=80=A6?= =?UTF-8?q?=20(#817)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix an issue with load_artifacts when several artifacts have the same hash Incorrect results could be returned from load_artifacts when multiple artifacts hash to the same value and are all requested at the same time. * Added comment on order --- metaflow/datastore/content_addressed_store.py | 3 ++- metaflow/datastore/datastore_storage.py | 2 ++ metaflow/datastore/task_datastore.py | 9 +++------ 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/metaflow/datastore/content_addressed_store.py b/metaflow/datastore/content_addressed_store.py index 4b261523ad9..d98f32f2ed6 100644 --- a/metaflow/datastore/content_addressed_store.py +++ b/metaflow/datastore/content_addressed_store.py @@ -118,7 +118,8 @@ def load_blobs(self, keys, force_raw=False): Returns ------- - Returns an iterator of (string, bytes) tuples + Returns an iterator of (string, bytes) tuples; the iterator will return the keys + in the same order as the input argument. """ load_paths = [] for key in keys: diff --git a/metaflow/datastore/datastore_storage.py b/metaflow/datastore/datastore_storage.py index 264d2b15833..d3f9020b339 100644 --- a/metaflow/datastore/datastore_storage.py +++ b/metaflow/datastore/datastore_storage.py @@ -262,5 +262,7 @@ def load_bytes(self, paths): Note that the file at `path` may no longer be accessible outside of the scope of the returned object. + + Note that the order of the iterator will be the same as the input paths. """ raise NotImplementedError diff --git a/metaflow/datastore/task_datastore.py b/metaflow/datastore/task_datastore.py index 418b253fa7a..c75be9ca4a2 100644 --- a/metaflow/datastore/task_datastore.py +++ b/metaflow/datastore/task_datastore.py @@ -338,7 +338,6 @@ def load_artifacts(self, names): "load artifacts" % self._path ) to_load = [] - sha_to_names = {} for name in names: info = self._info.get(name) # We use gzip+pickle-v2 as this is the oldest/most compatible. @@ -353,15 +352,13 @@ def load_artifacts(self, names): "Python 3.4 or later is required to load artifact '%s'" % name ) else: - sha = self._objects[name] - sha_to_names[sha] = name - to_load.append(sha) + to_load.append(self._objects[name]) # At this point, we load what we don't have from the CAS # We assume that if we have one "old" style artifact, all of them are # like that which is an easy assumption to make since artifacts are all # stored by the same implementation of the datastore for a given task. - for sha, blob in self._ca_store.load_blobs(to_load): - yield sha_to_names[sha], pickle.loads(blob) + for name, (_, blob) in zip(names, self._ca_store.load_blobs(to_load)): + yield name, pickle.loads(blob) @require_mode("r") def get_artifact_sizes(self, names): From 423f9fa91347e00d8232def1ec26b41e6cdf043b Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Wed, 10 Nov 2021 15:48:17 +0200 Subject: [PATCH 101/176] UBF with cluster_size --- metaflow/decorators.py | 22 +++++++++++++++++ metaflow/flowspec.py | 21 +++++++--------- metaflow/graph.py | 19 ++++++++------- metaflow/lint.py | 2 ++ metaflow/plugins/__init__.py | 2 -- metaflow/plugins/aws/batch/batch.py | 14 +++++------ metaflow/plugins/aws/batch/batch_cli.py | 9 +++---- metaflow/plugins/aws/batch/batch_client.py | 24 ++++++++++--------- metaflow/plugins/aws/batch/batch_decorator.py | 22 +++++++---------- metaflow/plugins/multinode_decorator.py | 13 ---------- metaflow/runtime.py | 14 +++++++++-- 11 files changed, 90 insertions(+), 72 deletions(-) delete mode 100644 metaflow/plugins/multinode_decorator.py diff --git a/metaflow/decorators.py b/metaflow/decorators.py index 241124db3b8..54d896421b1 100644 --- a/metaflow/decorators.py +++ b/metaflow/decorators.py @@ -343,6 +343,23 @@ def task_finished( pass +class MultinodeDecorator(StepDecorator): + name = "multinode" + defaults = {} + + def __init__(self, attributes=None, statically_defined=False): + super(MultinodeDecorator, self).__init__(attributes, statically_defined) + + def runtime_step_cli( + self, cli_args, retry_count, max_user_code_retries, ubf_context + ): + from .unbounded_foreach import UBF_CONTROL + + if ubf_context == UBF_CONTROL: + cluster_size = cli_args.task.ubf_iter.cluster_size + cli_args.command_options["cluster-size"] = str(cluster_size) + + def _base_flow_decorator(decofunc, *args, **kwargs): """ Decorator prototype for all flow (class) decorators. This function gets @@ -498,3 +515,8 @@ def _import_plugin_decorators(globals_dict): # add flow-level decorators for decotype in FLOW_DECORATORS: globals_dict[decotype.name] = partial(_base_flow_decorator, decotype) + + # Add multinode decorator + globals_dict[MultinodeDecorator.name] = partial( + _base_step_decorator, MultinodeDecorator + ) diff --git a/metaflow/flowspec.py b/metaflow/flowspec.py index b102bd1a649..4e44be7a2d5 100644 --- a/metaflow/flowspec.py +++ b/metaflow/flowspec.py @@ -37,8 +37,8 @@ class Multinode(UnboundedForeachInput): Unbounded-for-each placeholder for supporting multinode steps. """ - def __init__(self): - pass + def __init__(self, cluster_size): + self.cluster_size = cluster_size def __getitem__(self, item): return item @@ -144,10 +144,6 @@ def __iter__(self): return iter(self._steps) def __getattr__(self, name): - if name == "_multinode_ubf_iter": - # Special multinode UBF iterator object - return Multinode() - if self._datastore and name in self._datastore: # load the attribute from the datastore... x = self._datastore[name] @@ -444,6 +440,7 @@ def next(self, *dsts, **kwargs): step = self._current_step foreach = kwargs.pop("foreach", None) + cluster_size = kwargs.pop("cluster_size", None) condition = kwargs.pop("condition", None) if kwargs: kw = next(iter(kwargs)) @@ -481,12 +478,12 @@ def next(self, *dsts, **kwargs): raise InvalidNextException(msg) funcs.append(name) - # if only one dst, check if it is a multinode one. If it is, then create a synthetic - # UBF construct. - if len(dsts) == 1: - dst = dsts[0] - if any(deco.name == "multinode" for deco in dst.__func__.decorators): - foreach = "_multinode_ubf_iter" + if cluster_size: + assert ( + len(dsts) == 1 + ), "Only one destination allowed when cluster_size used in self.next()" + foreach = "_multinode_ubf_iter" + self._multinode_ubf_iter = Multinode(cluster_size) # check: foreach and condition are mutually exclusive if not (foreach is None or condition is None): diff --git a/metaflow/graph.py b/metaflow/graph.py index 1b036213dbd..da844c1297d 100644 --- a/metaflow/graph.py +++ b/metaflow/graph.py @@ -2,6 +2,8 @@ import ast import re +# from metaflow.decorators import MultinodeDecorator + def deindent_docstring(doc): if doc: @@ -63,7 +65,7 @@ def __init__(self, func_ast, decos, doc): self.in_funcs = set() self.split_parents = [] self.matching_join = None - self.is_multinode_step = any(deco.name == "multinode" for deco in decos) + # self.is_multinode_step = any(isinstance(deco, MultinodeDecorator) for deco in decos) # these attributes are populated by _postprocess self.is_inside_foreach = False @@ -94,7 +96,10 @@ def _parse(self, func_ast): self.invalid_tail_next = True self.tail_next_lineno = tail.lineno self.out_funcs = [e.attr for e in tail.value.args] - keywords = dict((k.arg, k.value.s) for k in tail.value.keywords) + + keywords = dict( + (k.arg, getattr(k.value, "s", None)) for k in tail.value.keywords + ) if len(keywords) == 1: if "foreach" in keywords: @@ -103,6 +108,10 @@ def _parse(self, func_ast): if len(self.out_funcs) == 1: self.foreach_param = keywords["foreach"] self.invalid_tail_next = False + elif "cluster_size" in keywords: + self.type = "foreach" + if len(self.out_funcs) == 1: + self.invalid_tail_next = False elif "condition" in keywords: # TYPE: split-or self.type = "split-or" @@ -193,12 +202,6 @@ def _postprocess(self): def _traverse_graph(self): def traverse(node, seen, split_parents): - # check multinode: if the next step is a multinode-step, then - # this step is of type "foreach" - if len(node.out_funcs) == 1: - child = self[node.out_funcs[0]] - if child.is_multinode_step: - node.type = "foreach" if node.type in ("split-or", "split-and", "foreach"): node.split_parents = split_parents split_parents = split_parents + [node.name] diff --git a/metaflow/lint.py b/metaflow/lint.py index 6dd84fa8521..04b16bcbfbf 100644 --- a/metaflow/lint.py +++ b/metaflow/lint.py @@ -31,6 +31,8 @@ def ensure_acyclicity(self, f): def ensure_non_nested_foreach(self, f): return self._decorate("require_non_nested_foreach", f) + # TODO: cluster_size matches with a multinode + def check(self, f): self._checks.append(f) f.attrs = [] diff --git a/metaflow/plugins/__init__.py b/metaflow/plugins/__init__.py index 3d1ceddcdaa..086a0c57dae 100644 --- a/metaflow/plugins/__init__.py +++ b/metaflow/plugins/__init__.py @@ -131,7 +131,6 @@ def _merge_lists(base, overrides, attr): InternalTestUnboundedForeachDecorator, InternalTestUnboundedForeachInput, ) -from .multinode_decorator import MultinodeDecorator from .conda.conda_step_decorator import CondaStepDecorator STEP_DECORATORS = _merge_lists( @@ -146,7 +145,6 @@ def _merge_lists(base, overrides, attr): StepFunctionsInternalDecorator, CondaStepDecorator, InternalTestUnboundedForeachDecorator, - MultinodeDecorator, ], _ext_plugins.STEP_DECORATORS, "name", diff --git a/metaflow/plugins/aws/batch/batch.py b/metaflow/plugins/aws/batch/batch.py index a8e548f2016..23c53bee001 100644 --- a/metaflow/plugins/aws/batch/batch.py +++ b/metaflow/plugins/aws/batch/batch.py @@ -175,7 +175,7 @@ def create_job( env={}, attrs={}, host_volumes=None, - nodes=1, + cluster_size=1, ): job_name = self._job_name( attrs.get("metaflow.user"), @@ -206,7 +206,7 @@ def create_job( max_swap, swappiness, host_volumes=host_volumes, - nodes=nodes, + cluster_size=cluster_size, ) .cpu(cpu) .gpu(gpu) @@ -275,7 +275,7 @@ def launch_job( max_swap=None, swappiness=None, host_volumes=None, - nodes=1, + cluster_size=1, env={}, attrs={}, ): @@ -307,9 +307,9 @@ def launch_job( env=env, attrs=attrs, host_volumes=host_volumes, - nodes=nodes, + cluster_size=cluster_size, ) - self.nodes = nodes + self.cluster_size = cluster_size self.job = job.execute() def wait(self, stdout_location, stderr_location, echo=None): @@ -362,8 +362,8 @@ def _print_available(tail, stream, should_persist=False): stderr_tail = S3Tail(stderr_location) child_jobs = [] - if self.nodes > 1: - for node in range(1, self.nodes): + if self.cluster_size > 1: + for node in range(1, self.cluster_size): child_job = copy.copy(self.job) child_job._id = child_job._id + "#{}".format(node) child_jobs.append(child_job) diff --git a/metaflow/plugins/aws/batch/batch_cli.py b/metaflow/plugins/aws/batch/batch_cli.py index 04db8f17784..2d2dfcbfc55 100644 --- a/metaflow/plugins/aws/batch/batch_cli.py +++ b/metaflow/plugins/aws/batch/batch_cli.py @@ -143,7 +143,7 @@ def kill(ctx, run_id, user, my_runs): # TODO: Maybe remove it altogether since it's not used here @click.option("--ubf-context", default=None, type=click.Choice([None, "ubf_control"])) @click.option("--host-volumes", multiple=True) -@click.option("--nodes", type=int) +@click.option("--cluster-size", type=int) @click.pass_context def step( ctx, @@ -163,7 +163,7 @@ def step( max_swap=None, swappiness=None, host_volumes=None, - nodes=None, + cluster_size=None, **kwargs ): def echo(msg, stream="stderr", batch_id=None): @@ -192,7 +192,8 @@ def echo(msg, stream="stderr", batch_id=None): kwargs["input_paths"] = "".join("${%s}" % s for s in split_vars.keys()) step_args = " ".join(util.dict_to_cli_options(kwargs)) - if nodes > 1: + cluster_size = cluster_size or 1 + if cluster_size and cluster_size > 1: # For multinode, we need to add a placeholder that can be mutated by the caller step_args += " [multinode-args]" step_cli = u"{entrypoint} {top_args} step {step} {step_args}".format( @@ -287,7 +288,7 @@ def _sync_metadata(): env=env, attrs=attrs, host_volumes=host_volumes, - nodes=nodes, + cluster_size=cluster_size, ) except Exception as e: traceback.print_exc() diff --git a/metaflow/plugins/aws/batch/batch_client.py b/metaflow/plugins/aws/batch/batch_client.py index 523bc4d4943..a5ddd85ef99 100644 --- a/metaflow/plugins/aws/batch/batch_client.py +++ b/metaflow/plugins/aws/batch/batch_client.py @@ -97,8 +97,8 @@ def execute(self): ) # Multinode - if getattr(self, "_nodes", 0) > 1: - num_nodes = self._nodes + if getattr(self, "cluster_size", 0) > 1: + num_nodes = self.cluster_size main_task_override = copy.deepcopy(self.payload["containerOverrides"]) # TERRIBLE HACK JUST TO TRY THIS WORKS @@ -153,7 +153,7 @@ def _register_job_definition( max_swap, swappiness, host_volumes, - nodes, + cluster_size, ): # identify platform from any compute environment associated with the # queue @@ -192,7 +192,7 @@ def _register_job_definition( } if platform == "FARGATE" or platform == "FARGATE_SPOT": - if nodes > 1: + if cluster_size > 1: raise BatchJobException("Fargate does not support multinode jobs.") if execution_role is None: raise BatchJobException( @@ -263,10 +263,13 @@ def _register_job_definition( {"sourceVolume": name, "containerPath": host_path} ) - nodes = int(nodes) - if nodes > 1: + self.cluster_size = cluster_size or 1 + if self.cluster_size > 1: job_definition["type"] = "multinode" - job_definition["nodeProperties"] = {"numNodes": nodes, "mainNode": 0} + job_definition["nodeProperties"] = { + "numNodes": self.cluster_size, + "mainNode": 0, + } job_definition["nodeProperties"]["nodeRangeProperties"] = [ { "targetNodes": "0:0", # The properties are same for main node and others, @@ -275,12 +278,11 @@ def _register_job_definition( "container": job_definition["containerProperties"], }, { - "targetNodes": "1:{}".format(nodes - 1), + "targetNodes": "1:{}".format(self.cluster_size - 1), "container": job_definition["containerProperties"], }, ] del job_definition["containerProperties"] # not used for multi-node - self._nodes = nodes # check if job definition already exists def_name = ( @@ -319,7 +321,7 @@ def job_def( max_swap, swappiness, host_volumes, - nodes, + cluster_size, ): self.payload["jobDefinition"] = self._register_job_definition( image, @@ -330,7 +332,7 @@ def job_def( max_swap, swappiness, host_volumes, - nodes, + cluster_size, ) return self diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index 018cc331b3e..25176e237f8 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -98,7 +98,6 @@ def my_step(self): "max_swap": None, "swappiness": None, "host_volumes": None, - "nodes": 1, } package_url = None package_sha = None @@ -153,10 +152,7 @@ def step_init(self, flow, graph, step, decos, environment, flow_datastore, logge my_val = self.attributes.get(k) if not (my_val is None and v is None): self.attributes[k] = str(max(int(my_val or 0), int(v or 0))) - elif ( - deco.__class__.__name__ == "MultinodeDecorator" - ): # avoid circular dependency - self.attributes["nodes"] = deco.nodes + # Set run time limit for the AWS Batch job. self.run_time_limit = get_run_time_limit_for_task(decos) if self.run_time_limit < 60: @@ -255,14 +251,14 @@ def task_pre_step( self._save_logs_sidecar = SidecarSubProcess("save_logs_periodically") - nodes = self.attributes["nodes"] - - if nodes > 1 and ubf_context == UBF_CONTROL: + cluster_size = int(os.environ.get("AWS_BATCH_JOB_NUM_NODES", 1)) + if cluster_size > 1 and ubf_context == UBF_CONTROL: # UBF handling for multinode case control_task_id = current.task_id top_task_id = control_task_id.replace("control-", "") # chop "-0" mapper_task_ids = [control_task_id] + [ - "%s-node-%d" % (top_task_id, node_idx) for node_idx in range(1, nodes) + "%s-node-%d" % (top_task_id, node_idx) + for node_idx in range(1, cluster_size) ] flow._control_mapper_tasks = [ "%s/%s/%s" % (run_id, step_name, mapper_task_id) @@ -270,8 +266,8 @@ def task_pre_step( ] flow._control_task_is_mapper_zero = True - if nodes > 1: - _set_multinode_environment() + if cluster_size > 1: + _setup_multinode_environment() def task_post_step( self, step_name, flow, graph, retry_count, max_user_code_retries @@ -362,8 +358,8 @@ def _save_package_once(cls, flow_datastore, package): )[0] -def _set_multinode_environment(): - # setup the multinode environment +def _setup_multinode_environment(): + # setup the multinode environment variables. import socket if "AWS_BATCH_JOB_MAIN_NODE_PRIVATE_IPV4_ADDRESS" not in os.environ: diff --git a/metaflow/plugins/multinode_decorator.py b/metaflow/plugins/multinode_decorator.py deleted file mode 100644 index ca0ce7a054f..00000000000 --- a/metaflow/plugins/multinode_decorator.py +++ /dev/null @@ -1,13 +0,0 @@ -from metaflow.decorators import StepDecorator -from metaflow.unbounded_foreach import UnboundedForeachInput - - -class MultinodeDecorator(StepDecorator): - name = "multinode" - defaults = { - "nodes": 2, - } - - def __init__(self, attributes=None, statically_defined=False): - self.nodes = attributes["nodes"] - super(MultinodeDecorator, self).__init__(attributes, statically_defined) diff --git a/metaflow/runtime.py b/metaflow/runtime.py index 0107a669acb..6ceacad5cfe 100644 --- a/metaflow/runtime.py +++ b/metaflow/runtime.py @@ -435,11 +435,17 @@ def _queue_task_foreach(self, task, next_steps): next_step = next_steps[0] unbounded_foreach = not task.results.is_none("_unbounded_foreach") - if unbounded_foreach: # Need to push control process related task. + ubf_iter_name = task.results.get("_foreach_var") + ubf_iter = task.results.get(ubf_iter_name) self._queue_push( - next_step, {"input_paths": [task.path], "ubf_context": UBF_CONTROL} + next_step, + { + "input_paths": [task.path], + "ubf_context": UBF_CONTROL, + "ubf_iter": ubf_iter, + }, ) else: num_splits = task.results["_foreach_num_splits"] @@ -596,6 +602,7 @@ def __init__( input_paths=None, split_index=None, ubf_context=None, + ubf_iter=None, clone_run_id=None, origin_ds_set=None, may_clone=False, @@ -614,6 +621,8 @@ def __init__( # easily. There is anyway a corresponding int id stored in the # metadata backend - so this should be fine. task_id = "control-%s-%s-%s" % (run, input_step, input_task) + else: + self.multinode_iterator = None # Register only regular Metaflow (non control) tasks. if task_id is None: task_id = str(metadata.new_task_id(run_id, step)) @@ -633,6 +642,7 @@ def __init__( self.input_paths = input_paths self.split_index = split_index self.ubf_context = ubf_context + self.ubf_iter = ubf_iter self.decos = decos self.entrypoint = entrypoint self.environment = environment From aa986609dfbb6c846c338d02fcc8fe2b90513fcb Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Wed, 10 Nov 2021 16:47:52 +0200 Subject: [PATCH 102/176] Local test for multinode --- metaflow/cli.py | 9 ++ metaflow/decorators.py | 110 +++++++++++++++++++++++++ metaflow/plugins/aws/batch/batch.py | 1 + metaflow/plugins/aws/eks/kubernetes.py | 1 + test/multinode/multinode_test_flow.py | 41 +++++++++ 5 files changed, 162 insertions(+) create mode 100644 test/multinode/multinode_test_flow.py diff --git a/metaflow/cli.py b/metaflow/cli.py index cac4c75737c..289d9137bb1 100644 --- a/metaflow/cli.py +++ b/metaflow/cli.py @@ -476,6 +476,12 @@ def echo_unicode(line, **kwargs): type=click.Choice(["none", UBF_CONTROL, UBF_TASK]), help="Provides additional context if this task is of type " "unbounded foreach.", ) +@click.option( + "--cluster-size", + default=0, + type=int, + help="Size of cluster. Ignored in local mode.", +) @click.pass_context def step( ctx, @@ -492,7 +498,9 @@ def step( clone_run_id=None, decospecs=None, ubf_context="none", + cluster_size=None, ): + print(sys.argv) if ubf_context == "none": ubf_context = None if opt_namespace is not None: @@ -902,6 +910,7 @@ def start( monitor=None, **deco_options ): + print("START", ctx) global echo if quiet: echo = echo_dev_null diff --git a/metaflow/decorators.py b/metaflow/decorators.py index 54d896421b1..df582d63922 100644 --- a/metaflow/decorators.py +++ b/metaflow/decorators.py @@ -1,6 +1,8 @@ import traceback from functools import partial import re +import os +import sys from .flowspec import FlowSpec from .exception import ( @@ -350,6 +352,11 @@ class MultinodeDecorator(StepDecorator): def __init__(self, attributes=None, statically_defined=False): super(MultinodeDecorator, self).__init__(attributes, statically_defined) + def step_init( + self, flow, graph, step_name, decorators, environment, flow_datastore, logger + ): + self.environment = environment + def runtime_step_cli( self, cli_args, retry_count, max_user_code_retries, ubf_context ): @@ -359,6 +366,29 @@ def runtime_step_cli( cluster_size = cli_args.task.ubf_iter.cluster_size cli_args.command_options["cluster-size"] = str(cluster_size) + def task_decorate( + self, step_func, flow, graph, retry_count, max_user_code_retries, ubf_context + ): + from .unbounded_foreach import UBF_CONTROL + + if ( + ubf_context == UBF_CONTROL + and os.environ.get("METAFLOW_RUNTIME_ENVIRONMENT", "local") == "local" + ): + from functools import partial + + env_to_use = getattr(self.environment, "base_env", self.environment) + + return partial( + _local_multinode_control_task_step_func, + flow, + env_to_use, + step_func, + retry_count, + ) + else: + return step_func + def _base_flow_decorator(decofunc, *args, **kwargs): """ @@ -520,3 +550,83 @@ def _import_plugin_decorators(globals_dict): globals_dict[MultinodeDecorator.name] = partial( _base_step_decorator, MultinodeDecorator ) + + +def _local_multinode_control_task_step_func(flow, env_to_use, step_func, retry_count): + """ + Used as multinode UBF control task when run in local mode. + """ + from metaflow import current + from metaflow.cli_args import cli_args + from metaflow.unbounded_foreach import UBF_TASK + import subprocess + + assert flow._unbounded_foreach + foreach_iter = flow._multinode_ubf_iter + if foreach_iter.__class__.__name__ != "Multinode": + raise MetaflowException( + "Expected Multinode iterator object, got:" + foreach_iter.__class__.__name__ + ) + + cluster_size = foreach_iter.cluster_size + os.environ["MF_MULTINODE_NUM_NODES"] = str(cluster_size) + os.environ["MF_MULTINODE_MAIN_IP"] = "127.0.0.1" + print("cluster size", cluster_size) + run_id = current.run_id + step_name = current.step_name + control_task_id = current.task_id + + (_, split_step_name, split_task_id) = control_task_id.split("-")[1:] + # UBF handling for multinode case + top_task_id = control_task_id.replace("control-", "") # chop "-0" + mapper_task_ids = [control_task_id] + # If we are running inside Conda, we use the base executable FIRST; + # the conda environment will then be used when runtime_step_cli is + # called. This is so that it can properly set up all the metaflow + # aliases needed. + executable = env_to_use.executable(step_name) + script = sys.argv[0] + + # start workers + subprocesses = [] + for node_index in range(1, cluster_size): + task_id = "%s-node-%d" % (top_task_id, node_index) + mapper_task_ids.append(task_id) + os.environ["MF_MULTINODE_NODE_INDEX"] = str(node_index) + input_paths = "%s/%s/%s" % (run_id, split_step_name, split_task_id) + # Override specific `step` kwargs. + kwargs = cli_args.step_kwargs + kwargs["split_index"] = str(node_index) + kwargs["run_id"] = run_id + kwargs["task_id"] = task_id + kwargs["input_paths"] = input_paths + kwargs["ubf_context"] = UBF_TASK + kwargs["retry_count"] = str(retry_count) + + cmd = cli_args.step_command(executable, script, step_name, step_kwargs=kwargs) + step_cli = u" ".join(cmd) + # Print cmdline for execution. Doesn't work without the temporary + # unicode object while using `print`. + print( + u"[${cwd}] Starting split#{split} with cmd:{cmd}".format( + cwd=os.getcwd(), split=node_index, cmd=step_cli + ) + ) + p = subprocess.Popen(cmd) + subprocesses.append(p) + + flow._control_mapper_tasks = [ + "%s/%s/%s" % (run_id, step_name, mapper_task_id) + for mapper_task_id in mapper_task_ids + ] + flow._control_task_is_mapper_zero = True + + # run the step function ourselves + os.environ["MF_MULTINODE_NODE_INDEX"] = "0" + step_func() + + # join the subprocesses + for p in subprocesses: + p.wait() + if p.returncode: + raise Exception("Subprocess failed, return code {}".format(p.returncode)) diff --git a/metaflow/plugins/aws/batch/batch.py b/metaflow/plugins/aws/batch/batch.py index 23c53bee001..c806700c9b3 100644 --- a/metaflow/plugins/aws/batch/batch.py +++ b/metaflow/plugins/aws/batch/batch.py @@ -229,6 +229,7 @@ def create_job( .environment_variable("METAFLOW_DATATOOLS_S3ROOT", DATATOOLS_S3ROOT) .environment_variable("METAFLOW_DEFAULT_DATASTORE", "s3") .environment_variable("METAFLOW_DEFAULT_METADATA", DEFAULT_METADATA) + .environment_variable("METAFLOW_RUNTIME_ENVIRONMENT", "aws-batch") ) # Skip setting METAFLOW_DATASTORE_SYSROOT_LOCAL because metadata sync between the local user # instance and the remote AWS Batch instance assumes metadata is stored in DATASTORE_LOCAL_DIR diff --git a/metaflow/plugins/aws/eks/kubernetes.py b/metaflow/plugins/aws/eks/kubernetes.py index eb61c006fae..94e620f311b 100644 --- a/metaflow/plugins/aws/eks/kubernetes.py +++ b/metaflow/plugins/aws/eks/kubernetes.py @@ -242,6 +242,7 @@ def create_job( .environment_variable("METAFLOW_DEFAULT_DATASTORE", "s3") .environment_variable("METAFLOW_DEFAULT_METADATA", DEFAULT_METADATA) .environment_variable("METAFLOW_KUBERNETES_WORKLOAD", 1) + .environment_variable("METAFLOW_RUNTIME_ENVIRONMENT", "eks") .label("app", "metaflow") .label("metaflow/flow_name", sanitize_label_value(self._flow_name)) .label("metaflow/run_id", sanitize_label_value(self._run_id)) diff --git a/test/multinode/multinode_test_flow.py b/test/multinode/multinode_test_flow.py new file mode 100644 index 00000000000..8b37bbbd397 --- /dev/null +++ b/test/multinode/multinode_test_flow.py @@ -0,0 +1,41 @@ +from metaflow import FlowSpec, step, batch, retry, resources, Parameter, multinode +import os + + +class MultinodeTest(FlowSpec): + + cluster_size = Parameter( + "cluster_size", help="Number of nodes in cluster", default=3 + ) + + @step + def start(self): + print("Start") + self.next(self.multinode_step, cluster_size=self.cluster_size) + + @multinode + # @batch + @step + def multinode_step(self): + self.node_index = int(os.environ["MF_MULTINODE_NODE_INDEX"]) + self.num_nodes = int(os.environ["MF_MULTINODE_NUM_NODES"]) + print("multinode_step: node {} finishing.".format(self.node_index)) + self.next(self.multinode_end) + + @step + def multinode_end(self, inputs): + j = 0 + for input in inputs: + assert input.node_index == j + assert input.num_nodes == self.cluster_size + j += 1 + assert j == self.cluster_size + self.next(self.end) + + @step + def end(self): + pass + + +if __name__ == "__main__": + MultinodeTest() From 6d0d61dc4621c07865477a023e032fd626f05d62 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Mon, 15 Nov 2021 11:49:29 +0200 Subject: [PATCH 103/176] rename multinode/clustersize, move to plugins --- metaflow/decorators.py | 19 ----------------- metaflow/flowspec.py | 21 +++++++++---------- metaflow/graph.py | 4 +--- metaflow/lint.py | 2 +- metaflow/plugins/__init__.py | 2 ++ metaflow/plugins/aws/batch/batch.py | 14 ++++++------- metaflow/plugins/aws/batch/batch_cli.py | 10 ++++----- metaflow/plugins/aws/batch/batch_client.py | 21 +++++++++---------- metaflow/plugins/aws/batch/batch_decorator.py | 13 ++++++------ metaflow/plugins/parallel_decorator.py | 17 +++++++++++++++ metaflow/runtime.py | 2 -- 11 files changed, 59 insertions(+), 66 deletions(-) create mode 100644 metaflow/plugins/parallel_decorator.py diff --git a/metaflow/decorators.py b/metaflow/decorators.py index 54d896421b1..8f1a97307f7 100644 --- a/metaflow/decorators.py +++ b/metaflow/decorators.py @@ -343,21 +343,6 @@ def task_finished( pass -class MultinodeDecorator(StepDecorator): - name = "multinode" - defaults = {} - - def __init__(self, attributes=None, statically_defined=False): - super(MultinodeDecorator, self).__init__(attributes, statically_defined) - - def runtime_step_cli( - self, cli_args, retry_count, max_user_code_retries, ubf_context - ): - from .unbounded_foreach import UBF_CONTROL - - if ubf_context == UBF_CONTROL: - cluster_size = cli_args.task.ubf_iter.cluster_size - cli_args.command_options["cluster-size"] = str(cluster_size) def _base_flow_decorator(decofunc, *args, **kwargs): @@ -516,7 +501,3 @@ def _import_plugin_decorators(globals_dict): for decotype in FLOW_DECORATORS: globals_dict[decotype.name] = partial(_base_flow_decorator, decotype) - # Add multinode decorator - globals_dict[MultinodeDecorator.name] = partial( - _base_step_decorator, MultinodeDecorator - ) diff --git a/metaflow/flowspec.py b/metaflow/flowspec.py index 4e44be7a2d5..ab4cac77188 100644 --- a/metaflow/flowspec.py +++ b/metaflow/flowspec.py @@ -32,13 +32,13 @@ def __init__(self, msg): super(InvalidNextException, self).__init__(msg, line_no) -class Multinode(UnboundedForeachInput): +class ParallelUBF(UnboundedForeachInput): """ - Unbounded-for-each placeholder for supporting multinode steps. + Unbounded-for-each placeholder for supporting parallel (multi-node) steps. """ - def __init__(self, cluster_size): - self.cluster_size = cluster_size + def __init__(self, num_parallel): + self.num_parallel = num_parallel def __getitem__(self, item): return item @@ -440,7 +440,7 @@ def next(self, *dsts, **kwargs): step = self._current_step foreach = kwargs.pop("foreach", None) - cluster_size = kwargs.pop("cluster_size", None) + num_parallel = kwargs.pop("num_parallel", None) condition = kwargs.pop("condition", None) if kwargs: kw = next(iter(kwargs)) @@ -478,12 +478,11 @@ def next(self, *dsts, **kwargs): raise InvalidNextException(msg) funcs.append(name) - if cluster_size: - assert ( - len(dsts) == 1 - ), "Only one destination allowed when cluster_size used in self.next()" - foreach = "_multinode_ubf_iter" - self._multinode_ubf_iter = Multinode(cluster_size) + if num_parallel: + if len(dsts) > 1: + raise InvalidNextException("Only one destination allowed when num_parallel used in self.next()") + foreach = "_parallel_ubf_iter" + self._parallel_ubf_iter = ParallelUBF(num_parallel) # check: foreach and condition are mutually exclusive if not (foreach is None or condition is None): diff --git a/metaflow/graph.py b/metaflow/graph.py index da844c1297d..2af2f9d98b2 100644 --- a/metaflow/graph.py +++ b/metaflow/graph.py @@ -2,8 +2,6 @@ import ast import re -# from metaflow.decorators import MultinodeDecorator - def deindent_docstring(doc): if doc: @@ -108,7 +106,7 @@ def _parse(self, func_ast): if len(self.out_funcs) == 1: self.foreach_param = keywords["foreach"] self.invalid_tail_next = False - elif "cluster_size" in keywords: + elif "num_parallel" in keywords: self.type = "foreach" if len(self.out_funcs) == 1: self.invalid_tail_next = False diff --git a/metaflow/lint.py b/metaflow/lint.py index 04b16bcbfbf..445da20a34d 100644 --- a/metaflow/lint.py +++ b/metaflow/lint.py @@ -31,7 +31,7 @@ def ensure_acyclicity(self, f): def ensure_non_nested_foreach(self, f): return self._decorate("require_non_nested_foreach", f) - # TODO: cluster_size matches with a multinode + # TODO: num_parallel matches with a multinode def check(self, f): self._checks.append(f) diff --git a/metaflow/plugins/__init__.py b/metaflow/plugins/__init__.py index 086a0c57dae..203389039d2 100644 --- a/metaflow/plugins/__init__.py +++ b/metaflow/plugins/__init__.py @@ -122,6 +122,7 @@ def _merge_lists(base, overrides, attr): from .catch_decorator import CatchDecorator from .timeout_decorator import TimeoutDecorator from .environment_decorator import EnvironmentDecorator +from .parallel_decorator import ParallelDecorator from .retry_decorator import RetryDecorator from .resources_decorator import ResourcesDecorator from .aws.batch.batch_decorator import BatchDecorator @@ -144,6 +145,7 @@ def _merge_lists(base, overrides, attr): KubernetesDecorator, StepFunctionsInternalDecorator, CondaStepDecorator, + ParallelDecorator, InternalTestUnboundedForeachDecorator, ], _ext_plugins.STEP_DECORATORS, diff --git a/metaflow/plugins/aws/batch/batch.py b/metaflow/plugins/aws/batch/batch.py index 23c53bee001..3a1410be0c8 100644 --- a/metaflow/plugins/aws/batch/batch.py +++ b/metaflow/plugins/aws/batch/batch.py @@ -175,7 +175,7 @@ def create_job( env={}, attrs={}, host_volumes=None, - cluster_size=1, + num_parallel=1, ): job_name = self._job_name( attrs.get("metaflow.user"), @@ -206,7 +206,7 @@ def create_job( max_swap, swappiness, host_volumes=host_volumes, - cluster_size=cluster_size, + num_parallel=num_parallel, ) .cpu(cpu) .gpu(gpu) @@ -275,7 +275,7 @@ def launch_job( max_swap=None, swappiness=None, host_volumes=None, - cluster_size=1, + num_parallel=1, env={}, attrs={}, ): @@ -307,9 +307,9 @@ def launch_job( env=env, attrs=attrs, host_volumes=host_volumes, - cluster_size=cluster_size, + num_parallel=num_parallel, ) - self.cluster_size = cluster_size + self.num_parallel = num_parallel self.job = job.execute() def wait(self, stdout_location, stderr_location, echo=None): @@ -362,8 +362,8 @@ def _print_available(tail, stream, should_persist=False): stderr_tail = S3Tail(stderr_location) child_jobs = [] - if self.cluster_size > 1: - for node in range(1, self.cluster_size): + if self.num_parallel > 1: + for node in range(1, self.num_parallel): child_job = copy.copy(self.job) child_job._id = child_job._id + "#{}".format(node) child_jobs.append(child_job) diff --git a/metaflow/plugins/aws/batch/batch_cli.py b/metaflow/plugins/aws/batch/batch_cli.py index 2d2dfcbfc55..c2309eb4b79 100644 --- a/metaflow/plugins/aws/batch/batch_cli.py +++ b/metaflow/plugins/aws/batch/batch_cli.py @@ -143,7 +143,7 @@ def kill(ctx, run_id, user, my_runs): # TODO: Maybe remove it altogether since it's not used here @click.option("--ubf-context", default=None, type=click.Choice([None, "ubf_control"])) @click.option("--host-volumes", multiple=True) -@click.option("--cluster-size", type=int) +@click.option("--num-parallel", type=int) @click.pass_context def step( ctx, @@ -163,7 +163,7 @@ def step( max_swap=None, swappiness=None, host_volumes=None, - cluster_size=None, + num_parallel=None, **kwargs ): def echo(msg, stream="stderr", batch_id=None): @@ -192,8 +192,8 @@ def echo(msg, stream="stderr", batch_id=None): kwargs["input_paths"] = "".join("${%s}" % s for s in split_vars.keys()) step_args = " ".join(util.dict_to_cli_options(kwargs)) - cluster_size = cluster_size or 1 - if cluster_size and cluster_size > 1: + num_parallel = num_parallel or 1 + if num_parallel and num_parallel > 1: # For multinode, we need to add a placeholder that can be mutated by the caller step_args += " [multinode-args]" step_cli = u"{entrypoint} {top_args} step {step} {step_args}".format( @@ -288,7 +288,7 @@ def _sync_metadata(): env=env, attrs=attrs, host_volumes=host_volumes, - cluster_size=cluster_size, + num_parallel=num_parallel, ) except Exception as e: traceback.print_exc() diff --git a/metaflow/plugins/aws/batch/batch_client.py b/metaflow/plugins/aws/batch/batch_client.py index a5ddd85ef99..5a948bd854b 100644 --- a/metaflow/plugins/aws/batch/batch_client.py +++ b/metaflow/plugins/aws/batch/batch_client.py @@ -97,10 +97,9 @@ def execute(self): ) # Multinode - if getattr(self, "cluster_size", 0) > 1: - num_nodes = self.cluster_size + if getattr(self, "num_parallel", 0) > 1: + num_nodes = self.num_parallel main_task_override = copy.deepcopy(self.payload["containerOverrides"]) - # TERRIBLE HACK JUST TO TRY THIS WORKS # main commands = self.payload["containerOverrides"]["command"][-1] @@ -153,7 +152,7 @@ def _register_job_definition( max_swap, swappiness, host_volumes, - cluster_size, + num_parallel, ): # identify platform from any compute environment associated with the # queue @@ -192,7 +191,7 @@ def _register_job_definition( } if platform == "FARGATE" or platform == "FARGATE_SPOT": - if cluster_size > 1: + if num_parallel > 1: raise BatchJobException("Fargate does not support multinode jobs.") if execution_role is None: raise BatchJobException( @@ -263,11 +262,11 @@ def _register_job_definition( {"sourceVolume": name, "containerPath": host_path} ) - self.cluster_size = cluster_size or 1 - if self.cluster_size > 1: + self.num_parallel = num_parallel or 1 + if self.num_parallel > 1: job_definition["type"] = "multinode" job_definition["nodeProperties"] = { - "numNodes": self.cluster_size, + "numNodes": self.num_parallel, "mainNode": 0, } job_definition["nodeProperties"]["nodeRangeProperties"] = [ @@ -278,7 +277,7 @@ def _register_job_definition( "container": job_definition["containerProperties"], }, { - "targetNodes": "1:{}".format(self.cluster_size - 1), + "targetNodes": "1:{}".format(self.num_parallel - 1), "container": job_definition["containerProperties"], }, ] @@ -321,7 +320,7 @@ def job_def( max_swap, swappiness, host_volumes, - cluster_size, + num_parallel, ): self.payload["jobDefinition"] = self._register_job_definition( image, @@ -332,7 +331,7 @@ def job_def( max_swap, swappiness, host_volumes, - cluster_size, + num_parallel, ) return self diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index 25176e237f8..9c014919b10 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -251,14 +251,14 @@ def task_pre_step( self._save_logs_sidecar = SidecarSubProcess("save_logs_periodically") - cluster_size = int(os.environ.get("AWS_BATCH_JOB_NUM_NODES", 1)) - if cluster_size > 1 and ubf_context == UBF_CONTROL: + num_parallel = int(os.environ.get("AWS_BATCH_JOB_NUM_NODES", 1)) + if num_parallel > 1 and ubf_context == UBF_CONTROL: # UBF handling for multinode case control_task_id = current.task_id top_task_id = control_task_id.replace("control-", "") # chop "-0" mapper_task_ids = [control_task_id] + [ "%s-node-%d" % (top_task_id, node_idx) - for node_idx in range(1, cluster_size) + for node_idx in range(1, num_parallel) ] flow._control_mapper_tasks = [ "%s/%s/%s" % (run_id, step_name, mapper_task_id) @@ -266,7 +266,7 @@ def task_pre_step( ] flow._control_task_is_mapper_zero = True - if cluster_size > 1: + if num_parallel > 1: _setup_multinode_environment() def task_post_step( @@ -339,12 +339,11 @@ def _wait_for_mapper_tasks(self, flow, step_name): return True else: print( - "Not sufficient number of tasks:", + "Waiting for all parallel tasks to finish. Finished: {}/{}".format( len(tasks), len(flow._control_mapper_tasks), - ) + )) except Exception as e: - print(e) pass raise Exception( "Batch secondary workers did not finish in %s seconds" % TIMEOUT diff --git a/metaflow/plugins/parallel_decorator.py b/metaflow/plugins/parallel_decorator.py new file mode 100644 index 00000000000..ce0b0f6fd62 --- /dev/null +++ b/metaflow/plugins/parallel_decorator.py @@ -0,0 +1,17 @@ +from metaflow.decorators import StepDecorator +from metaflow.unbounded_foreach import UBF_CONTROL + +class ParallelDecorator(StepDecorator): + name = "parallel" + defaults = {} + + def __init__(self, attributes=None, statically_defined=False): + super(ParallelDecorator, self).__init__(attributes, statically_defined) + + def runtime_step_cli( + self, cli_args, retry_count, max_user_code_retries, ubf_context + ): + + if ubf_context == UBF_CONTROL: + num_parallel = cli_args.task.ubf_iter.num_parallel + cli_args.command_options["num-parallel"] = str(num_parallel) diff --git a/metaflow/runtime.py b/metaflow/runtime.py index 6ceacad5cfe..d6da85dd0f0 100644 --- a/metaflow/runtime.py +++ b/metaflow/runtime.py @@ -621,8 +621,6 @@ def __init__( # easily. There is anyway a corresponding int id stored in the # metadata backend - so this should be fine. task_id = "control-%s-%s-%s" % (run, input_step, input_task) - else: - self.multinode_iterator = None # Register only regular Metaflow (non control) tasks. if task_id is None: task_id = str(metadata.new_task_id(run_id, step)) From 32ad057d5d505b6337b18074ec5b8c6b9f70aded Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Mon, 15 Nov 2021 12:42:45 +0200 Subject: [PATCH 104/176] lint parallel steps --- metaflow/decorators.py | 3 -- metaflow/flowspec.py | 4 ++- metaflow/graph.py | 9 +++--- metaflow/lint.py | 29 +++++++++++++++++++ metaflow/plugins/aws/batch/batch_decorator.py | 7 +++-- metaflow/plugins/parallel_decorator.py | 2 ++ 6 files changed, 43 insertions(+), 11 deletions(-) diff --git a/metaflow/decorators.py b/metaflow/decorators.py index 8f1a97307f7..241124db3b8 100644 --- a/metaflow/decorators.py +++ b/metaflow/decorators.py @@ -343,8 +343,6 @@ def task_finished( pass - - def _base_flow_decorator(decofunc, *args, **kwargs): """ Decorator prototype for all flow (class) decorators. This function gets @@ -500,4 +498,3 @@ def _import_plugin_decorators(globals_dict): # add flow-level decorators for decotype in FLOW_DECORATORS: globals_dict[decotype.name] = partial(_base_flow_decorator, decotype) - diff --git a/metaflow/flowspec.py b/metaflow/flowspec.py index ab4cac77188..5d9068534bc 100644 --- a/metaflow/flowspec.py +++ b/metaflow/flowspec.py @@ -480,7 +480,9 @@ def next(self, *dsts, **kwargs): if num_parallel: if len(dsts) > 1: - raise InvalidNextException("Only one destination allowed when num_parallel used in self.next()") + raise InvalidNextException( + "Only one destination allowed when num_parallel used in self.next()" + ) foreach = "_parallel_ubf_iter" self._parallel_ubf_iter = ParallelUBF(num_parallel) diff --git a/metaflow/graph.py b/metaflow/graph.py index 2af2f9d98b2..fd7d2e2f1d9 100644 --- a/metaflow/graph.py +++ b/metaflow/graph.py @@ -47,6 +47,7 @@ def __init__(self, func_ast, decos, doc): self.func_lineno = func_ast.lineno self.decorators = decos self.doc = deindent_docstring(doc) + self.parallel_step = any(getattr(deco, "IS_PARALLEL", False) for deco in decos) # these attributes are populated by _parse self.tail_next_lineno = 0 @@ -57,14 +58,13 @@ def __init__(self, func_ast, decos, doc): self.num_args = 0 self.condition = None self.foreach_param = None + self.parallel_foreach = False self._parse(func_ast) # these attributes are populated by _traverse_graph self.in_funcs = set() self.split_parents = [] self.matching_join = None - # self.is_multinode_step = any(isinstance(deco, MultinodeDecorator) for deco in decos) - # these attributes are populated by _postprocess self.is_inside_foreach = False @@ -98,7 +98,6 @@ def _parse(self, func_ast): keywords = dict( (k.arg, getattr(k.value, "s", None)) for k in tail.value.keywords ) - if len(keywords) == 1: if "foreach" in keywords: # TYPE: foreach @@ -108,6 +107,7 @@ def _parse(self, func_ast): self.invalid_tail_next = False elif "num_parallel" in keywords: self.type = "foreach" + self.parallel_foreach = True if len(self.out_funcs) == 1: self.invalid_tail_next = False elif "condition" in keywords: @@ -128,7 +128,6 @@ def _parse(self, func_ast): else: self.type = "linear" self.invalid_tail_next = False - except AttributeError: return @@ -145,6 +144,8 @@ def __str__(self): invalid_tail_next={0.invalid_tail_next} condition={0.condition} foreach_param={0.foreach_param} + parallel_step={0.parallel_step} + parallel_foreach={0.parallel_foreach} -> {out}""".format( self, matching_join=self.matching_join and "[%s]" % self.matching_join, diff --git a/metaflow/lint.py b/metaflow/lint.py index 445da20a34d..e7cc79514e6 100644 --- a/metaflow/lint.py +++ b/metaflow/lint.py @@ -280,6 +280,35 @@ def check_empty_foreaches(graph): raise LintWarn(msg.format(node, join=joins[0])) +@linter.ensure_static_graph +@linter.check +def check_parallel_step_after_next(graph): + msg = ( + "Step *{0.name}* is called as a parallel step with self.next(num_parallel=..) " + "but does not have a @parallel decorator." + ) + for node in graph: + if node.parallel_foreach and not all( + graph[out_node].parallel_step for out_node in node.out_funcs + ): + raise LintWarn(msg.format(node)) + + +@linter.ensure_static_graph +@linter.check +def check_parallel_foreach_calls_parallel_step(graph): + msg = ( + "Step *{0.name}* has a @parallel decorator, but is not called " + " with self.next(num_parallel=...) from step *{1.name}*." + ) + for node in graph: + if node.parallel_step: + for node2 in graph: + if node2.out_funcs and node.name in node2.out_funcs: + if not node2.parallel_foreach: + raise LintWarn(msg.format(node, node2)) + + @linter.ensure_non_nested_foreach @linter.check def check_nested_foreach(graph): diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index 9c014919b10..00007dd5311 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -340,9 +340,10 @@ def _wait_for_mapper_tasks(self, flow, step_name): else: print( "Waiting for all parallel tasks to finish. Finished: {}/{}".format( - len(tasks), - len(flow._control_mapper_tasks), - )) + len(tasks), + len(flow._control_mapper_tasks), + ) + ) except Exception as e: pass raise Exception( diff --git a/metaflow/plugins/parallel_decorator.py b/metaflow/plugins/parallel_decorator.py index ce0b0f6fd62..4922b57b2b8 100644 --- a/metaflow/plugins/parallel_decorator.py +++ b/metaflow/plugins/parallel_decorator.py @@ -1,9 +1,11 @@ from metaflow.decorators import StepDecorator from metaflow.unbounded_foreach import UBF_CONTROL + class ParallelDecorator(StepDecorator): name = "parallel" defaults = {} + IS_PARALLEL = True def __init__(self, attributes=None, statically_defined=False): super(ParallelDecorator, self).__init__(attributes, statically_defined) From b206c135002e9713dc5a9e2ba40aabc6a6c4f99b Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Mon, 15 Nov 2021 13:06:36 +0200 Subject: [PATCH 105/176] Fail-fast for AWS Step functions and Kubernetes with parallel nodes. --- metaflow/plugins/aws/eks/kubernetes_decorator.py | 5 +++++ metaflow/plugins/aws/step_functions/step_functions.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/metaflow/plugins/aws/eks/kubernetes_decorator.py b/metaflow/plugins/aws/eks/kubernetes_decorator.py index 9f28ff60e98..a4d26390604 100644 --- a/metaflow/plugins/aws/eks/kubernetes_decorator.py +++ b/metaflow/plugins/aws/eks/kubernetes_decorator.py @@ -130,6 +130,11 @@ def step_init(self, flow, graph, step, decos, environment, flow_datastore, logge if not (my_val is None and v is None): self.attributes[k] = str(max(int(my_val or 0), int(v or 0))) + if getattr(deco, "IS_PARALLEL", False): + raise KubernetesException( + "Kubernetes decorator does not support parallel execution yet." + ) + # Set run time limit for the Kubernetes job. self.run_time_limit = get_run_time_limit_for_task(decos) if self.run_time_limit < 60: diff --git a/metaflow/plugins/aws/step_functions/step_functions.py b/metaflow/plugins/aws/step_functions/step_functions.py index f71b560fdbc..007de027c1b 100644 --- a/metaflow/plugins/aws/step_functions/step_functions.py +++ b/metaflow/plugins/aws/step_functions/step_functions.py @@ -437,6 +437,11 @@ def _batch(self, node): # JsonPath-foo embedded in the parent states, we have this # information easily available. + if node.parallel_foreach: + raise StepFunctionsException( + "Parallel steps are not supported yet with AWS step functions." + ) + # Handle foreach join. if ( node.type == "join" From 0f78f89c6bbbb516beb7a57df65791d2b7a56d10 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Mon, 15 Nov 2021 14:22:28 +0200 Subject: [PATCH 106/176] current parallel status --- metaflow/current.py | 14 ++++++++++++++ metaflow/plugins/aws/batch/batch_decorator.py | 12 ++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/metaflow/current.py b/metaflow/current.py index e5092fbd4e1..66245ccec0f 100644 --- a/metaflow/current.py +++ b/metaflow/current.py @@ -1,3 +1,9 @@ +from collections import namedtuple +import os + +Parallel = namedtuple("Parallel", ["main_ip", "num_nodes", "node_index"]) + + class Current(object): def __init__(self): self._flow_name = None @@ -83,6 +89,14 @@ def namespace(self): def username(self): return self._username + @property + def parallel(self): + return Parallel( + main_ip=os.environ.get("MF_PARALLEL_MAIN_IP", "127.0.0.1"), + num_nodes=int(os.environ.get("MF_PARALLEL_NUM_NODES", "1")), + node_index=int(os.environ.get("MF_PARALLEL_NODE_INDEX", "0")), + ) + # instantiate the Current singleton. This will be populated # by task.MetaflowTask before a task is executed. diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index 00007dd5311..aa335282401 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -366,14 +366,10 @@ def _setup_multinode_environment(): # we are the main node local_ips = socket.gethostbyname_ex(socket.gethostname())[-1] assert local_ips, "Could not find local ip address" - os.environ["MF_MULTINODE_MAIN_IP"] = local_ips[0] + os.environ["MF_PARALLEL_MAIN_IP"] = local_ips[0] else: - os.environ["MF_MULTINODE_MAIN_IP"] = os.environ[ + os.environ["MF_PARALLEL_MAIN_IP"] = os.environ[ "AWS_BATCH_JOB_MAIN_NODE_PRIVATE_IPV4_ADDRESS" ] - os.environ["MF_MULTINODE_NUM_NODES"] = os.environ["AWS_BATCH_JOB_NUM_NODES"] - os.environ["MF_MULTINODE_NODE_INDEX"] = os.environ["AWS_BATCH_JOB_NODE_INDEX"] - print( - "Multinode environment:", - {k: v for k, v in os.environ.items() if k.startswith("MF_MULTINODE")}, - ) + os.environ["MF_PARALLEL_NUM_NODES"] = os.environ["AWS_BATCH_JOB_NUM_NODES"] + os.environ["MF_PARALLEL_NODE_INDEX"] = os.environ["AWS_BATCH_JOB_NODE_INDEX"] From 8009bc2a3a28870ea1818c62f03a9c8510f86d24 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Mon, 15 Nov 2021 21:09:15 +0200 Subject: [PATCH 107/176] fix num tasks in waiting, enable conda for controls --- metaflow/plugins/aws/batch/batch_decorator.py | 2 +- metaflow/plugins/conda/conda_step_decorator.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index aa335282401..18d3ebed079 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -332,7 +332,7 @@ def _wait_for_mapper_tasks(self, flow, step_name): try: step_path = "%s/%s/%s" % (flow.name, current.run_id, step_name) tasks = [task for task in Step(step_path)] - if len(tasks) == len(flow._control_mapper_tasks) - 1: + if len(tasks) == len(flow._control_mapper_tasks): if all( task.finished_at is not None for task in tasks ): # for some reason task.finished fails diff --git a/metaflow/plugins/conda/conda_step_decorator.py b/metaflow/plugins/conda/conda_step_decorator.py index fd5b71b59c3..e29682f4389 100644 --- a/metaflow/plugins/conda/conda_step_decorator.py +++ b/metaflow/plugins/conda/conda_step_decorator.py @@ -81,9 +81,6 @@ def _python_version(self): ) def is_enabled(self, ubf_context=None): - if ubf_context == UBF_CONTROL: - # Disable `@conda` for ubf_control tasks. - return False return not next( x for x in [ From 0f09a1dd736b30e433a65362290690deeb0d679d Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Mon, 15 Nov 2021 12:10:06 -0800 Subject: [PATCH 108/176] read default k8s namespace from config (#823) --- metaflow/metaflow_config.py | 2 +- metaflow/plugins/aws/eks/kubernetes_client.py | 3 --- metaflow/plugins/aws/eks/kubernetes_decorator.py | 3 +++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metaflow/metaflow_config.py b/metaflow/metaflow_config.py index 2dabb3126cd..f434884bac6 100644 --- a/metaflow/metaflow_config.py +++ b/metaflow/metaflow_config.py @@ -173,7 +173,7 @@ def from_conf(name, default=None): # Kubernetes configuration ### # Kubernetes namespace to use for all objects created by Metaflow -KUBERNETES_NAMESPACE = from_conf("METAFLOW_KUBERNETES_NAMESPACE") +KUBERNETES_NAMESPACE = from_conf("METAFLOW_KUBERNETES_NAMESPACE", "default") # Service account to use by K8S jobs created by Metaflow KUBERNETES_SERVICE_ACCOUNT = from_conf("METAFLOW_KUBERNETES_SERVICE_ACCOUNT") # Default container image for K8S diff --git a/metaflow/plugins/aws/eks/kubernetes_client.py b/metaflow/plugins/aws/eks/kubernetes_client.py index 1e3214a335a..ce89d7c5be4 100644 --- a/metaflow/plugins/aws/eks/kubernetes_client.py +++ b/metaflow/plugins/aws/eks/kubernetes_client.py @@ -107,9 +107,6 @@ def __init__(self, client_wrapper, **kwargs): self._client_wrapper = client_wrapper self._kwargs = kwargs - # Kubernetes namespace defaults to `default` - self._kwargs["namespace"] = self._kwargs["namespace"] or "default" - def create(self): # Check that job attributes are sensible. diff --git a/metaflow/plugins/aws/eks/kubernetes_decorator.py b/metaflow/plugins/aws/eks/kubernetes_decorator.py index 60e2371f5aa..55bbe93f246 100644 --- a/metaflow/plugins/aws/eks/kubernetes_decorator.py +++ b/metaflow/plugins/aws/eks/kubernetes_decorator.py @@ -11,6 +11,7 @@ KUBERNETES_CONTAINER_IMAGE, KUBERNETES_CONTAINER_REGISTRY, DATASTORE_LOCAL_DIR, + KUBERNETES_NAMESPACE, ) from metaflow.plugins import ResourcesDecorator from metaflow.plugins.timeout_decorator import get_run_time_limit_for_task @@ -78,6 +79,8 @@ def my_step(self): def __init__(self, attributes=None, statically_defined=False): super(KubernetesDecorator, self).__init__(attributes, statically_defined) + if not self.attributes["namespace"]: + self.attributes["namespace"] = KUBERNETES_NAMESPACE # TODO: Unify the logic with AWS Batch # If no docker image is explicitly specified, impute a default image. if not self.attributes["image"]: From cb2e2eabe7f961c3266d2c3fef9683c524dd4c2f Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Wed, 17 Nov 2021 12:46:39 +0200 Subject: [PATCH 109/176] unit test for basic parallel --- metaflow/plugins/parallel_decorator.py | 8 ------ test/core/contexts.json | 30 ++++++++++++++++++++ test/core/graphs/parallel.json | 21 ++++++++++++++ test/core/metaflow_test/formatter.py | 5 ++++ test/core/tests/basic_parallel.py | 38 ++++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 8 deletions(-) create mode 100644 test/core/graphs/parallel.json create mode 100644 test/core/tests/basic_parallel.py diff --git a/metaflow/plugins/parallel_decorator.py b/metaflow/plugins/parallel_decorator.py index 48f72d30279..a135ac69a96 100644 --- a/metaflow/plugins/parallel_decorator.py +++ b/metaflow/plugins/parallel_decorator.py @@ -101,14 +101,6 @@ def _local_multinode_control_task_step_func(flow, env_to_use, step_func, retry_c kwargs["retry_count"] = str(retry_count) cmd = cli_args.step_command(executable, script, step_name, step_kwargs=kwargs) - step_cli = u" ".join(cmd) - # Print cmdline for execution. Doesn't work without the temporary - # unicode object while using `print`. - print( - u"[${cwd}] Starting split#{split} with cmd:{cmd}".format( - cwd=os.getcwd(), split=node_index, cmd=step_cli - ) - ) p = subprocess.Popen(cmd) subprocesses.append(p) diff --git a/test/core/contexts.json b/test/core/contexts.json index c4bf3d98cee..23823c32deb 100644 --- a/test/core/contexts.json +++ b/test/core/contexts.json @@ -31,6 +31,36 @@ "S3FailureTest" ] }, + { + "name": "python3-only", + "disabled": false, + "env": { + "METAFLOW_USER": "tester", + "METAFLOW_RUN_BOOL_PARAM": "False", + "METAFLOW_RUN_NO_DEFAULT_PARAM": "test_str", + "METAFLOW_DEFAULT_METADATA": "local" + }, + "python": "python3", + "top_options": [ + "--metadata=local", + "--datastore=local", + "--environment=local", + "--event-logger=nullSidecarLogger", + "--no-pylint", + "--quiet" + ], + "run_options": [ + "--max-workers", "50", + "--max-num-splits", "10000", + "--tag", "\u523a\u8eab means sashimi", + "--tag", "multiple tags should be ok" + ], + "checks": ["python3-cli", "python3-metadata"], + "disabled_tests": [ + "LargeArtifactTest", + "S3FailureTest" + ] + }, { "name": "python3-all-local", "disabled": false, diff --git a/test/core/graphs/parallel.json b/test/core/graphs/parallel.json new file mode 100644 index 00000000000..b4d2545683f --- /dev/null +++ b/test/core/graphs/parallel.json @@ -0,0 +1,21 @@ +{ + "name": "small-parallel", + "graph": { + "start": {"linear": "parallel_split"}, + "parallel_split": { + "num_parallel": 4, + "parallel": "parallel_inner", + "quals": ["parallel-split"] + }, + "parallel_inner": { + "linear": "parallel_join", + "quals": ["parallel-inner"] + }, + "parallel_join": { + "linear": "end", + "join": true, + "quals": ["parallel-join"] + }, + "end": {} + } +} diff --git a/test/core/metaflow_test/formatter.py b/test/core/metaflow_test/formatter.py index e421f8ed59a..09eaa0d12b7 100644 --- a/test/core/metaflow_test/formatter.py +++ b/test/core/metaflow_test/formatter.py @@ -131,6 +131,11 @@ def _flow_lines(self): node["foreach"], node["foreach_var"], ) + elif "num_parallel" in node: + yield 2, "self.next(self.%s, num_parallel=%d)" % ( + node["parallel"], + node["num_parallel"], + ) yield 0, "if __name__ == '__main__':" yield 1, "%s()" % self.flow_name diff --git a/test/core/tests/basic_parallel.py b/test/core/tests/basic_parallel.py new file mode 100644 index 00000000000..a5474dea2e5 --- /dev/null +++ b/test/core/tests/basic_parallel.py @@ -0,0 +1,38 @@ +from metaflow_test import MetaflowTest, ExpectationFailed, steps, tag + + +class BasicParallelTest(MetaflowTest): + PRIORITY = 1 + + @steps(0, ["parallel-split"], required=True) + def split(self): + self.my_node_index = None + + @tag("parallel") + @steps(0, ["parallel-inner"], required=True) + def inner(self): + from metaflow import current + + assert_equals(4, current.parallel.num_nodes) + self.my_node_index = current.parallel.node_index + + @steps(0, ["parallel-join"], required=True) + def join(self, inputs): + got = sorted([inp.my_node_index for inp in inputs]) + assert_equals(list(range(4)), got) + + @steps(1, ["all"]) + def step_all(self): + pass + + def check_results(self, flow, checker): + run = checker.get_run() + if type(checker).__name__ == "CliCheck": + # CliCheck doesn't support enlisting of tasks. + assert run is None + else: + assert run is not None + tasks = run["parallel_inner"].tasks() + task_list = list(tasks) + assert_equals(4, len(task_list)) + assert_equals(1, len(list(run["parallel_inner"].control_tasks()))) From 584b7e1996fa1798a1cb9d6ede03ecf145d7ddc4 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Wed, 17 Nov 2021 18:14:41 +0200 Subject: [PATCH 110/176] Add 'last modified' to S3 object (#778) * last_modified to S3Object * run black --- metaflow/datatools/s3.py | 25 ++++++++++++++++++++++--- metaflow/datatools/s3op.py | 4 ++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/metaflow/datatools/s3.py b/metaflow/datatools/s3.py index e08c9dc679d..6710049ad5b 100644 --- a/metaflow/datatools/s3.py +++ b/metaflow/datatools/s3.py @@ -97,6 +97,7 @@ def __init__( content_type=None, metadata=None, range_info=None, + last_modified=None, ): # all fields of S3Object should return a unicode object @@ -107,6 +108,7 @@ def __init__( self._path = path self._key = None self._content_type = content_type + self._last_modified = last_modified self._metadata = None if metadata is not None and "metaflow-user-attributes" in metadata: @@ -237,6 +239,14 @@ def range_info(self): """ return self._range_info + @property + def last_modified(self): + """ + Returns the last modified unix timestamp of the object, or None + if not fetched. + """ + return self._last_modified + def __str__(self): if self._path: return "" % (self._url, self._size) @@ -486,6 +496,7 @@ def _info(s3, tmp): "content_type": resp["ContentType"], "metadata": resp["Metadata"], "size": resp["ContentLength"], + "last_modified": resp["LastModified"].timestamp(), } info_results = None @@ -504,6 +515,7 @@ def _info(s3, tmp): size=info_results["size"], content_type=info_results["content_type"], metadata=info_results["metadata"], + last_modified=info_results["last_modified"], ) return S3Object(self._s3root, url, None) @@ -547,7 +559,7 @@ def _head(): else: yield self._s3root, s3url, None, info["size"], info[ "content_type" - ], info["metadata"] + ], info["metadata"], None, info["last_modified"] else: # This should not happen; we should always get a response # even if it contains an error inside it @@ -593,6 +605,7 @@ def _download(s3, tmp): return { "content_type": resp["ContentType"], "metadata": resp["Metadata"], + "last_modified": resp["LastModified"].timestamp(), } return None @@ -611,6 +624,7 @@ def _download(s3, tmp): path, content_type=addl_info["content_type"], metadata=addl_info["metadata"], + last_modified=addl_info["last_modified"], ) return S3Object(self._s3root, url, path) @@ -652,7 +666,9 @@ def _get(): info = json.load(f) yield self._s3root, s3url, os.path.join( self._tmpdir, fname - ), None, info["content_type"], info["metadata"] + ), None, info["content_type"], info["metadata"], None, info[ + "last_modified" + ] else: yield self._s3root, s3prefix, None else: @@ -694,7 +710,9 @@ def _get(): info = json.load(f) yield self._s3root, s3url, os.path.join( self._tmpdir, fname - ), None, info["content_type"], info["metadata"] + ), None, info["content_type"], info["metadata"], None, info[ + "last_modified" + ] else: yield s3prefix, s3url, os.path.join(self._tmpdir, fname) @@ -1023,6 +1041,7 @@ def _s3op_with_retries(self, mode, **options): raise MetaflowS3NotFound(err_out) elif ex.returncode == s3op.ERROR_URL_ACCESS_DENIED: raise MetaflowS3AccessDenied(err_out) + print("Error with S3 operation:", err_out) time.sleep(2 ** i + random.randint(0, 10)) return None, err_out diff --git a/metaflow/datatools/s3op.py b/metaflow/datatools/s3op.py index ec7cda21d6d..2df124be3a1 100644 --- a/metaflow/datatools/s3op.py +++ b/metaflow/datatools/s3op.py @@ -120,6 +120,7 @@ def op_info(url): "size": head["ContentLength"], "content_type": head["ContentType"], "metadata": head["Metadata"], + "last_modified": head["LastModified"].timestamp(), } except client_error as err: error_code = normalize_client_error(err) @@ -183,12 +184,15 @@ def op_info(url): # TODO specific error message for out of disk space # If we need the metadata, get it and write it out if pre_op_info: + with open("%s_meta" % url.local, mode="w") as f: args = {"size": resp["ContentLength"]} if resp["ContentType"]: args["content_type"] = resp["ContentType"] if resp["Metadata"] is not None: args["metadata"] = resp["Metadata"] + if resp["LastModified"]: + args["last_modified"] = resp["LastModified"].timestamp() json.dump(args, f) # Finally, we push out the size to the result_pipe since # the size is used for verification and other purposes and From d5e51e973098bc7579fa7a4c6b874a6c2459eb24 Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Mon, 22 Nov 2021 11:33:23 -0800 Subject: [PATCH 111/176] a couple of small issues in s3 error handling (#821) --- metaflow/datatools/s3op.py | 4 ++-- metaflow/datatools/s3tail.py | 5 +++++ metaflow/mflog/__init__.py | 3 +-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/metaflow/datatools/s3op.py b/metaflow/datatools/s3op.py index 2df124be3a1..bc72394e3c9 100644 --- a/metaflow/datatools/s3op.py +++ b/metaflow/datatools/s3op.py @@ -89,7 +89,7 @@ def normalize_client_error(err): try: return int(error_code) except ValueError: - if error_code == "AccessDenied": + if error_code in ("AccessDenied", "AllAccessDisabled"): return 403 if error_code == "NoSuchKey": return 404 @@ -400,7 +400,7 @@ def list_prefix(self, prefix_url, delimiter=""): except self.s3.exceptions.NoSuchBucket: return False, prefix_url, ERROR_URL_NOT_FOUND except self.client_error as err: - if err.response["Error"]["Code"] == "AccessDenied": + if err.response["Error"]["Code"] in ("AccessDenied", "AllAccessDisabled"): return False, prefix_url, ERROR_URL_ACCESS_DENIED else: raise diff --git a/metaflow/datatools/s3tail.py b/metaflow/datatools/s3tail.py index aec7cd6db6d..26e98f38480 100644 --- a/metaflow/datatools/s3tail.py +++ b/metaflow/datatools/s3tail.py @@ -18,6 +18,11 @@ def __init__(self, s3url): self._pos = 0 self._tail = b"" + def reset_client(self, hard_reset=False): + # This method is required by @aws_retry + if hard_reset or self.s3 is None: + self.s3, self.ClientError = get_s3_client() + def clone(self, s3url): tail = S3Tail(s3url) tail._pos = self._pos diff --git a/metaflow/mflog/__init__.py b/metaflow/mflog/__init__.py index f6fd4688785..691c1568ea8 100644 --- a/metaflow/mflog/__init__.py +++ b/metaflow/mflog/__init__.py @@ -119,8 +119,7 @@ def _available_logs(tail, stream, echo, should_persist=False): echo(line.strip().decode("utf-8", errors="replace"), stream) except Exception as ex: echo( - "%s[ temporary error in fetching logs: %s ]" % to_unicode(prefix), - ex, + "%s[ temporary error in fetching logs: %s ]" % (to_unicode(prefix), ex), "stderr", ) From 931c0c7a13b489bb06e3047c1c7eb68301e4b8fd Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Mon, 29 Nov 2021 15:57:46 -0800 Subject: [PATCH 112/176] bump version to 2.4.4 (#843) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fdf93968226..aae36f718bd 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = "2.4.3" +version = "2.4.4" setup( name="metaflow", From 5297918493d8f4c0ba0c6638fc74d3abd9f8b43f Mon Sep 17 00:00:00 2001 From: David Neuzerling Date: Tue, 30 Nov 2021 20:58:18 +1100 Subject: [PATCH 113/176] Allow CLI args that are falsy but not False or None (#828) * Allow CLI args that are falsy but not false or None * Copy _options changes to cli_args.py * Format with black --- metaflow/cli_args.py | 27 ++++++++++++++++----------- metaflow/runtime.py | 27 ++++++++++++++++----------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/metaflow/cli_args.py b/metaflow/cli_args.py index c378de7c1cf..680fd1bdb3d 100644 --- a/metaflow/cli_args.py +++ b/metaflow/cli_args.py @@ -55,17 +55,22 @@ def step_command( @staticmethod def _options(mapping): for k, v in mapping.items(): - if v: - # we need special handling for 'with' since it is a reserved - # keyword in Python, so we call it 'decospecs' in click args - if k == "decospecs": - k = "with" - k = k.replace("_", "-") - v = v if isinstance(v, (list, tuple, set)) else [v] - for value in v: - yield "--%s" % k - if not isinstance(value, bool): - yield to_unicode(value) + + # None or False arguments are ignored + # v needs to be explicitly False, not falsy, eg. 0 is an acceptable value + if v is None or v is False: + continue + + # we need special handling for 'with' since it is a reserved + # keyword in Python, so we call it 'decospecs' in click args + if k == "decospecs": + k = "with" + k = k.replace("_", "-") + v = v if isinstance(v, (list, tuple, set)) else [v] + for value in v: + yield "--%s" % k + if not isinstance(value, bool): + yield to_unicode(value) cli_args = CLIArgs() diff --git a/metaflow/runtime.py b/metaflow/runtime.py index fad0746f2af..f6786603fd0 100644 --- a/metaflow/runtime.py +++ b/metaflow/runtime.py @@ -910,17 +910,22 @@ def get_args(self): # TODO: Make one with dict_to_cli_options; see cli_args.py for more detail def _options(mapping): for k, v in mapping.items(): - if v: - # we need special handling for 'with' since it is a reserved - # keyword in Python, so we call it 'decospecs' in click args - if k == "decospecs": - k = "with" - k = k.replace("_", "-") - v = v if isinstance(v, (list, tuple, set)) else [v] - for value in v: - yield "--%s" % k - if not isinstance(value, bool): - yield to_unicode(value) + + # None or False arguments are ignored + # v needs to be explicitly False, not falsy, eg. 0 is an acceptable value + if v is None or v is False: + continue + + # we need special handling for 'with' since it is a reserved + # keyword in Python, so we call it 'decospecs' in click args + if k == "decospecs": + k = "with" + k = k.replace("_", "-") + v = v if isinstance(v, (list, tuple, set)) else [v] + for value in v: + yield "--%s" % k + if not isinstance(value, bool): + yield to_unicode(value) args = list(self.entrypoint) args.extend(_options(self.top_level_options)) From f17bd79985d88806fa33fcf3ab4d6d851c9be1e7 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Mon, 29 Nov 2021 11:31:30 -0800 Subject: [PATCH 114/176] Fix control task handling for catch decorator. --- metaflow/flowspec.py | 2 +- metaflow/lint.py | 2 -- metaflow/plugins/catch_decorator.py | 14 +++++++++----- test/core/graphs/parallel.json | 3 ++- test/core/metaflow_test/cli_check.py | 3 +++ test/core/metaflow_test/formatter.py | 10 ++++++++-- test/core/metaflow_test/metadata_check.py | 5 +++++ test/core/tests/basic_parallel.py | 5 ++--- test/core/tests/catch_retry.py | 2 +- test/core/tests/tag_catch.py | 17 +++++++++++++---- 10 files changed, 44 insertions(+), 19 deletions(-) diff --git a/metaflow/flowspec.py b/metaflow/flowspec.py index 823826e2335..9a920e83117 100644 --- a/metaflow/flowspec.py +++ b/metaflow/flowspec.py @@ -517,7 +517,7 @@ def next(self, *dsts, **kwargs): raise InvalidNextException(msg) funcs.append(name) - if num_parallel: + if num_parallel is not None and num_parallel >= 1: if len(dsts) > 1: raise InvalidNextException( "Only one destination allowed when num_parallel used in self.next()" diff --git a/metaflow/lint.py b/metaflow/lint.py index e7cc79514e6..4eb52346e18 100644 --- a/metaflow/lint.py +++ b/metaflow/lint.py @@ -31,8 +31,6 @@ def ensure_acyclicity(self, f): def ensure_non_nested_foreach(self, f): return self._decorate("require_non_nested_foreach", f) - # TODO: num_parallel matches with a multinode - def check(self, f): self._checks.append(f) f.attrs = [] diff --git a/metaflow/plugins/catch_decorator.py b/metaflow/plugins/catch_decorator.py index 7597fabc28e..c59d8b27c56 100644 --- a/metaflow/plugins/catch_decorator.py +++ b/metaflow/plugins/catch_decorator.py @@ -2,7 +2,7 @@ from metaflow.exception import MetaflowException, MetaflowExceptionWrapper from metaflow.decorators import StepDecorator -from metaflow.unbounded_foreach import UBF_CONTROL +from metaflow import current NUM_FALLBACK_RETRIES = 3 @@ -84,6 +84,12 @@ def task_exception( # pretend that self.next() was called as usual flow._transition = (graph[step].out_funcs, None, None) + + # If this task is a UBF control task, it will return itself as the singleton + # list of tasks. + flow._control_mapper_tasks = [ + "/".join((current.run_id, current.step_name, current.task_id)) + ] # store the exception picklable = MetaflowExceptionWrapper(exception) flow._catch_exception = picklable @@ -105,14 +111,12 @@ def task_decorate( # if the user code has failed max_user_code_retries times, @catch # runs a piece of fallback code instead. This way we can continue - # running the flow downsteam, as we have a proper entry for this task. + # running the flow downstream, as we have a proper entry for this task. def fallback_step(inputs=None): raise FailureHandledByCatch(retry_count) - # We don't run fallback for `ubf_control` since it doesn't support - # any retries. - if ubf_context != UBF_CONTROL and retry_count > max_user_code_retries: + if retry_count > max_user_code_retries: return fallback_step else: return step_func diff --git a/test/core/graphs/parallel.json b/test/core/graphs/parallel.json index b4d2545683f..3863bb72162 100644 --- a/test/core/graphs/parallel.json +++ b/test/core/graphs/parallel.json @@ -9,7 +9,8 @@ }, "parallel_inner": { "linear": "parallel_join", - "quals": ["parallel-inner"] + "quals": ["parallel-step"], + "parallel_step": true }, "parallel_join": { "linear": "end", diff --git a/test/core/metaflow_test/cli_check.py b/test/core/metaflow_test/cli_check.py index 078a5d47f51..2e6160a9d78 100644 --- a/test/core/metaflow_test/cli_check.py +++ b/test/core/metaflow_test/cli_check.py @@ -89,6 +89,9 @@ def artifact_dict(self, step, name): # if the step had multiple tasks, this will fail return pickle.load(f) + def artifact_dict_if_exists(self, step, name): + return self.artifact_dict(step, name) + def assert_log(self, step, logtype, value, exact_match=True): log = self.get_log(step, logtype) if (exact_match and log != value) or (not exact_match and value not in log): diff --git a/test/core/metaflow_test/formatter.py b/test/core/metaflow_test/formatter.py index 09eaa0d12b7..69d948d6941 100644 --- a/test/core/metaflow_test/formatter.py +++ b/test/core/metaflow_test/formatter.py @@ -16,8 +16,8 @@ def __init__(self, graphspec, test): self.steps = self._index_steps(test) self.flow_code = self._pretty_print(self._flow_lines()) self.check_code = self._pretty_print(self._check_lines()) - self.valid = True + for step in self.steps: if step.required and not step in self.used: self.valid = False @@ -57,6 +57,8 @@ def _node_quals(self, name, node): quals.add(name) if "join" in node: quals.add("join") + if "parallel_step" in node: + quals.add("parallel-step") if "linear" in node: quals.add("linear") for qual in node.get("quals", []): @@ -80,7 +82,7 @@ def _flow_lines(self): tags.extend(tag.split("(")[0] for tag in step.tags) yield 0, "# -*- coding: utf-8 -*-" - yield 0, "from metaflow import FlowSpec, step, Parameter, project, IncludeFile, JSONType, current" + yield 0, "from metaflow import FlowSpec, step, Parameter, project, IncludeFile, JSONType, current, parallel" yield 0, "from metaflow_test import assert_equals, " "assert_exception, " "ExpectationFailed, " "is_resumed, " "ResumeFromHere, " "TestRetry" if tags: yield 0, "from metaflow import %s" % ",".join(tags) @@ -105,6 +107,10 @@ def _flow_lines(self): for tagspec in step.tags: yield 1, "@%s" % tagspec + + if "parallel_step" in node: + yield 1, "@parallel" + yield 1, "@step" if "join" in node: diff --git a/test/core/metaflow_test/metadata_check.py b/test/core/metaflow_test/metadata_check.py index f7c82163027..86a3eb2aa66 100644 --- a/test/core/metaflow_test/metadata_check.py +++ b/test/core/metaflow_test/metadata_check.py @@ -90,6 +90,11 @@ def assert_artifact(self, step, name, value, fields=None): def artifact_dict(self, step, name): return {task.id: {name: task[name].data} for task in self.run[step]} + def artifact_dict_if_exists(self, step, name): + return { + task.id: {name: task[name].data} for task in self.run[step] if name in task + } + def assert_log(self, step, logtype, value, exact_match=True): log_value = self.get_log(step, logtype) if log_value == value: diff --git a/test/core/tests/basic_parallel.py b/test/core/tests/basic_parallel.py index a5474dea2e5..5b7a2ab4610 100644 --- a/test/core/tests/basic_parallel.py +++ b/test/core/tests/basic_parallel.py @@ -8,15 +8,14 @@ class BasicParallelTest(MetaflowTest): def split(self): self.my_node_index = None - @tag("parallel") - @steps(0, ["parallel-inner"], required=True) + @steps(0, ["parallel-step"], required=True) def inner(self): from metaflow import current assert_equals(4, current.parallel.num_nodes) self.my_node_index = current.parallel.node_index - @steps(0, ["parallel-join"], required=True) + @steps(0, ["join"], required=True) def join(self, inputs): got = sorted([inp.my_node_index for inp in inputs]) assert_equals(list(range(4)), got) diff --git a/test/core/tests/catch_retry.py b/test/core/tests/catch_retry.py index 9a672e7374b..283390f4d68 100644 --- a/test/core/tests/catch_retry.py +++ b/test/core/tests/catch_retry.py @@ -19,7 +19,7 @@ def step_start(self): # foreach splits don't support @catch but @retry should work @tag("retry(times=2,minutes_between_retries=0)") - @steps(0, ["foreach-split"]) + @steps(0, ["foreach-split", "parallel-split"]) def step_split(self): import os diff --git a/test/core/tests/tag_catch.py b/test/core/tests/tag_catch.py index 7ff3972d5e3..7d30d2bb051 100644 --- a/test/core/tests/tag_catch.py +++ b/test/core/tests/tag_catch.py @@ -19,7 +19,7 @@ def step_start(self): # foreach splits don't support @catch but @retry should work @tag("retry(times=2)") - @steps(0, ["foreach-split"]) + @steps(0, ["foreach-split", "parallel-split"]) def step_split(self): import os @@ -30,7 +30,7 @@ def step_split(self): @tag("retry(times=2)") @steps(0, ["join"]) - def step_join(self): + def step_join(self, inputs): import os if current.retry_count == 2: @@ -95,7 +95,9 @@ def check_results(self, flow, checker): checker.assert_artifact("end", "test_attempt", 3) else: - for task in checker.artifact_dict(step.name, "ex").values(): + # Use artifact_dict_if_exists because for parallel tasks, only the + # control task will have the 'ex' artifact. + for task in checker.artifact_dict_if_exists(step.name, "ex").values(): extype = "metaflow.plugins.catch_decorator." "FailureHandledByCatch" assert_equals(extype, str(task["ex"].type)) break @@ -116,7 +118,14 @@ def check_results(self, flow, checker): for task in step: data = task.data got = sorted(m.value for m in task.metadata if m.type == "attempt") - assert_equals(list(map(str, range(attempts))), got) + if flow._graph[step.id].parallel_step: + if "control" in task.id: + assert_equals(list(map(str, range(attempts))), got) + else: + # non-control tasks have one attempt less for parallel steps + assert_equals(list(map(str, range(attempts - 1))), got) + else: + assert_equals(list(map(str, range(attempts))), got) assert_equals(False, "invisible" in run["start"].task.data) assert_equals(3, run["start"].task.data.test_attempt) From 0f2b23d5ca4c617a2df37e48695ddcaab179c39f Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Tue, 30 Nov 2021 14:55:35 -0800 Subject: [PATCH 115/176] use underscores for task ids --- metaflow/plugins/parallel_decorator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metaflow/plugins/parallel_decorator.py b/metaflow/plugins/parallel_decorator.py index a135ac69a96..bcd65492fd6 100644 --- a/metaflow/plugins/parallel_decorator.py +++ b/metaflow/plugins/parallel_decorator.py @@ -87,7 +87,7 @@ def _local_multinode_control_task_step_func(flow, env_to_use, step_func, retry_c # start workers subprocesses = [] for node_index in range(1, num_parallel): - task_id = "%s-node-%d" % (top_task_id, node_index) + task_id = "%s_node_%d" % (top_task_id, node_index) mapper_task_ids.append(task_id) os.environ["MF_PARALLEL_NODE_INDEX"] = str(node_index) input_paths = "%s/%s/%s" % (run_id, split_step_name, split_task_id) From e72712a7c4021329864defe683b721a5dfe23588 Mon Sep 17 00:00:00 2001 From: Alaa Ben Fatma <9027148+alaabenfatma@users.noreply.github.com> Date: Sun, 5 Dec 2021 09:04:58 -0800 Subject: [PATCH 116/176] Remove unused module + enhance code patterns (#849) * Respect alphabetical order when importing modules * import sys * Remove unused module * Elegantly check the Python version before importing the modules and converting to local time. * Remove unused module * Decouple LBYL from PR --- metaflow/mflog/mflog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metaflow/mflog/mflog.py b/metaflow/mflog/mflog.py index 1f63377281b..fe29bdde4a2 100644 --- a/metaflow/mflog/mflog.py +++ b/metaflow/mflog/mflog.py @@ -1,10 +1,10 @@ +import heapq import re import time import uuid -import heapq + from datetime import datetime from collections import namedtuple -from metaflow.exception import MetaflowException from metaflow.util import to_bytes, to_fileobj, to_unicode VERSION = b"0" From aa51c0b11e47213e5e27852a2f0fbb9fd631e420 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Sun, 5 Dec 2021 09:05:18 -0800 Subject: [PATCH 117/176] Fix timestamp issue for python2 in s3 (#846) --- metaflow/datatools/s3.py | 6 +++--- metaflow/datatools/s3op.py | 8 +++++--- metaflow/datatools/s3util.py | 8 ++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/metaflow/datatools/s3.py b/metaflow/datatools/s3.py index 6710049ad5b..a55f41b9e02 100644 --- a/metaflow/datatools/s3.py +++ b/metaflow/datatools/s3.py @@ -31,7 +31,7 @@ # python3 from urllib.parse import urlparse -from .s3util import get_s3_client, read_in_chunks +from .s3util import get_s3_client, read_in_chunks, get_timestamp try: import boto3 @@ -496,7 +496,7 @@ def _info(s3, tmp): "content_type": resp["ContentType"], "metadata": resp["Metadata"], "size": resp["ContentLength"], - "last_modified": resp["LastModified"].timestamp(), + "last_modified": get_timestamp(resp["LastModified"]), } info_results = None @@ -605,7 +605,7 @@ def _download(s3, tmp): return { "content_type": resp["ContentType"], "metadata": resp["Metadata"], - "last_modified": resp["LastModified"].timestamp(), + "last_modified": get_timestamp(resp["LastModified"]), } return None diff --git a/metaflow/datatools/s3op.py b/metaflow/datatools/s3op.py index bc72394e3c9..2912ea39d41 100644 --- a/metaflow/datatools/s3op.py +++ b/metaflow/datatools/s3op.py @@ -32,7 +32,7 @@ # multiprocessing.Pool because https://bugs.python.org/issue31886 from metaflow.util import TempDir, url_quote, url_unquote from metaflow.multicore_utils import parallel_map -from metaflow.datatools.s3util import aws_retry, read_in_chunks +from metaflow.datatools.s3util import aws_retry, read_in_chunks, get_timestamp NUM_WORKERS_DEFAULT = 64 @@ -120,7 +120,7 @@ def op_info(url): "size": head["ContentLength"], "content_type": head["ContentType"], "metadata": head["Metadata"], - "last_modified": head["LastModified"].timestamp(), + "last_modified": get_timestamp(head["LastModified"]), } except client_error as err: error_code = normalize_client_error(err) @@ -192,7 +192,9 @@ def op_info(url): if resp["Metadata"] is not None: args["metadata"] = resp["Metadata"] if resp["LastModified"]: - args["last_modified"] = resp["LastModified"].timestamp() + args["last_modified"] = get_timestamp( + resp["LastModified"] + ) json.dump(args, f) # Finally, we push out the size to the result_pipe since # the size is used for verification and other purposes and diff --git a/metaflow/datatools/s3util.py b/metaflow/datatools/s3util.py index f10f076b189..eb3c30a9253 100644 --- a/metaflow/datatools/s3util.py +++ b/metaflow/datatools/s3util.py @@ -1,4 +1,5 @@ from __future__ import print_function +from datetime import datetime import random import time import sys @@ -78,3 +79,10 @@ def read_in_chunks(dst, src, src_sz, max_chunk_size): # separately dst.write(buf) remaining -= len(buf) + + +def get_timestamp(dt): + """ + Python2 compatible way to compute the timestamp (seconds since 1/1/1970) + """ + return (dt.replace(tzinfo=None) - datetime(1970, 1, 1)).total_seconds() From e55df0016c5b6d8d8e2589828e0fc9cfb77705dc Mon Sep 17 00:00:00 2001 From: Romain Date: Sun, 5 Dec 2021 11:06:32 -0800 Subject: [PATCH 118/176] Address another issue with load_artifacts (#833) When an artifact is present in the blob cache, load_blobs would return things out of order and load_artifacts would therefore return the wrong blob for the artifact name. This patch hopefully addresses this issue fully and also adds a small optimization whereas blobs are not requested multiple times if they are identical. --- metaflow/datastore/content_addressed_store.py | 7 ++++--- metaflow/datastore/datastore_storage.py | 17 ++++++++++------- metaflow/datastore/task_datastore.py | 14 ++++++++++---- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/metaflow/datastore/content_addressed_store.py b/metaflow/datastore/content_addressed_store.py index d98f32f2ed6..e88ca1275de 100644 --- a/metaflow/datastore/content_addressed_store.py +++ b/metaflow/datastore/content_addressed_store.py @@ -118,8 +118,8 @@ def load_blobs(self, keys, force_raw=False): Returns ------- - Returns an iterator of (string, bytes) tuples; the iterator will return the keys - in the same order as the input argument. + Returns an iterator of (string, bytes) tuples; the iterator may return keys + in a different order than were passed in. """ load_paths = [] for key in keys: @@ -133,7 +133,8 @@ def load_blobs(self, keys, force_raw=False): load_paths.append((key, path)) with self._storage_impl.load_bytes([p for _, p in load_paths]) as loaded: - for (key, _), (_, file_path, meta) in zip(load_paths, loaded): + for (path_key, file_path, meta) in loaded: + key = self._storage_impl.path_split(path_key)[-1] # At this point, we either return the object as is (if raw) or # decode it according to the encoding version with open(file_path, "rb") as f: diff --git a/metaflow/datastore/datastore_storage.py b/metaflow/datastore/datastore_storage.py index d3f9020b339..dfbbd0ef2c0 100644 --- a/metaflow/datastore/datastore_storage.py +++ b/metaflow/datastore/datastore_storage.py @@ -238,7 +238,7 @@ def save_bytes(self, path_and_bytes_iter, overwrite=False, len_hint=0): """ raise NotImplementedError - def load_bytes(self, paths): + def load_bytes(self, keys): """ Gets objects from the datastore @@ -248,21 +248,24 @@ def load_bytes(self, paths): Parameters ---------- - paths : List[string] - Paths to fetch + keys : List[string] + Keys to fetch Returns ------- CloseAfterUse : A CloseAfterUse which should be used in a with statement. The data - in the CloseAfterUse will be an iterator over (key, path, metadata) - tuples. Path and metadata will be None if the key was missing. + in the CloseAfterUse will be an iterator over (key, file_path, metadata) + tuples. File_path and metadata will be None if the key was missing. Metadata will be None if no metadata is present; otherwise it is a dictionary of metadata associated with the object. - Note that the file at `path` may no longer be accessible outside of + Note that the file at `file_path` may no longer be accessible outside of the scope of the returned object. - Note that the order of the iterator will be the same as the input paths. + The order of items in the list is not to be relied on (ie: rely on the key + in the returned tuple and not on the order of the list). This function will, + however, return as many elements as passed in even in the presence of + duplicate keys. """ raise NotImplementedError diff --git a/metaflow/datastore/task_datastore.py b/metaflow/datastore/task_datastore.py index c75be9ca4a2..713361461ed 100644 --- a/metaflow/datastore/task_datastore.py +++ b/metaflow/datastore/task_datastore.py @@ -1,3 +1,4 @@ +from collections import defaultdict import json import pickle import sys @@ -337,7 +338,7 @@ def load_artifacts(self, names): "Datastore for task '%s' does not have the required metadata to " "load artifacts" % self._path ) - to_load = [] + to_load = defaultdict(list) for name in names: info = self._info.get(name) # We use gzip+pickle-v2 as this is the oldest/most compatible. @@ -352,13 +353,18 @@ def load_artifacts(self, names): "Python 3.4 or later is required to load artifact '%s'" % name ) else: - to_load.append(self._objects[name]) + to_load[self._objects[name]].append(name) # At this point, we load what we don't have from the CAS # We assume that if we have one "old" style artifact, all of them are # like that which is an easy assumption to make since artifacts are all # stored by the same implementation of the datastore for a given task. - for name, (_, blob) in zip(names, self._ca_store.load_blobs(to_load)): - yield name, pickle.loads(blob) + for (key, blob) in self._ca_store.load_blobs(to_load.keys()): + names = to_load[key] + for name in names: + # We unpickle everytime to have fully distinct objects (the user + # would not expect two artifacts with different names to actually + # be aliases of one another) + yield name, pickle.loads(blob) @require_mode("r") def get_artifact_sizes(self, names): From 90fc5861decd13477b915a3a9dc5f052c6d2fbe3 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Tue, 7 Dec 2021 15:33:06 -0800 Subject: [PATCH 119/176] Address comments from Romain --- metaflow/cli.py | 2 +- metaflow/lint.py | 2 +- metaflow/plugins/aws/batch/batch_decorator.py | 5 ++--- metaflow/unbounded_foreach.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/metaflow/cli.py b/metaflow/cli.py index 4191ac2c94c..cd669d78e7c 100644 --- a/metaflow/cli.py +++ b/metaflow/cli.py @@ -480,7 +480,7 @@ def echo_unicode(line, **kwargs): "--num-parallel", default=0, type=int, - help="Num of parallel instances of a step. Ignored in local mode.", + help="Number of parallel instances of a step. Ignored in local mode (see parallel decorator code).", ) @click.pass_context def step( diff --git a/metaflow/lint.py b/metaflow/lint.py index 4eb52346e18..3b0806642ab 100644 --- a/metaflow/lint.py +++ b/metaflow/lint.py @@ -297,7 +297,7 @@ def check_parallel_step_after_next(graph): def check_parallel_foreach_calls_parallel_step(graph): msg = ( "Step *{0.name}* has a @parallel decorator, but is not called " - " with self.next(num_parallel=...) from step *{1.name}*." + "with self.next(num_parallel=...) from step *{1.name}*." ) for node in graph: if node.parallel_step: diff --git a/metaflow/plugins/aws/batch/batch_decorator.py b/metaflow/plugins/aws/batch/batch_decorator.py index 18d3ebed079..bc27bb6d051 100644 --- a/metaflow/plugins/aws/batch/batch_decorator.py +++ b/metaflow/plugins/aws/batch/batch_decorator.py @@ -324,10 +324,9 @@ def _wait_for_mapper_tasks(self, flow, step_name): """ from metaflow import Step # avoid circular dependency - t = time.time() - TIMEOUT = 600 + last_completion_timeout = time.time() + 600 # wait for 10 minutes print("Waiting for batch secondary tasks to finish") - while t + TIMEOUT > time.time(): + while last_completion_timeout > time.time(): time.sleep(2) try: step_path = "%s/%s/%s" % (flow.name, current.run_id, step_name) diff --git a/metaflow/unbounded_foreach.py b/metaflow/unbounded_foreach.py index 05438ea6a47..a5228bdddad 100644 --- a/metaflow/unbounded_foreach.py +++ b/metaflow/unbounded_foreach.py @@ -1,5 +1,5 @@ CONTROL_TASK_TAG = "control_task" -CONTROL_AND_MAPPER_TAG = "control_and_mapper" +CONTROL_AND_MAPPER_TAG = "control_and_mapper" # Set if the task is both a control task and a vanilla UBF task. UBF_CONTROL = "ubf_control" UBF_TASK = "ubf_task" From 176a4c38839d047baeb107462b0f2334d9d19df2 Mon Sep 17 00:00:00 2001 From: Aapo Kyrola Date: Tue, 7 Dec 2021 15:43:23 -0800 Subject: [PATCH 120/176] Add disable_parallel to test contexts --- test/core/contexts.json | 3 +++ test/core/run_tests.py | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test/core/contexts.json b/test/core/contexts.json index 23823c32deb..b4027ae707f 100644 --- a/test/core/contexts.json +++ b/test/core/contexts.json @@ -3,6 +3,7 @@ { "name": "python2-all-local", "disabled": false, + "disable_parallel": true, "env": { "METAFLOW_USER": "tester", "METAFLOW_RUN_BOOL_PARAM": "False", @@ -125,6 +126,7 @@ { "name": "python3-batch", "disabled": true, + "disable_parallel": true, "python": "python3", "top_options": [ "--event-logger=nullSidecarLogger", @@ -159,6 +161,7 @@ { "name": "python3-k8s", "disabled": true, + "disable_parallel": true, "python": "python3", "top_options": [ "--event-logger=nullSidecarLogger", diff --git a/test/core/run_tests.py b/test/core/run_tests.py index 8fe7f4442a8..d668c88a127 100644 --- a/test/core/run_tests.py +++ b/test/core/run_tests.py @@ -226,7 +226,10 @@ def run_test_cases(args): if formatter.valid: for context in contexts["contexts"]: - + if context.get("disable_parallel", False) and any( + "num_parallel" in node for node in graph["graph"].values() + ): + continue if ok_contexts: if context["name"].lower() not in ok_contexts: continue From 4dbd3c68c22cf0078b158c3bad947e94d8df3d87 Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Wed, 8 Dec 2021 11:57:05 -0800 Subject: [PATCH 121/176] use mflog redirect_stream wrapper correctly with SFN (#851) --- metaflow/plugins/aws/batch/batch.py | 5 +++-- metaflow/plugins/aws/batch/batch_cli.py | 4 ++-- .../aws/step_functions/step_functions.py | 21 ++++++++++++------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/metaflow/plugins/aws/batch/batch.py b/metaflow/plugins/aws/batch/batch.py index 13c0d9fd9d8..72257fda012 100644 --- a/metaflow/plugins/aws/batch/batch.py +++ b/metaflow/plugins/aws/batch/batch.py @@ -62,8 +62,9 @@ def _command(self, environment, code_package_url, step_name, step_cmds, task_spe step_expr = " && ".join( [ capture_output_to_mflog(a) - for a in (environment.bootstrap_commands(step_name) + step_cmds) + for a in (environment.bootstrap_commands(step_name)) ] + + step_cmds ) # construct an entry point that @@ -287,7 +288,7 @@ def launch_job( ) job = self.create_job( step_name, - step_cli, + capture_output_to_mflog(step_cli), task_spec, code_package_sha, code_package_url, diff --git a/metaflow/plugins/aws/batch/batch_cli.py b/metaflow/plugins/aws/batch/batch_cli.py index a838472e2a2..a63bfa1e3b3 100644 --- a/metaflow/plugins/aws/batch/batch_cli.py +++ b/metaflow/plugins/aws/batch/batch_cli.py @@ -11,7 +11,7 @@ from metaflow.exception import CommandException, METAFLOW_EXIT_DISALLOW_RETRY from metaflow.metadata.util import sync_local_metadata_from_datastore from metaflow.metaflow_config import DATASTORE_LOCAL_DIR -from metaflow.mflog import TASK_LOG_SOURCE +from metaflow.mflog import TASK_LOG_SOURCE, capture_output_to_mflog from .batch import Batch, BatchKilledException @@ -263,7 +263,7 @@ def _sync_metadata(): with ctx.obj.monitor.measure("metaflow.aws.batch.launch_job"): batch.launch_job( step_name, - step_cli, + capture_output_to_mflog(step_cli), task_spec, code_package_sha, code_package_url, diff --git a/metaflow/plugins/aws/step_functions/step_functions.py b/metaflow/plugins/aws/step_functions/step_functions.py index f71b560fdbc..1d1b407aafa 100644 --- a/metaflow/plugins/aws/step_functions/step_functions.py +++ b/metaflow/plugins/aws/step_functions/step_functions.py @@ -25,6 +25,8 @@ from .event_bridge_client import EventBridgeClient from ..batch.batch import Batch +from metaflow.mflog import capture_output_to_mflog + class StepFunctionsException(MetaflowException): headline = "AWS Step Functions error" @@ -701,11 +703,16 @@ def _step_cli(self, node, paths, code_package_url, user_code_retries): param_file = "".join( random.choice(string.ascii_lowercase) for _ in range(10) ) - export_params = ( - "python -m " - "metaflow.plugins.aws.step_functions.set_batch_environment " - "parameters %s && . `pwd`/%s" % (param_file, param_file) + export_params = " && ".join( + [ + capture_output_to_mflog( + "python -m metaflow.plugins.aws.step_functions.set_batch_environment parameters %s" + % param_file + ), + ". `pwd`/%s" % param_file, + ] ) + params = ( entrypoint + top_opts @@ -737,7 +744,7 @@ def _step_cli(self, node, paths, code_package_url, user_code_retries): cmd = "if ! %s >/dev/null 2>/dev/null; then %s && %s; fi" % ( " ".join(exists), export_params, - " ".join(params), + capture_output_to_mflog(" ".join(params)), ) cmds.append(cmd) paths = "sfn-${METAFLOW_RUN_ID}/_parameters/%s" % (task_id_params) @@ -746,7 +753,7 @@ def _step_cli(self, node, paths, code_package_url, user_code_retries): parent_tasks_file = "".join( random.choice(string.ascii_lowercase) for _ in range(10) ) - export_parent_tasks = ( + export_parent_tasks = capture_output_to_mflog( "python -m " "metaflow.plugins.aws.step_functions.set_batch_environment " "parent_tasks %s && . `pwd`/%s" % (parent_tasks_file, parent_tasks_file) @@ -787,7 +794,7 @@ def _step_cli(self, node, paths, code_package_url, user_code_retries): step.extend("--tag %s" % tag for tag in self.tags) if self.namespace is not None: step.append("--namespace=%s" % self.namespace) - cmds.append(" ".join(entrypoint + top_level + step)) + cmds.append(capture_output_to_mflog(" ".join(entrypoint + top_level + step))) return " && ".join(cmds) From 747b86c8bfa2ae2be5c24f903e831396771574ec Mon Sep 17 00:00:00 2001 From: Oleg Avdeev Date: Wed, 8 Dec 2021 22:42:00 -0800 Subject: [PATCH 122/176] Bump version to 2.4.5 (#856) * bump version to 2.4.5 * fix mflog double-wrapping bug on batch --- metaflow/plugins/aws/batch/batch_cli.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metaflow/plugins/aws/batch/batch_cli.py b/metaflow/plugins/aws/batch/batch_cli.py index a63bfa1e3b3..bb9922c89eb 100644 --- a/metaflow/plugins/aws/batch/batch_cli.py +++ b/metaflow/plugins/aws/batch/batch_cli.py @@ -263,7 +263,7 @@ def _sync_metadata(): with ctx.obj.monitor.measure("metaflow.aws.batch.launch_job"): batch.launch_job( step_name, - capture_output_to_mflog(step_cli), + step_cli, task_spec, code_package_sha, code_package_url, diff --git a/setup.py b/setup.py index aae36f718bd..4737ef89a74 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = "2.4.4" +version = "2.4.5" setup( name="metaflow", From 5305899059969956b83c38c52570521beb52d94c Mon Sep 17 00:00:00 2001 From: Valay Dave Date: Thu, 9 Dec 2021 22:31:48 +0530 Subject: [PATCH 123/176] `add_to_package` decorator hook. (#834) * `add_to_package` decorator hook. - Cherry picking diff from card branch. * comment fix. * supported including the same file as different name - Fixed comment. * Updated comment. * updated comment with more context. * Refactoring logic for readability/correctness. * - Simplifying logic - nit fixes (spaces after commas) - lifecycle PNG updated. --- docs/lifecycle.dot | 4 +++- docs/lifecycle.png | Bin 379797 -> 387123 bytes metaflow/decorators.py | 14 ++++++++++++++ metaflow/package.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/docs/lifecycle.dot b/docs/lifecycle.dot index c499dce21f8..961aca6e5cc 100644 --- a/docs/lifecycle.dot +++ b/docs/lifecycle.dot @@ -41,6 +41,7 @@ digraph Metaflow { validate_dag [label="{graph|validate}", fillcolor=lightpink2] init_environment [label="{environment|init_environment}", fillcolor=palegreen2] package_init [label="{decorator|package_init}", fillcolor=lightblue2] + add_custom_package [label="{decorator|add_to_package}", fillcolor=lightblue2] add_to_package [label="{environment|add_to_package}", fillcolor=palegreen2] package [label="{package|create}", fillcolor=lightpink2] } @@ -148,7 +149,8 @@ digraph Metaflow { /* package */ validate_dag -> init_environment init_environment -> package_init - package_init -> add_to_package + package_init -> add_custom_package + add_custom_package -> add_to_package add_to_package -> package package -> command_run package -> stepfunctions_create diff --git a/docs/lifecycle.png b/docs/lifecycle.png index c2cc3fd13412f23104c85ed18c0c50ce878205b2..a412b651af90832a39aed15f5a3be2d054112eb1 100644 GIT binary patch literal 387123 zcmd432UwF?w=l{KpE!;R<5(EF3W`b_=^Y&v3D(mTOI zCxjk)R7!vVfdmMlBzM0s^PT_v|MT2)?rHzU$0v|?m$g^lYpr*^zN@2g{Acc;Sy)(( z!*1WyV_`YO%EEG{>nAqw%~YpoFZf`4q@{6_g$e!3Xv~XdVL8hJyLsKgpWFsz2IBXlgIUd z-C@k*3-9CG*AAWj$IU|qH*fyKc8VYnurl_nj7t{1w6SA>ZxWY>xRZQH_acjo>lv{Ha;EcfK!6ix1j@I~!@q zCGR#-qs-Wv&IxA;c;1~PBO*r#fEQSU$Jas(`$brzvvt!|OMd(jDU_ui(b=1MLyHgK zn1x%E@Nev2RU`TMwT7o6?04273VYVq^qi+8#l+)vHG0Jl5zpafGz27zcR`pJRcvO z@6;0>NqNa3kjA09l`*&RnL{llOuX)r20y|^yI>P%EjwhItQR;3F;T|1D+XLowioi; zkyn$QinSdrNX4d=S7v9;CM>?~!~Dnb_k4<^qH@WAt<~;CDPiGMFFl`8+jy^q{tKU9 z9`E|z66V~R3hz;%f9J#`2+{lFn_>#lO^Pr{myr)C;CttAdhbZ7HFXFbBvZd!W#%*H zn3wY8`|H~6#lmvW+0Ll$^(bn?-sZRhK%yFj*vjKxL^UNv@4Vo&Z2uN2kXGe7o@-KW zw=h}n%UI7-bM2jii2GsI`4?k0ofP@2)H!UIYxQKDPO9SUpC@(NPwFbaBcU;>PYpSG9isr;w_Wam~@=NVty7x2}QJF{KU}EkOFN2 zJ^+?g-k#4|0Njq80W63E@5IZ96pt?QPFa`Vz23CgrkErrZOjc@T>Qc}|_RLMBx%CJ=0*<~5H)=5|R|%p`Qs zMj;*0EtSW;p>(c(7jWNnTNG|0&wClY9|dUR$*nwgEwqM2TmyklWb4kvWL4Dm~k09EluPzWU zDbJa9fDAswLQXuTp0FprL&gM~R=FhJ3Oi%l8FQIrPK+~i>#&Wth?~PE+XFf8(hZ_S zo2vOvA&7p(l_e8=ieBxj-#$w{{PbtQtcX=dHY{2a%nTs=6S}Z+gcN-pg$ow-K9dQ> zkotXa)L?%6&%^Az395nHGpNFn&Z80yZx0Iud>Ez!2AHWMm2`c0_}O#4PeT}HTJ4r9 zQc`msn#xDUVSh)WzTl`h^+@sa6A7RM8ERFape;{poQWO9? z-6|f3Gy8zXPd#$-qRq~7wJvRCQs2nP2)+v>u%o1YrLLo_2xm&_OcjC?d$3<#pM>OY zeQC6O?()Y~n6%Si{xLqq!4PM#PexF<5!;XHzwMjCeB-R6d#MM;QRqKu9RF*l-?S}Y z9UKD23)t6@p9ov6a{blr1idzJf~S9sKk95wc5hPPqb(#D@tMXd{`?^=jf;ARjf1}k z@&KOx3XKq2V8${QyzijFtl@!I!-AUorur56z)AS@-{d6x3m$l8YU|Uw{v5v;qX-#5 z7SeUDcD7m~#ov>7!8cA(y`yn=rmL2yM(*%;Z!})Kv_HncWRb-skzJYEENWHOFnCqd zt+j@$mZagE)0U~y|K2rBuMM(ETlYJ!$+kBMuhd^B7g`#omQl*IQ=na**4<^3{Y9W* z=laU-x70+&$hv5dzlY*7{&Jwsa`XyoJwBX1T^oGIcDnP}Zfo;IP~S3w{yJXY(j?7d zXE|v)$kQBEh3ZUi+RN(>a`Wk_VT4z>;LC=&1vSFEca4cet>9ffH_X`V#d5#wPHKbICRdqmE5pu(A-nBv^1J_apE2va zV|6Zf$9x>kB>ZJOzqH@sQn~>19KgG25>->B=F6@puj(HH#~-PPd3L<{xx!IO*yrGP zx2R`wXh#|cDr^TDwwl9gH@`T;yE&p@%<>&!n4NA?YKyx>4?H+Xe0-(Mb(6aSD7fud zWsm8T^atM#b8YTCbQ~zq=5N~krtIHVuJE;+)oJ(%Ey9<&bv()*C$W_H2KA<4cd!c) z%-E4367Qr%zi=1tmbroMNa=u=P4g>VgxM*p3K|OpdnZ*prtRTI)C{#zk_!^io_h;; zB80C@f?`F_s46dQWyx2{dudSGbx7KRV<~B^Iuci*kvqGPoZb^ts^1^@fL1Y8;C+!G zrGQb`l&gL6y+y`Y(5AV3e9~CIy1%YabTQnT1-Tm4n zwUJo>LpDYcl&Y-){4Yi(-Y-0s@`Uuo0FCMi5+>8$Da`34C%duM&90Mlbn8o&^(-t> z&BPln#72m~SFC!p3pb0ALaaSpZwn7wrTU9wg#C|?70z`wdfV>weyTkR|Kc0j-t5kP z2f5@uVM$+pORyUg9Ps>BbJTfm=d&^Mt+2bFB-Jv_81ACKGW7Htt6pQ7qEvcP;th3P zB%f;e=a)f$f}^q|zsNh{{%tClxEGE=1o*5}!nJE-g-#ouUhp?16q#2^;-=^>*fn)+ z?pZ2lfU2!+Av#7xL_&+Wr)b~t>4M4c_SYc_>GDwux_^@s44X1Fpf;^lhwBzDo7vJr ze3&E3p~1w}3euQ3Op=snlDN@bJ&9J+e%vliVhyVP#%x{X&HT?2>eeXzEgWI+kI-luk$Y;6!vW z$Qw;T?QN>NS%~Dh(OP_u44RO%W`xAVdomsJRD4WRgSRI(I^%@I#;fn#GQ5Q)aS*6c zFRYOu0Y*MV?IvFbFN9mmcy3aowFj&srBSHJDv~oj#4|fjEDWF zri3?pP^Sw$nXxqcO?bK5=ErjawxMZC+3!*3^;>mJrYE$b8e%AKC|*3hu8qXWyBn1U ztN(2K^-KvEq+uCg6W35JB)2<$9O91dp^PXboighq^R=9?pq0uq{G`DcSg59NUvMjhp zo=Uq-$y;oa^%j3~9%^ux=4q6We*;yNI{pm8=>rLR@}<~=C~pm};; zpDk{L74zh*wK%^%bvvz#+>shTQQJeqJ_%5F+oPFLrzU8_&W-Wu0SdHTq6&gsyx~s} zwy>ZTXFBJnOMo>#I2yz$vQ+XtnR{l^WP@nLs##BjPi$n>_fPloJN8xGYAi0V9ilb4 zUglmT9t9IzxI3G569RQ7ss~S0-R7xtA1XxY8(CJp9smzn0wgc9;s^c@kEQmVpt>ES zkV?%_3%U$ULE>h6#hr%0N4ept$XNkVUEnwjzAxNSB z7pKd;S9Qp?NiA)s%|N!KyV?C%_cG^XT73Ew1t6oNz%VGR2)E@p6`~Z$v}I0$r8B{; z_G_G|MT2yx#gqFh_1~NB)VTg39=Omi)RC-D=yoMEilKF$$aI|%)7n{ChPS`EQPy_P zzsR_XqpjGyNisCe*I{J`y+irVDgo)6QS!`Q8SZVM-p%>vH^Z2TL5@} zmDaf5XkpY&0HjNPwf*Yn{J(oWk(iy{kC$h@8zJs{Tr0HVXD%^Fl^Wiu?wi@8CcB&8m1#s`*|PY!9s0WeY;2xY$4NQWcN}Me-af<# z7!mgv#K2@JgwC0!+TZcV8ZScZ9qGT6V6)*XW*N9U934Gkjo0-OGLqQ0Df9W2!!s4yIWOn)muQFVYnF)q(NND){F8132QfM4ElVgPPt zcW#6*^Zhj^-aMk_GHo$S)e0c?DF?)*Q*Gqbafv3ClWlj340W^{ydR;A4ec?}G%aK} zb6_jRIF9&!jdSRf@8KwUd9?{@XyR^shkywiquh+SHw-Zkf+H<{SgBeE8Q``>q?F?r zMZ-09MRE_eo*JNwdb;!M=H8wNwF$*7y%nmRMk+}%hv@X}Mk}1SKN%^cu=M=1=f>MY zsX63T2&XQUSR?PLtc^r>fO+_8?O|ejSkxU5Bmz(oDIpTL!O+|id=MH7yp#dgYt<=1 zlM9acN(&HNgm=BDhCGCxy{=kEV8Dc{(2+rVnf>h z?v=p2IB+8)fXZ+mu6ynd{4d8blp`?0**ebH-SO`3G-;(hMx_3S@k4;Qg7YG%3d1+P1tq6y_0Jv)2%*qJ@wAT%IthNa3Oe{jL?0UnAkUEwygA)KUP zJ8eazWOs53+KsS8=Nr~(L(MGTk4Gm4p1Ek@v<)Y ze{X{5jo8HD?3hxUIOx6wG4eSpwd}b~oR7u0?b>b&VinI}Hy14>#jKO51O>6DuM=fD=L2+jiy+SH{9XJQiG9s5M2gP>I#Tmf`NagSQyil40( zGH%%{_dV+U+(sywP7IVNyr1gbLOETCM};;*(V{K+l4BDrDvTVBsxiMpNaG@HJ{6xt z3hkt>)69$nwq{xBK~ZF4aPTI^r1p`m4+&$Yv%LXZsIbaYL{k~hhX~KAt3DOpiyq7A zeOXfdgz;Sn2Cl0ya#zqB<%XoLQWK~q6mP|+! zfJFFRg!S-v)pQjMS(Q3IH%Le`B~74W3(LIv`VC}?QSdo!6|1t9y%Knpg^_*oRX24% z-m$FV+C*J!QEV_gKk8r5cCAr&Db-NKXcn?OOGG=o-`f@?bYnEQjx8VzaHcI2Kygd% z)Q);vS`#gm^4{OOmCLByWEC+gz8?!TkzaHXzDj>w+I>J7Qk6#HRsclkPJUF$BEh&7 zqetvc_G{t@oCv}Xv8(vr2C5vG!?M=ecz(p?+0(NNvKixkxcbaRXEZ<(i|QM|mEIG1 z*5KluT#DgJM&%lgQWI31dtC9}_0n#`@+aL=-*yBirc~!*VuhKjHVjfPQ#1OEHgs3% zPL-uu+gZ7i_^Nt-u;+h^PY+g+3tY|};%I*2V^pFxy)A081&l)x&hKA&wMpPw&5Kxj z2l4g9s=M62kh!-uC~uJt?<*H@A+ti8CChCD2yEBvl@uB+o6R;$ada2i+uN=qvi2>> zN=0q$PFX#*JHZ|@e8W3VpCj%G_;29$J&#rsTRMFr9Sx7j5?8&_bw;AAhRGE# z>=tkyR%XVWlmwxO+HYNQMKr&9feL@cm4T~KmNbW{lx@40QCw-NdFvfzwD{Rb^5nZC z)x`|eCQ2t|U8T}Ja%<&;j4rykP+Nr%s_KNnx836uhy`Rv%rAyg=DnkadC9bvAg=Z4 z>parL){#N~%BsKAq%Dx|3;yfPP^L#HOC!pXI8s(t25D+2__ZBb*#X^`%F~lmvfgRVAD0d#Qg>4-#)~bmz6959KXTKWTavW99;xbkPFS4X zaCtdIPp|o@b7tfZ8!vaqHe?C874yZn5@j5Xt`uGw$kYgRsoXHSfGsbeIk~30Uyg84 zIWMRY+5Pt14OO3s{CioFUh};%tLYa2S2?t+*GqojmvNpHt=}NH7Plo=x>Z@iUBT^u zB>HODz7f!~_-cs=f)OdEKmG6B9YXafTy@h% zF^H;*R7O@h8S1Do0a5`+oJd61PrA0t$O34;nHd4$=AKd~`ZS&^_(Cp*Ml5dd;%M}B9 z_wL;z-A5o0+v^L_0P38Z2dj&N)#0Cx2xo&V`EV(P;#vI@l;RjX3BDfy$xV=t!VC{R zgOYO~Ppy>zg`qOO>9OhQ<)!&KVTYcbK7CqY=IxFCT!+ z88qz55x~4q|IkViF*rykx$vQxNdi(22eT* zN=kY-n^s>`IP|?A^Z<`oxQjaV4(AsF-qzE|NQ>Z>xdPJKSGmK^=m+3e=O(er}_um0k1 zgCSF_?Ap`0+FE=ee((3w1G;Pku6E()=ONk2U|jyMEfV-YnEWlblvPTKQVB*eY|7&9 zL70D#)%L7RTJYkU`A$eMPRSiB<{fi<^ZS6V-3*!`b#$)L@L*iJeCIXpm=4{Du84T% zB`UG$-oJb=TL99B2FAu{{|S^~@;R)qyDK=)Q$3WZH;d+Uj=k_qTp_x<0*1@p>){(9 zTCwZuRkhZ%93^0xyG!6+F=K5a&s+|0BJW8)o-Y_B!daao{OK=XwJZH$lzyq{@<;SC zuccu@;mDe^{sLaNu?hHGZLD{Ekeq~y)H$swtC&<@Qra|*|J@#k63VVyhJZP%rFuUl zx+&UVB7qDHl3Ya@Xxbq zn&olDP7%qp))e#Kw^l_{)Fje*y#7e4bVyta4m_)-O#QM;@(oJJ3Qe_;B3!UPrW59B zYX7xYzig}V1!8GMi6UU179$YVAT_x;C5K}!uLUsIrw7WslLHttQhH#ov<9X`?UG0y zamL*|A=7Hd2sy-Y6+tYPqlQSZW4k$p^SbhA_ZwSlLg9S{A^kFH;5M0&OuH|-{Hd1O#OHlLa9E|Wqk~skmdLb$q!E%I1YwN1d5yZ zMv1@MM`9NYsJ#W=no1^W4vvfIJl|=^HK8*3=HOy zLR;}MaczN!n?2i0X$T4fGZTJPqNCLQI1UvnLjAs#=Q%Md*k8M24>AFbeFr2w)g<-d zLGGO`Gs~cddhBW&R?crgcCNQnOVxL&HwjM`dm_v+83eD}Tr$-E$3>WJ7K+#JlcPy# zfJy5fIWC&k?Ox(B2#0!L$Ab+rkFSd}GvZpj^veG!S7u zYB#(NvK=~;YUQo9fVYjP#8Ii-98hosm^~$(UBJkBB6N0~2i=g8i%k-SDXvTfM$jYW zmy3itKKo7}h3H=CLC)a|{ni`BHE_!`OL&nXNp@kZ+_>07$!8&nHkFt@Ub5$HnX+mF zs3``9q|rDVjSFXX?9sTC(!a4)a!fRVc1H9zGA15P?~`+ym=S2jS_KxNJg)G_pf(0P zlZO@N6F!b2s@+WFgDsIZ8_q~hs=syl&8CS#LKb z0x&D*aLc}n2|}Yy*2IA6H-usCYNy%GfDcdZam%?T9bq!hj9)&P-t$c3I7(*anHeKt z(Xg=Jt<3@{6rIi+4|H#_%v{TwZn331F;XCrK1ruicGAs}M#a@w_cm1vxXdGDIAa5+ z#-L5MS3-7$8wBjn+`*?c>fIZ?gj3zbo3}gZ1O4P0Hh|a;t!X3wopv1 zHLSdE=}ctl=c-DnTKKQTqAqF=Cy*=Sb7wlKO-V3a<+vUdzudBWh8I;OeIk^j`GbXK zPoG`@%ng7k9!2N)4WZWecJEcbX-1%NJ>kUkoe_esHGUr@w_~0i8nD~p^ZepyKNou$ zN8}_hOmlVpE=pZgOYZ7X6(nP-KiHWAhE4Hdv><}}{A&ZgU?8TXY>P;Fj#pxO@~te> zR5;`>rH}bzTqbi}Bl&%N?3&~iF!UZd$>rK3h_7*KbN0lPM9P~^ULOkQK%K=7FJe0} z@hW}P+`A8fuw6WwsA6r(c-&%(SYNpcD1Q!!PM3z=R3;oRJ&H2nucZzvBbZi__H*kK z!BhcHM#hbDL+TlmvL_yPxP&;NQMb@kq!818fK2T#KPl_CZzf)Tas712Yr9&ar=P}2 z3}$Yr)w&Nxz>q528@TwUV3Q!mh6FyGM}9p<)PfQeM4KkhzEdN_2^Hk$cVki>JLxC@ z#sNh-$P`LMur3OOg8-LVF;?q^I%LNyyAZ~f`c$Fs13egdP+io?hqp?tcdse2i}cbbbt!14K&XEn31lAdZaquh4!zBII=e;AvL`1sim^$&pydHM~ z+pdH|gze@`$LZ9s#Q7*KQ{GLh-fy@B(U9Yegif2=Bt+w3MYoO}vi)3GG=#i`-U9d) zo?`m%k-mf=5vjEF7uwQ9$ulO!^kXW+_F*4=T0H0OT#})FNC)S4?apB9?i01R`R=- zYLiUcOskzSumf)WDZu_LSn%JUnD{R$i~P&uc?BG~`vPiws-ob3(co1Glm}rIw_=}Z za40EFZZ29Z{?_cL_Z?XEX5r6g4B33f3w_z;zb8QT8PWTtn5ta=V@D{kog$UPX~*%l zGZMjZLWs2mVK}*+2U04(3h&eluQZB8?b55To2uX#PhbAQbvpv6Rl6@I4$q>(db&RQ zP9+I(`R~l{Ql7)hygHD-=IzwK!!#&km>m~*_L?};gCqrmH_}ahCl_mTaey_xo-8aI z{9$WmYn+v4^G(U!e#Dm{*p@<%)oY{Q)<_GQV~dSu?l%4H>op$cO)4I*n2b3A=|t6c zG0z6?H#~lbw5e91p{V{S<6=z{`dnkPSF?w(+c0h4AwT7I@*-?syN)w=dAB>58nH{PwV$=x+31en zCdXQ(Fe^>D7;Ax;rpGr|n%1_3IwX3SrX_U7A#Y0oh$FOejtRl~ftIV-W>$B)jW^w> z8h@Bq8R^e(k0K^p{q`2}GKspXf;V2K$AjRh`DW~ZGrwX#aLeulIhp6)d4ie)nIM1P zH0Jnpt^0aJpSr`Fl*M5Zn_K=f{o>va7j1dCLhCiHt1&LK8S zF>^5Q|$;z&alS157)OO|6TwknZ1$CJ26Szqcc(3(ozv9nB;mB~w^O-A&n0q0O*rHeRM zL6v|iH35-3<(eKpPD89`Rz|9A7JW0R^t81E=ZjSGzp3cU>{ho2(6Vf0!)?!c%K|?r zQFS44eaZj9eqBOrgy-s-uw4tqHgRh;1X5mAu9(1!D)jrvnZ`rp8OWg3l;gCKcl>tW z33W(C?a9E(a8H9r=Uoi7lV-l0;LHa3T$rAbWt#4Cqt#Tf`@^}$tM*yM!(dN`p8!XW zlBDc~V5*aO1a4y8`iLM*k~q5ng8VxI$O7OKP;9lN=pAiPO~?-+FYQ8gEGQF93D)$$ zCG=?3-sZx3fv&q(RdKazGOD%GUB+$72~?UQTf*f0g1`<>UtZ(7ohlYUTo&p84skhM3$NSCsBV+}PUBSzqv&95uC4AY4qGxK)CCD0BVeW$ zT$8lRAbZ{%rvp~39QTM9>X;F3EB07vSt}Gc(;q~P1cnb7>C>?hk_#Pek_)djsEMkF zFTyrQFu@Rw7@J=k5zakD+MZ$iQ2_6(W7$25wE4tNb;HE1_7RyLZGb~?m&wN49~J^t ztj=ltJDb>(w29PstAv=eK#tqUfL!tqmqjFS0wr6(ku$W#v9gbaV<5r3Rtln2<-^Nk z#P$JPcX$VG4pK29`c~~rTmzI)5%I1p->8o3ZO34tkrqy^WO#4$9e=k!^ej?!$<^Eb zY-TQv+I8#miH?vk}JI4zvcC6oO!TWk(cLL+66yJAx7 zX}D6uSeyv_>@CyAp+pZ;Ao;76$oP;$*gL${>MrrsZ`=kqCOj{>mT51wT>Bc_5px=E zJpq7st~73TOv;(Zn*hDipWGTF4XLr?8eJum%ab7*axbq z2ddqz(xo7zt5E*$mN)z-QSx6tJ=ucE*n`SrNy0EtWNu2j4?<~k-HH0&D*(D!j)zLV z#jbogscTM508yuP5?%rn438Xp4Ly7VN;K`{K-9^=$S7zjfkL6AfR_l8FP{LQ8fT$0)QzGOykRf6R2upZ`+x5C#~`_ET?uk-*Tu^% zBNa+foQV-kN<5B?!{5PPJ{5C{_x$O5gEtRd=jtJIVA8D2Lko|%eQmIEE;U-BE)nTV zH@ehRNu&Nya7*_2!5c^*;(X#j4uIK<$k;DXhH(E6+q2*U{6Mvwkfi`WJlDB?DjdnW zw{5qEiS@eBH-G8i`}QAVo_*L3B)Ews?7kGYy8XLir<%`tb)%^iQSgFg!vn8V`{SOP zhf4lJ*BorJs(iHVzb*1DrSIOs0-k)G*K9<>TF!^HP%v$_cB(I>YxhHn;IW{>Z+tAR z@@Ej&NK6&|HV%#an~9z%{H_8lSE==aRpWcxf!s?@WO0i<5^|UHj{=j&x`$+4<7_7c zG}xW{>fIOa->m5$a6_5Tq81gsiWa32!-{k3Yrl#4(X{!(^EDr+G)k_8xR^k-upmCImT|1&dZ-V%cc$s1q|)iTduLp2 z%E|{Q3*>^k535C|#nY4uOHSwH(fLD|w&!5+O+>7Uj=Hcu_8 z2BbU>4M%#BE?kV4NWss^L|Nub_3Kin&{&^W!h+WV1jT$+YG zp%{F{D6myquY+=W!@m2(g%pJxy{6MM7CpvDegt&)1-!ERT(Xx&KZ-s!Ov3Gnbk*1j=;(b))Zd;j6!Uv;#|P7^erPRIs%tMse}GJC8o4F@VzS~f zg~#)ErG>^JLVLXn0@fdfZpgikU0m3iP47Z)Mw|Zbm;$X{Cw_NdKf~GaC(QzdQ2JYK%f7uOY<{}N9G`6Fk zupOS9&AENt$WHxb4*mMB_9p|DqG-YvME6i`&gXdmb|%(n$2O%AaDg zk+WCw7mj`$CZI1^4&$uQ`Z-TchtJxHJD?f1hdgRCkY-pJn9pHKXpPiQx71s-ZhR@%ad;ew-__i8-Treiel%a*9^1od3V z`dVvx*^=cQ-TKS%pWN!wyq44O$$a~9Ocs_I&ZT1&SM7Da0B^3)05e@KscTTh@SW(G zF1s(P*_c*T5VB*(8>~%;>0>WM@3#M^MGNz?1EnsFp zKE}dAIqJhUmf-yQHBv~aUWvILsdI=SyMi`r>(=pn?A1-tADO(NJ?5*ID13#qXvKK} zc1!CC0gf$?+x5N*R<(#Hi_M$Gows$nBz=}AxE4)68(%%~4Rz|S+1!tZRuahTr4CqLh1;RD zB|SRo2hYdSq_2^#zB399{-H}t%Z4N?U4j(#Ma@8&jU6hetIFg}bJ&`PB7G-w$C8{? za(rLn-lpGW8~aU{tmiZ-ax+++Sz==Sa{kM=pHjo$3?63Pp5Lx49QI(3wA#$6SzlCc;BawqSP14*FJmi_Ws~%fiWGh zJo+xRF~rn67zU(mFxU5$j`}U96*ECd1+=6pm&8h`_V#G=3}pw}$BG(%OH0&>MQ8um z^P*yU99==n`#m*6B{O}vcyON6tbMe_u%M&kjS|8fY0wZ8CA1J~sO2q6X%Aeq+3}DRlrm^s`y*P2`N_gqEOvPj%Dl8vOi`&i3{}xiDt2b{D4! z-eG9(!iN9O-p-sDkPNhLa#ms%8!7dz8&U{=e#MVo>}#0u;9Q`j=TkO0_Y8ETaQ>XW z_KdN}-Q+ll{>164e!fGk8m511^Q$JJXTt7<`7RE^kQF}5V@!o%8%f9Ylv{o%@12xL zTPg9@w#%FIWJQ*d)lPb9$Ql?w_cvsaAY<@pxWM`|?&S(IvI%`AOKA)97r zzdj*$$(NYdWkh2H;<84IJYx$xTZ-FRiH$yViPYV_w-aHAjyz;~Wq1Sl;(~>CjPEFA zxHLFbE$_LwxQUxcP)y~T!x-Jq0lXi!I8jRtY{-#ST}8oCepQNik9j>qDKST>|jaB`f?Uk%%X; z?Zba-0M@jFcZNYV!A}SAH*mdGKX|1H?DHRWEcSW)zi`QK54@g+dPoPP_dyOL_<5=< zq8s~+hZ8&0dJ7&A`Ik>K14E#W@dD$K<}^7q{~HUxo{|+u)_yXVg=MY#+Wu6QX1o8( z=c8n+m^I-e$+F5-)B}(xtew}CKV)H+hmsToF0Xw*l$GUxU0N{R{+6wwDj{caa(~!0 z9r*8xER@LnK#Wt%I&FPsK+NDvc|xYi*gAV(#Ngn+8k%+NX4<+}Xi3zD^8VzfyH6#5 zL1rh*xj9VQOI5hu$QN%-H!MZwobJtdw;h@O{`m{@^@W;D#Shlt<%!|>bEseM>!R=M zUPHzNY`lrnPJW-?Snsug)}o+KW&_XV^hiabHv<@|PCO!zSY%2zJ`ej&2+Oro$D5im==k-qm>*e;Zxo}k$0OM1V=D~hOyX^Yiu#OeQFFShp>IoEJ?J(J3RoRIg~-^VmqjlT16dA!ae>se@E}kQRFX;_G!Mfr%)mU<5AI!6uNA_o9e5fxORKxrqlVz({Dx$N0we52J znCGH81Go)V`x&Z4^P09XKDsR2d*lhhKRuP%)_PN#RQF@Ol z<)e^HU}>0nSdA94XOA9g8AP>EVN!}cOWIcD*6pG=*RAJ&u3Vfl1~rob*`(X@a82M; zl~`s8x7$c|AQKC|;ncETrK%CUcmFkfiSI7M6oIt@YRojMdSOU6^Pc^TMec+Ju0HOG z2{42@u|GPQgk6IH+g1pCVolxp0;}MyVZVmE59KOJQS&!@l9 z&G*|=|JkMgP9?XRn3DfaeHy1Pr3NTuJ+!d%;@2h<)%r9q?f&A)r`5nnO_>f(8s+NQ z!t{0v)YkLgoD#cO9<3i|E$((p2nGyPl(O3XX4fp+dH}B|a`0!;$Ak1&-`?}QGGC-E za>kj`Wg%T-5@^x%_Awq_a)AL}{gYPe_}F)~OZTBKhm0cuW}BAp!bi4j_`(O5-y?*n zbn6`Ar*n57T2TpYd4!muA1E4%q7U02kkco4;Z}=Q{5{&Y>A}D)d9ZFqufo8_WZ9u@ zpM`)8Uius#F?RK`FpMH@Z28%~yNhB199BJK^o+rWjE(!d>B8g$!EfD{VR{9W)8%V# z_Q7X)EVp#*&-1=}?@MF?D)*8lirXf_PUFMS?C*pwSil-ch^g6t*|~28tmD;J1XaYk z?45}{ck$d!Z(jdY&9CH7`Lm}rtzB}Zty1Iytf`>^x*!5sYP-kw2jkgwPZ85Qh4wlF znUhLhdt-)tYF>Ab3S-W}{K_2RFugTBRb;k+k=RgaJ1azdV0(eWj?sDRo!Z@ftZ_$^V{iL&wc=^GfwP_NfSlIcLc#E|;YP=H6!R0= zQ~Tgtbntr7jYCIk9cH@eNmm~|ZS{4vS8?<4)Wf}Vzp}buX8QnI6m66tD`u4~)n>iE zf{JiDGjT&s5AEEsXD8PiX&I3gTne+C6dV@H49lj)A^M^`lj}B8n zn8CFhhYbh2%v57{J$LK)zkuyW^HR~FAQ+R?ld44UHFt2xe;yKo_y&a_Sy&XT#gted z&H_%@bvo3EHu@UahIlebH~I<`u|cI<$YImAmf}}VszWo8({KVe{Tfj&@Z9}aXEpjY zYk|RkIb7zknRu&_K){#PM_gVp}WoP$pv=UF(W9kx{e28{(_ly-$xPCTg) zsBG^2VnzT`do<+^mqY3O@W`jUHj;@Ic35ujm5I^6V;J-^xPLSD@jra9|6qjX|Hn|S zjmS6+-o6-pxhqQO(BZ@O3U==9=$ab&kO3%q1}?b*G0&bX)%h$tiLsDdY$OXn4~{E_ zs6p%fI3o+oTPDPYq``vdrKJ@nCM`8J)sMEam+jvZkG4D18S8waFt%7Xs>J3Ep7 zYb~4t`sWXDGJcKgryJ|@eK*d8&6cE(j*l1GcEw#Wt(dpgf#R58eat|A#cmMQDNlA&*KrYte==Ji^B3ak5-`+x{j6hokXC3?VVVy6z zWPN>Iw2haScd#kYzqY34x0t^MFyDg#Acz~Nu@L|6UFOPUJR-#myyKoq?zP?^rMa8hf;T{5k=;-R|YM(#YpZ=rm z*)mXFL-hXI{KKuAH!n#_>h<;Yy@`kz%F&813y1so#D#=}YzTwzzkU0L%mx~@<5B7T zwlkORUTbP;ptG21I|Cs^V|-@ZrO;roeii zm5Bn-Tty4n(9mFz`NttTon8QozDU{JGOaRMW_5-e* z2#NU*n_sfBvT9^(lVYCl z^HFo|UA9k7pv?<~Ml-H46&GK45dZlFe@3<@53Rk(ZuV)ymy18{RjSz&@qn*wUD z7!~Cg=;vJ#6T{9w*}H(_D=#Gce4a5k{7W}z{97_>bx0O!fdn*B*wSlD}c z%#bu(tdB3T*w`F*F9d82^-b#QJ6OZ>lx@awEnQtaXww7;hqemdGaSg%#ru;R^Yn5w zx%ie#FWvu;vAwxmDC0cbK0a=yoh)mZEbHnu*Ugocb%5#_aQ)^v4Oj~BxFcx)1Frq$ zMn(5C*^YDn#@F>rOBg4Ljew#}L+1Wx_)(0Eg@PLgym&as4hDn>1 z?N~1rr-n|HvPXe-S?X<_hCiRWl<_g-urqLDem;6se|fADK>6^W8(-&g5M<|-&lsz8 z77-P_BM`h>3K-+!;lqcstY=N1J$q(a=T$HFc|DY-3N4^TIsJrl-YI}pe8Xp%>HD^7{ZlJFIvaj?2+l;GKFK6LLImHb? zdTs->f`YiLtUQYu_}k{@rr*Ypv0<&JZ6oNPrFr{y@0-&iK$>m5qz~{EsxB216}a^+ z)NeM{c(B+^4it*NIlp=WFx!35Ija|-5Ml$1TF;jG+Xwq`MEG110AX1uQwAEg<$>BO zhp)}g)HOOefn?=?9#aHG&zV=;vM%}M_PBqZFY_+joYa}HTYEb+h%aPpJ5K>N8k)aN z*90jw8Q&Fi&^kz4J4w0^^nVJEJTOYD&6ilIiPcknTIs=z+NnUAg$lJ>O1-TkUNcU# zKv!+g;PU0mVAohA68X z0C~WF&QZI^Hg^(Ybnsq{v_tRNO4sq>d3o7Pg95#ztJW_!$pkcbH9kr;sBMev%zfU#9z&Iv*F-}nyn{MRM`RSob4ycha!ft{g^ z!XvGte`ICdm`iqd0(n%^M!(8Fj{_vHyq5f(qpQ9BE-p=Fc4?^yw0i2ak>uj*umciv z2jo@n0>*uxl@$?r;HEw{j(LU{`SZxh3h;79Bj`Qm>Ek1-7>NZfvhW7Ecep@E$&Tb6Nis4^JV`l-z&A&JGs8`^O*8 zf#rM;RJswso|GaxBNt$YTgj(rt|zr{gM{b#Yxvotoz2P)zz%ypehhW;i->$)5A~-T z%GKsM&aZqQXbmv?fd5VdILfWt*aLwlHC-0>KX?!`QiPs z?i>9B12T`lUORN?Pyy(prQY^2@RYdueRR_{$_|^9^Xk=4S223lZ^y)P5|3$#$3vb6EoxHJH&jNmzM`}zGz=UVEp8> zeL$P<#yqPvH#bi)DlsnrV0BJSP0h=1g@UTg%&vF|=g~6Tf36rV`Fnh_ihq^4g~d<< z0uI1DaSC`&VrEr?pxaq?clVvY55xmlI4mbS+hM8!0UDs4#QofT7$PxnKR{WV_CHsA zBl$b$(c{MrJw3~SR0aj((t&8f1Euo@_0T_TZ z2-2x2IFU|im6De3QjyOBos=|4w{(Z1ARr*!APtk0ZuWSX`2TyI^`EuZb@sZ>I{R>4 z|N30;&iOt$#y#$FKjSf|il8fV7!5M82B1d4ZwvE|AG}H)NV3`i`uqR|-O%n9wc3PM z@!8Ks!WM#wW1ym7V`nSn}&={Fn~AOBVVT{$c@uCP93$FoseMZRVa~H?#FX;PGtp z+E=U!KyiI4tDz-R@7{D@i8^#0`|Z(nYHe3pc+YfC0Rn@Ze`e(}%*qc?F66ju*uu^^ zm$|*YwXp;)Eg5Pp2E#Ie%5PUOnR$6CP&D#}>!!Uwv$`_B-3$~B{dzUkKPV_Db!Z+7 zUNkm7-gskq`X3|zy!*LT5V#^b8k##`lWyp4flI!JU0<1XYYzh1=Dy%s>fsHu} z2Kku3on2gVtj45s44W%I;kSHwE^K3V=%60}eu+R9)9>rouV1&cvs1}2YMtSY15;w> z=l}BM%kwuLze8IHydSJ3$$saBxAOAxH^6OS466|{pHIy!4;}Jdh`(h z4h^h+G?(D2nvBy+;(q=56}oz!>aK{0yX)G>*@`y26Up_0<=&ZP-~;fK*0*}Gqg`53 zQc+$W3kYxi=a^Q3jmTjQclqDhYY{_>8tU*X9=&{iv59 zN`%6n%~+ayi?EVvs;b63Djl4h5MTj=*#b4-3^lX%d>^cnSTKy^P)t+7*|`{?X9B!l z0(Urx&_J{LBO>k8ewc;5U*9@sK=`e7zF`#=#kI-TPMB2X5(KU>e0~-Y&q8Ak03Lm<1^=d0@|GF^#cE=PEkH1zfnrGa%9;rEl!RF6d2_L8u6w^92{j`I^(y$AnS17?fnUce2eXW&nvcx%+B zPoJDhM{I{G-xgYqynFMj^;33sMYm?`&R&m4CzAyCg&B=Wh z=239h#AHg-JkKRm%VkLk$cV?V!0nh5muGsD!QCZ*>dMZ~W@Tt>?3%c^_;hd4igyD* z^Gp2v{9p4n=*bFE)HR3EdNsv~)qejaREDrMX*#-^a{I+~BREp@iSx^m@ zx4$DTE-6{6qH-e-#-o!C;+k@e;RfLyMTpP|>wZm4Btkfh`{H^0*wLenP%SdYVL{)t zGxH7#f2H8i~#m<0s|TQhXTkvjx8z~?X@gLd1&<7AH^yfW&{jD?|~wATPL z01-pZ207WRw-7@1 z?}_R29)3)!@);Q!{~STE;P0P5KLFeSIfW>!k^nC+FQM&K9f*yZ9D8)x>Dkz10Sg2M z2DXbf@INTu2lHVBnW17zOWKz*Y*3w2WDo{ zA|oSB&Z$I-><1Sw1q4RXEE<{C6xD?&OCjRCE(}$Ph0g_$O&+1eo*2@D*35<^lAQPyDQ&?E@DC04J1B89R1;3tIAP3Bz3{Yg{q5_~XCjP&!Ub%9^q=OAUbQ^KTzP`S;Qyo#q`p#r7kp> zDsKt}z+}tQ-Pcr9RFKmX-d>IVQc-~n9@twIFi)~z6H!n~bdLS@WP5sEfVGd~Hk5si zYu9p;0@rHPS6uMwH2p{Dl@Y)~;GUYH;6~~sH7zX;XftJS4zPtd?%cUE0hM}{0POU+ z?YLm_UVTFV5_UR}qnTM*&BYE2mQ$*)0J#B}GiXQkFd%osL`PtC51|b&ktB4N6A|?jHoJo(eGA}T61x)?;Jt!jm zh+lp^Y#OjE{*C~a!O6+w)m3}2%=+D-iHV6=YCF4?!$Q~i_|(7?93v$BG0(&g%R~^0 zhD+SL*9;3QwQ725ss-qZ$ytCzSn7j^4(Zbuf?Z~wXq&EbTcI1X1AY%cP%hvo;M-{d zoYGvoW}sSCU5x>J*>*OjqqCF#(j^fL27@@IfeJ61sgCP5<3B&E9gxVA=z>7DZ&67u-Z@_fkoK+ zJ18PTJK#E2G3EYsK(1-TW3HPD_R}kdH@d{P~<$Kf%jv z))x(-rTrGA0PbyOy3P8xL$KIg!l!A6_392w{IR0d)zu=`2K*-A^*|s31_4rLfC8&c zLYe{G2nh)x2Gr4UU9jqom>5)I2ngo4OIxC$b?c|F>z=+cGux2QC>vR>IaXXkFze>v zx1a*VB%n>h--F*EbZiHkmxxHK{rmUZy|J;hEF2SnXb{3Of9px++MVgLdtayC1qCI- zZW)R)8^F`n_NO6Ky9=O0F-ei0LfBP6eQXK1Tv+g6{bqRe|B0_sX<&yO1|K&5^D<=0 zrRFuW(Ah<4Y%#hq8YxPBD-`%r~@yV1a2RtF%UW) z`>W3u!flAf=;`R_)H$|E^CHqaVlbq<{QT-Z5#V(pydn|QhDjRv_3Pb;=M}6Uq=%qM z!j^qIwu4`fLZ&JUUu|Wd02d5_WN)Djl*{l>wQ_WHG=lS-g--^aqF}>m*#w2Z>bWM3 zixbV9kG>uOW>@M1+fCTQ5Y{MyH9A8|8g1AdHvvGA&j#m5#&kata1EM4WMA9f?A!*X zMCsFmn_IxAc4g>zNk<5!!X*RJP{ricu`>)YetxIJ;FBI>3-j~xuyw7Wfh8L=MJOhb zlzh`?KSTs5Ue9S8Ol~rSJ*uxHcbAfT=-1=7ErTE`M6?xZF~t1=rVF`dy=r^(?XJnN zfQ?^XF7g(ldiZG@0)AN*To6i$L%g8wQ#SYII54O8b4MIc*Vor0Vri+Ke-^Q6*cbqm z#xNcXvXSm_l0hE<+8IKhNd(3Kb@>GcC#9Eds{y1v5W=QFlmqM^;Hyl<#BUPw0F{JR4mdubxprUE zK@KqhVsv0wn}C?swLU%U4&=WaiX>n~zxBrFdUn6AwZO^%uqzJk)PS$!0y2HQ!EHMl z!i@X`=I`t9Y z5j`ho>p0JMT0~dC5GMmkPyVNcm!O3ya8rtel+;M4$J*DC+CZSWx!@fQf0!~cfOwB! zVICtUZUZ2vhXe)B#IwzM`2b_zg}~?^yb@yjQ28meLBP|u1IKy*Y{@Q7ru+Q_JpO&0 zXLn@E(U)3+DRhJwAjhPW9Wj#5!1Y2qT4Ck>qbKCR1+xLlgE=$?z$GUBS@<1TZ`RV) zTh9&=q1^79J+|8fJ^J(0Ls3riK8@pKOusrk9DYIq8LcmXKL3~#9w4Fz5g8923PBP? z*=iKq%b}DU#9il^Y`-uGF%`Mn3me!-Y`v%-F{qB0pu-; zzy}yRJcs+4fCo54WEEk?{kV5)0_}6x07yy#3$-06s|41vO?j|@h^>x^jWq;>gGf)v zmKp!ypfNzM!(wtd4GAG1q4m2!xxv$daN0=%l5ovHPybmv4}h<&`XEV*kT*DYl)-DX zcybnOK=a2tuNUEFibRb9%Rd}H;>@2xigNMuVJZ|DBf1kMfk%2IiOFU*!)?wwZJd@3K%d6 z@bilSXG;�{8%+xf}oZKs4bGFEp%>#1Y(DY(CGf`!`4j46S3F2Vi0ldkKID?Qc$# zj-P*$RwQ8QB7m<%$^ex1gc@n%K=xhdlMg35do~_2-k_93!Ag=`^oz##_+g+7e%84E zofU{IV6p^&Ef|O324UVsxO4;XZr}K0hZ=Ae$WeW7Xprf5SqTG&U}TLCXWiYN!=Q)4 zf`Pj~BEcgtf*1e}T*^V@(3mm*9EFlW9O6KI%ESFg+0dU?BD*hs2LWf--|ya?0D~|I zD+^(809kt0F&fOko`Owif<2G8O_XN=Zz)!Vx3TurejZ$3`}?(#DTk2jhUM^Gm*q|a zz`q=P%4u0Zg~J_9>JX2%>;@v*ZoPj7s1wfpR6?5WG2jHG11Q~`zXw449t5}Oqfa1f z|CWMZ8P3u*zGf{=00tMKfIlz(4)}#D$-Ezwhc6oyWCUYTjDt8`==ED~y}uV1ty5LA z#J~@s;e%yJi*o!uWO$Q559h=#CU}@L(mIh&B0AI z;VAc-kU$3(wQ2l5ex`UqROL8RFlG(hVaZaPI(iJ04Jp9sLNHx1{+O4Rfk6T_2(phz zAplX8;pXaZSP|@e=XbwD#5;=cId!!Omoenr5$XXVY6(i}B|}u&_37a=pyLg#i%k|F z7BhrAGZ0*IfL{RSB*9QBgWN?-9G{W=ZULf(k3hj8x*Ja6BE`=n0OLl)8$pV9wA0{N zA8>&%Kk^VvazfG%;%ho!pi%Y#EWlfDJWo5? zrN|b-1yc!#bo=uBC?fOG%)#g)?#$A{;yPavJ3Bi>+le6bFoNND<>R9Qa26=M188Xg z%f`)jWq0u0qv!~Ji{O<`36K)NK_l1aYgtj|1_4#9VyYS^gj);{JAqO4&C0U!HmLdW zqXp8tNImoHFo_Y2g@lBJJo5T*uTUZ+H()P{d5_^|_&X!K;U^?PZan$5qcdASl*zEL z<5Oln6GCzO#DVLQC*-fM9Blqhvl81QB1O|pdvC)?q0GEfGN4(PYve|AjCQ<>1=ClW z<{wQxQXB-|TnO%5KX?cDnSssiC&lqjKMx!<8z>IM9)!rmL$f2&8N~Zr<0>V1jV<3>g{dMXYM->XgjP(h#RQ zIQR&IjfjYVd$5wwOTh=>envfm{2zcEhLy^__g?xU;+G622j!_@HgfY2ZwO6B&Xb*k z!#^x66=+TdR#q91OJS)Y7D$8z209Smyu3WfHX)wr;DG~l)YP|;Dn+}j6ku$ajWK@q zlqerwVhDwdCcO3x1%*qU5QK#Q9kRhxGiI>kAF{Hru*J7#-#&)J4=>?=`{2}b5B$H} zF(>fc*S>wUH_7oHeBXxy|LVbxP)(7C52N5bb*Sbcv;74H1)D4%dcgFAYT_+rYw$NV z!U|R2s&C&6fo7-=rYvW!dFd9g`=*3?Z0Pwz@N3_&2k)qnZUN=8!JbS_$v-*%Yi0K# z_haveTdJM|2MryX{&DKb@t5A-w0L*BFF~YO3r z*-+6y`1wEWL8ccE$?JnR;j*$epz^{v1VlgQ+? zf?8X)bRhQvddOP;T>e2I^TfctgN)AFmj1~{NUDy5d9|@c^%nN+Q&gAkJx6}_>@^J7 z1xw|l`FDW10*;zNNT{zfw)`b1N1_2^{Lj<2S}(Hu(S85Rv5qg5TGdBsY>3%Vejm1_ z4cvN{0A>f=txQ3^FF2#Yl~8yB261F^(n-NeqZT0B z@Wh1b%p~3exF5;!cDMSRGAUoCsaNC?OzvvN!NXQa{ zpuwV9CrztF4P0!&=o%!RQTTnH?LoOzofRkKy5*ldVS9LKw|njwrRu8ZLqkL8%afqV zT@cxU0r!Jl3L6n6x{&Dyrb(ClMb^ENxU<^Ma*&hH+ercS7evd2keG;uyfYHOb<*Is zL}YNHB|iZobigAd0A-J0wuT1FSP)kT%bRHxAWcu zR0TM3@MjyucW2I?H3ljG)RylneW=_-b+nJ(WG1VXYRBun+;P!0(}d2Tx}wG?#7Pzy zMULTcNg(-9fea|0%X)T+3kW0-58G1|ejfc|A7>vrwoC>zif486rG{EP(*Efy)zj-8sk_ zp{wy=6}vW+TGa5CE7ByXr@kD>6r`#Et6!!c0Zf7yg$Ek3bgdE;>p%icBRo7D4G>f$ z`6#?KA>#g#uPvufl&um(Zg6Duo@<*7~`@)r+pKx{O(vce6*r|B(7Y8u-+IGn@20T^-~ zj1DN2*r?9|#K~TPynVKh9T1qtZ5&pklA_q(UyieqgScWj0+8ngm~&1Ou8;xH(1!9@ zd;yDZ54VyM+S%C+LDai^9nuiy@Q*5BnchDne<7!+sJr`Wh);mwFqq8)`o7{(#_k&l zE7*|i45$idAUR|T*i)2~8zIXa9T{2fDy2HQ01YOl6nH!v2!g2mf&yqT{II9`Q(wJ) z{de%>A*;KRk~cy(4R-f|JI+Mf-%)ZEoHaHEa(#j8!H~y0cs)! zbFvL74j~Aw+2#NRkW_?_CJ2&IWC!@22D833>y{qjx?cUzZV*0!wgF;Cn?`!EC>`Uz zp8x=$;cPpI%aM{2B{3^{;CW9C+kvn#n{5ZtIuTUg=xN7G zr1p(OO9;|lBqO7R#Z4jTnBCGo1&9a>V;_HN43;4c1~d-o5KPiw(B+hkv+b@Fngh~ApWp_{3ARA$lHZjV z3a4|sQA{}w0W4A|pj!M|GB*brWO-OMHS9z93WZ-H*O-iKCoomuFuM_ z;AQVPhx#Kp4t~0VO=jRv91RngkH^Ev29uwx!U<7^!Azs;e7O z7vS>A1W4$^a5618LyAlV$S;b~dl*c{P8s{3rPHngvmntFv9pOX$_n5Jn_zF=3)O}T zJWE#xkE5d}NaO+37N}C9PnNs@m(IO(b%(!8ruHH2}DIj zAw)H&2OwnEYZHi4Zgt^;>PYAt?uesDxiO@>LzyvPrhooKsSIf#uh7;-vR+7)SKi)k(qKs=<5r26Q#nAmyjlrYftgnlO30g9WbqRK)i8xJ0 z*0llCN$w-K{6RDVl=khbpzI>VR01(G?E+Z=(+c?MYtYX+2gvHWbT}07wP^_DZGNY_ z*}t=;oj79!MC#XJWR2~eoGx&M)Pc{)hhsCp!Mk2iPHyeXgP5u^ zl-FuX2M%sr&@8mJAa?^E@&&B1lVoK2@Rzt26#X^Y1HyR&rVZQx`oB91!aF*Wz|OzZ z4Ha@Jlr*NP26w>J8XUO}1O7j|1#gHg3w;ZtCW$JbK$jQFCZl5;kmw=y z$=l9@T6#!a1B=C|EhPvbW_s5wTvLlqB!F*DnVXw4_O=5m1(dFI5V(Q$0m`m}sJCh< zRKP(VR#cKgyf;1&oSH_*W^{F9O3YndOJR1EqQ!`SB=NUbf8x+^P17~%yMK-|T$G#* zvH(y-@-G(jseyI{564S?EPY)6z+vDG)v*|^ZR2Glk$m@(njT&X9Re-@*Oilul)aD} ztnCYrh)9QRl?3@~ph#{8RZH<_i8qXC5N&aP z1*#5VU@f;T8!#C)?d>WoEJ`yYE%;~RZjt{TGCDRkHif4s_sJeJ8Noy3eUeYZJQA!S z+n55($i=QJSFSAfInAO7l6cx}qr3m~`SWT)BX}sNsQw(D1?ek9FAR<00Rf{=zyrns zMWCpxihorrrO(mmCV-=xg)Shu56t0m-z{j@b7mpCgO`yX&U`n_3BOo!2vV#7K zCwD*X&U@m}7kZHR)2OJl8;6@W#k>RzF8}h(-s|0-dpvms{Qo3aJrzL&ve6a9SVnwT z61I;J3;^|V5>P;^QW@zSJOyCh>Xy=8(=Nc&(E`|nMITiMS|i}{QB-gBYVY%v@cA3y zF%>K+?G7lhT|>56PuFhHP1b;H|E2gI4{LL%c>PQ$fI(fjum7Y zfgZ=^Kd;;i+oL0(!r0&H2X0>4ZJWrqtN)$sz-@mnw;?{7H0|lBJz^&50F2c>v&u#D zQhBYL9xrUCAiT#9qnX3hziKNbsaVi7kMrI11{p_rCuPqM8^>)P?NE!*CZF{D#F z8dg+`v*e9%wO_x{drn!~j=NNWqf9*|hd+<;lNb@vWQVHE62+ynufIjb^#*w@<=t7x z(Qmgc=r(!v)-0*)jiVx^nVEknD#r>xc?`gKRg*Z`GR(XcuC49WT;Vdkoc#Lr_Db{0 zX`J@G4cF?HVuTb^ka8js{M2MZVk5MSK6V=g=& zQxfNE&@f&~?>js;oeESy4)ai?e(JW?+QEbC52j|rK1u8~l@^++mho_WtwHEr5r7%V z?bl!adPa@Waq5pxW8E@;xxL}Wo0KM{K3-wbK$j$~iD@b(n$)$*U~uRnTyt=^hoxP- zkU-WT5EV-wGW{@;I@W-qIho{58)?bJ8W1e~H2Ty#15M`(%kSI8Mac&aGNtm%$y={{Adol!>9t}=12tSp;ft8!t2Zd#YO_Y=EZy@mTeuiZ~%uNS_UqO+XK zx1BR=W@R&OM`QAXm5FY8de101O%dKs3=;~xhB-F(o7o~QAwT~%xE|w|xo%9V3A960 z%@>aQS|=vl4Vy8En<#UVxGwirVv#6uLi=M~ZLcVHcmTJY%(Y_NWD&q>Rb ztklpad>j2Qeqlkexmz5&*kb^fn?EpZoh!U}h{bV0Mx|tX+m_<=gWyzS@Q|^KTJatjoP5zM3Ug`#>|=G6jEW|FSA{%H0E&K?77yP3l_~Qd&@NJ z#G#KZ$5=Fex+rMNcOSAFZ#_xU`K;W^(^DaK9i< z{x^H#z({PMmpAr>cTFQZd`J)rclosE8a-17|A_6tFNPQAV{TxVv^(@i7g4gvggN@pHk zVyK%d;^!~gn{S1ygwK1eB1+Oty@+wq;8tLBR1_Sgs#mOh$>j2TA?U?Ylh_hOpxyNbC`b~zP75lWmunL?7h>xEvQ!Jj^Uf1gN;s-V9nC+o2W_P{rORW!*x-2yx66{LND{=W8N2O>&e%?Yx$%)wC=WJW@Ke&dpPyhLQoOb;X<|CsGlc6dmdr$h72|j*_jO5wZ~&8{?{A-px1N|n zMa39tJFe!%e!jSxlisq>a=dW!@f=B!U4r%`xtG zI!ivqHZS`Z>hBJtdy+TX37YJe#iB&FZgJ{NTyZEYFcy&%-zZ4ni<*amvrpen=G@TB zBEGnBC#yY8u|c@xlC})%$l1{LZEbHYJMY5%9*)`S=F^Ada^!RZ?EVWCfWsrQ+a&4V{bIj^{H0_iTYuu)SAL6oys%~tBq~9sIu5xq|zbF&>@v$w~EiB}o*Y&0*GtEXeIny^TvpZGn9RjjYw0A12lXYgKLRNjPrK9t)stP@?!zsRnT?2FN&mW zN7>1TvQdR}C+23i=17TD-p)qR0dxIQop?*|+Y@v5Ewny6#8OJAxZd>mSbFrAqsYj~-N!trGU>tz``#yMObec?hyBLK zh6WOeiK1&2(ap(FX40m@?mwG+|4(P>|LY$_d4E;OElck{?cb?cwYyEC;8kWo6%?8b0SZ^gFsCiHAxZynFYq zW)J;Fs$C(x9ebRZ*!IX)8pw}rc}k)B`;ou!?xp4=BG!<05^6$mZ{*`sY5od$sRa^} zd~D);{8I$Z^lj9MAnprk&uNtR_@^!4pXgEcJV!0vu^u>npiYmGkfaYM;TJ*@_ZJbK zZ$PywX=pgNT7e({a@YPEcw8>BBo5?P0#K`{?9bc;N^ypHnnV>B`Q|CyNBCD(3k#(V-X;MzWCQokov$OhoAqJQ3IN+OPl&p zR%Z)b0KHeug9Petv$#Q|z)gL0Wr;ifVR`Q)0`3Eh!s@Af_Mm7a1KFWN+=R-*z0W;b5tMC5d7$Z{B&jMmf?dnk*GLSd=X0)JSY(BvNuEv*(Xx-926 zq4UYF51tLve9;_hdbb%%f3GFRfOPH98eb9q1iyR#ZSub_G@02KT_~)ld?K=TZQ%?hY1(ph7h|DQc1gq8p|W>%y>T;TvuU2JEQ;$s z6P6hTbTIUX%7pHlX_*DD^a$vg)wYcLh$V|XA@brqb&Zb_(^T!s; z`Qgr)QDI?i&oxh+U`Ad*S&jz1cHDTYOS%NsdsZ-wb!Rn}U7p?g^6b-FA4OYCZ*g*I1t*Hz5=W@5vcMxk5J;)oflWYE1AaQlSw$wSHn@jl^g-7bB~eJ=%a zDB6_6IR$c@zL}M9Zwdm}(gf%adqW>8drT+0@+I5yVkm-#fT;wma)*S%|;4wo6*&C zCH#w;6ulOTd7NcZSijusQXF1C{A411_}GlRIx3+L4r_;THYWpAVhf3JQb>uvvzPXZ_vtmpQ zImccRHnA4D%1MiPKRthdo=xD>ps>kq=iNg@$nO&|1sC7tJ;&?l*x6iIvNVm2>1$YN z`>`6dZZxlA#GjH{yjenxfd>lhHcN2ai4jm0?9C(5=9#d#lH?T$tQv`7s;4iRQLT#l z#V_T|$8*2aah>M4G8$~7JKZ23o%VL|mlp{G55xTz4iL-Go@ygHCI_+jO({mZ?b;IQPg%_1p3<~dI)ribu-70q!P(QyCkxmcQQSz&Y z2=&L1PF$NZ9Vt-Otg`W!(;fdMbGQ{&Nvo;aikR>%eD5SacRx?gw~DW)cQDJu<)j#p zSZ5j!DHg&&HG*LS~ zsbZYCruuthR0K(4cXO}n!U&a+hfq>`X80BlKeKw1+pPla1=|FwdP!1PqvtKK z7>$HWE$h+=#4(xoS=${a24icjl*;$gr;0RFgeimbR`WC%F1rv74b6+L#eWsI@TKM4 zI`7%BQRJAfWlUF{zbWHODK)Ird{3Rs>^*fX)7|&f-vpjai$ABS9v%I_JQh)`p)OM5 ze#(XQj27{@hQ{^+gwLwtt8E~m4Lxl_Zji?lmy~-C5_be9BRzXlQ+*VHOLW< zy?Xsh>4OR#w@e=UTSKe7#F6rVuDQ-bBV)si4z8JBx;+E!8E0*bi?%FH4KFky^X6rB z*o|*o)=e+r@0`7)jYP+}``-?F92f8DIh&Zwgx>Wj{oS^={yuB9_Am+mI( zr(i?kL(~W#%Z*;&ytR3H<8g|245eIC7G_R4HVYORvw=o`d00 z>So0z4Mx~`{u!7N zx}Vf0P5;bQW7m5bykt&GqV03xeDqGtHv220)bIkeNns6h^VwBdaUw_4pG0pRIBJ|y z&)b?MtefJDx4b58oM$cGZWw9Y@xxc+mt_HsCeO6e5eo|oEF8IynOp{$xZ_;K;Z{qL z+E*;eH4Hx(Ti&-^W%$9W_}DtZs4aD33OhBQxnQlkak8_=D2h*b_TGE z0?q=%GgeE1As>35yB~$TwW720ra9bz{_irsP{@OGaAqj&5XE_(qBqI&N9X0DS^QVF z+}joUj7QE6s+jE+ol}IXeE+{Hfc?Z<9n9Sd$w47eFOcr@}&U%@PeLE!FI*{AsJOZhvo%dd+0oBjZUeM+ZTPwai; zf3F01j_i|ZHCL>2ki21gFGG^!7QW3&Zvpvhh_lxJN=Gkms_Tlk)U?YM1Snu~(0&qHZc2_D}R)C>YCG_5a2AQTq5yDTHRyzqFM(d3Hp1m=2)TUljzkiWL*l@k_Q5dH!}+WK6ws{0)d?P(x;k+++x}&P7Vg zSTop|{>R`)ZLrj5h9`-ms$sl0Y8E%{9bK5Z2A3gazcmcz)r~{G_M!FR0*8FVji%}K z@y|XjPLIr&Hr)hkv(19KIs8L{l1q}8YnmGr^ z^schW{51AggcO?-krqNvfX2qx1t^pnk1Ld#3@|Lc<9n7K5UdQVlUiP0sV25%W34av zoUnqn)V2f~kiasPL?dItm>e=`wFG8uQ;Q=6c}yKwqB~@N{QbeW?R}2$y|;y2P0y%= z5`HB-IQJrIjdZMb)U0_mb_fiZ-J0u8e6oIJU5j0FsY3G2)G7WIi6S$D%=&~`s`aD| zj$Rr}>Y=_uQ0|C?!0~In3u$x? z)lSnO6QE5Y%sJIF$2~c*;3ag-`&a-q@^<36lyZtDeu78*sFRbGreEYKGnmHggN%&1i^FW0dZSv>6B1x}X4=3i;Ut63v6KXIH$QYs6erl-4dH+q}5WSUL+ZWB9G9M{1GtJXZV}=F*ilAuDVD4)#?x1TtqdnJjtoz~lsz3(0w5NdU^^R&*z=%4d}BY^>Du z1gwRy5*Gv%*5PUZ9z!T05!#(+76I7Rs3k`(sH#Nn%pX0dzX#i=^S?q;+W*Jw(EumCM`sD79(1;!4kA`s36cYJxxJ{OSqX%R2X&L zT+AW8wayvujrQq1yiQnQn1225*ROrifhEnnbhxDg?dJdjHDzm6C9RnZ4Tn|x@4aI< z*6LMePX|vvtmAEnZjZesR;nxHWQ_cB!8b50{%W_+_fdU?$G)--t&5gMf+Ho$+zSQ0 zINAe}vT)oM&M>)U^@H*c)llO7-EeAmm$a&$;wMPHTy{bjBj1$OPNEuHTeO19OJ&7a zu2`r%BPL=<$_^PM9gY^$WSHI9pAHZUSM$gA$sHj`&PeD;kY7qZsS*8@5$_nvQ{7=6 zjV;_dwL9>?amT&RJBlCT|K*veJ+5$Wr?Y65E*WR#Rvz*27=`Zb{U>ApzVbEgq|@t? z=qAOf?ME$--&{9(befd(jhxPDC;0d6;^hlV=lKI3+&ppm%!wkom}hrG`l-Ga1ie1^ z)LwUaA%)X8ENjJSc_ZBA{^oDh{t|AHoA?BAO{rB{9JWq4% z7GC#v@bYdx*8M5Iks>!0{T$t6e^0!T;aG?cilyA`qViovMRNOEOrD?PE>(8_VJh3h zUsrtuOAl8PoLNY9ynXTvl5mb}>BnQN4}Zk1-nfpWUjFg^?&P&R*X1^(#% z=)|G_VS*-(HY0v>m*5*2rYde$3(eaAm#X0pQ}lv!>b6k&^O zU4!dQiFt?LG2Q?4%W6dlPG>~8EMws4Md?5yJg^VP{qPb^1_-!gK}s4?WDb>}cvpWt zD;s#Z`&VK3Ya>)@xcH=ee_~3ChKV+OG_%{R$o~Q2Mcvn>$nRD!&h`t#>G$NURCqa* zVvT3iqD^X*$tg=9ZL{WR#Z! zwIaGr^E8jfqj5_pd+y<<2t&dSl5y(SEGLEFz<~lt=LhjEzXol~`9`K51JZR2ndX(A zJoEljGH6-Lx*a8;PlS_;gyd|OFijx)HOcMNrwQME>vj^%**&AVU!P+sJw{SMJYN7c z;O_|@s;!2TI00}t#c#Ka23__UT0((==cG+Bs#%K4oJ>n}OoySh=av^a92{4(VktdG z9&`=+a6rY1hnL{3ejTt!XseGt+kTh2(v#kz^Xa0%%G{1_Gld=f5;@bg4eKP9CDa6w z9*%eBM{lCfl+R!LzwFxo@@-%wb&;QTWpkqDM<`cLOEkQ6-zQMTCN9lqH4Fz^HT>}N z5rV$$WMnjuG`y>I4NUYEVpyU>&-L?t5yAOiZx9L#VcE2ybz8lexLZ?swe94g`JAN0 zM2wVFok}~i#wwBI!t^9skg^hIVkzC5>|I*^?r|NVImhj&+m+UMUC^%cDyH+1O;(&t zEz#LccAXc({u92!^@SlO;b9vWgQe^`3Z#24q`#@R84L4o7cYIcwlEr}lSesWH$Ty- zyD{6^RxCJ^qhZ(Au(1C9WDtjwe%t*bCt`t_VUvQ5z8p@K2rI`?XLiOdH)}DoqRm0V zqLuod)i0A6Y+(4gfNMLpDq7BijX%N`Fm1gl2BleR%AGm0?l`IBYvg)#p7hc{U{ulq?$*W=f4b*j0K9X)XE8O=q>Zk=XVVe32a8*FjoqLu3}X=;>Hp; zW~$S@XCnFu3%Gl^*37hAXM#elIy?;Jdijbdob8j&bJfgw%``+(T?}|8!+X_5dJ}`a zLvlT6zERj}=A;X|#ncDuwfaM^9R;>-*Yjno1aq0cH(lXnV@^t582lt|^{v4z)UA~B zx>v+?wqzSEM)T5T2_fgP)a=<7nrS|`fn0B9ceb>T5F8)xvTBgjQ>xH#Zn&yyv&4y| z-k@^2CEcad&1b_MD$GVzx}fZ8SM^NQsX_`%4Fl|0&s%JlMsx#7)R?|7HkJ$KMLN<&Q-vvgvBucp-$gr&BO_-D)-%o;iPjR7x>ji!a7QR-)<7~0TQyV2tl8e5z;#3t`>G~ss>@Dfv>_xUo+_VvsOn7( ztVihv&E*uLzkGFJKl_2}1d4`lJc_8)38UN0Sa@QbXb2C_E z@A<4KH-{oxj}9#huCHoXuZY?xv(`s&bJwY5|CkEmD4pAIr6@rz6nC7F;P`0R?{pdr zpN+P8hKq4ZZ|d?=5osmVy`;6$Z>TYs;mMTlH5lkVY+m}lQYr)rPb1xSN1Kvu$s3eJM%5L-MjsH!+1T zH7wM6`m@?f@n!>053cF9{kUP$!*g}p=H+xUt5AgL@{Cga%s{s}rtXWUVb*#u27A3C zqF$eWMlD`{>UaYSE>EnAHPoSN7_+vpM$nw;IHIvMzL1lA(id3s*QJ{qEIQ^e3-0S* zYs#OZ5u;B}_>mA#nV#kH`RSU&R9d%Df8JyVJ#(_U!Pg-YgU_1|Hb!1r78dKKKk3uo zI$wsB!=v=}JhN?ap0*ojV5p-rs4o(&XEIzF7?S5<8=eg@SmdZ^iYykmU|KYEwtf>X zU5?iu_+paL-DmT(95Y)Ma+I}fF}B}Mb@%4!mTOpthDk^ z9#u_TV(1+EK7@>g#&(th`%LUfht*|u%g-b}($D5!O}WmUOf+lea-y)crhRkHpRt~g z6WIy03%Rug=w$lr>~j0l?UW}wMvfbp^%oeQ=QIp{xYGB2fTTBKh1R=4G_NO6U zwl3J9K1J2R?n^#(b<#9HHHLC^GGbvp;bd(Os@@kVWD)daKgDvh_Hc7&)_gfL_wZ~U z)bs3okUVV(o654$r&L=#&niN6ej>E9V4K^(pu?>{h?<@?{X2^bN!{i6Z2>qEfWeAN z^Kv96R!^&F2XB~{3SpXzTDf?au*t>|w*`J>YwLb~>RB)*qcU20Aw79IW8<^<@P{n7 zh1dlyvo5A%`A66DvA#XeT{kNu$Z~BeQ*VZR1wVLMjd2CD zm6P-J`R$f{+Ot}5eb9-0OBhut_(=?=vi^A!g9TLKs%HwPa*?$%DH(IEbxQg2VR5@C zh-Z1k`o8gW_Qa>rvu!Ty#`0N3hA&G7aA_5wEM|$rNIx0#pQbJ9LNC_kqA#@1*<^nzE z-1D7t?j7HE@43G}j$=4RG%IV(HOo8S_kG^^JUoHy!BEGQsK#n(l&{uZO+{^L@Kni4 zhwA7|_2(!?l^EOkkz+4gx$#tOY8vKr@vv9qYMs8*9w90vmZqjsDIwF>tGY#Iv?F92 z#J0Q&Pmye~j#tG)YVwfuQ;H`zDde!eI2C+!3kP{cO9Se_-eh}LE}Na2rdI-UlF|Uj zZ8lfjQHe?(s*gt5skDUZHQgDS&i^E~RXLH9S@c172sOJ5K2aUBoV%ADW_C6W&N7>` zP>1}d5y*eLF9SW5@8v5^=6rF}N1G$}HXdnaG(i9P_(a!%6QDi?CnL7`VM9r7C;yfG zMmPP^_fOX`jlIZOJ<8?BDaC4O6wM&$2cP72cq9@o{a*%G|K(R}#Ee0^y?cqcPKf3S$cZXN7|-c6yw}lq()9Xh>30UBIxYIgg#ln zGpuP!NJzl2z!^TOwy-ob{}@f0+)Y_q?bY*e|sd{agAQY-ulNuY9>{muUI@e)s$DpBda zSh%h!=h_Wu2Mr`5l1WWzHo-HsE6?3!y-CD!a)lr?pHfIo3MQH8V-0~#W*7{*K8Q6G zTx{7wTyAc5(rckh()ksx&Lu+L5?7=I%Fwy&YdEXeX2!kvP1jzcC2g8O{<1ykHlEcX zB^Tcs6&WmCb`j<_FvXivXkK?_@u`*@F|VUpu!DYK{KSd1_6ei$e)o2@Bpwx(4Z`1b z^-|)Xij}X?Uw=8j^u95TqvPpKmu4x&#r92aTA{;sU01$F@aEd)lk(}-UZJ?rpQ_Fi_rcZorGgKx=>I z3fUd8<+YJ`nYv@ZhLU0*d9Yn^jsZSRm*&!G@P?boeZt^Pr#wi-Hem7d>ur52HuhY$ z^IoW_q)o-BVYwh@lXs@vgrl4uwzxVQwl;j7ksLo)6>7#o6oH$;KWoP03Pfb1JGUPd zF4LcdE0PC%w|NzYUDEC$=7^3(@VsK@K6wl$LQ8mS-Zg%s950<0O3@ht((rH!){9+6 zXz2{JI7r6@8K{2D!#)=j>6=y4$ezS6ls8)KtsTq*3YglZRIqcrHE^hAqOB6QLl+ia z8Pod@+6^XJVaFcN#m88b5--(pRwzKerv%_?`lsIERB02Ox`6pUTL#kMN)MzM54!_l9d=J7ScUR1MGWCcmIDPSnANo9AX9U27KN;8dOG9i-JF(9wbqnf*mf+AQOSigNgo--elJZbyfSw$|iBSUn<-)gP zL4c0@hj&-LLo-faimPT6l&~A~vQ=;e%%%+HK7AZhs#Ir()BrC3z16H*JmhwV z{WjAtN7lZ_;%LLCN5zm6uYqe`nZDe1yM(=OszyIZj`55ZnSNTNHbfb{U}?`Hd{4z* zQ^P>VW*fV>@ioT}hFy;BGt4HjIl!|}^x0>wfZ?Ml25j&{#{#ppRt)l2Kh#>Au?Xf= zDS?0{yC}z?B^1l$Zrx@$F$!g)LagkoYEFp?dcd#Nn+KqSU0bl5;mJqO4M79k#e#x9U{ML`b}{7fm$xQ zwQ6(}R|)fPYj7~K=_ol1$`}_61UTB-y1i=-ote@@b8z+8@81=4@adf=XE&ysnxO#~ z6o6*^3MnSPzZ7_(f=!7!@NOQp=J*cIh%me$Ia4uSZ8Wvbp8Dc&P5SauW>(%?F7Kt! z%9jaZkhw^{ofR5hL4wQ^iD-wD zrUTbNXQRT&HxLG`KXQnFUZxiA;fFtbfZG7f!{&({s*8mF>_geS!_k-I=Ty#^*$k0< zhhw;=dQjCqi?c%+24$Cwrc2?m_nmik? zD?KR99w*`43wcv^!&BFmV{z2HX0ZK+b3{A6U5>3x)d)^y^_pX6-TFfJcKNm{>oL?OmD87N*Ol!((Ha#B z=Y8Fxl%v-qugDSZXVEaW5Fd+m6$xmp!;!ED(ZFPe=vl5j4O%5^$*}h;$XyvaA4K#t zibIZl0L=$M$-qYAcKCdf;>1cCM?9I6j+`yu$~*+ZZ|R1-^z7!x-RQRiVm4zRgHPgX zw#i_#zn&Dfhs}H7`O(ytf91g({@_YQNhhMJy>tX=!$fBS_OLMc^ zpjDq0-xrkaN|d5K zCM#5(9$&F&9&3$}mze3DHa3#6S)RTJ@=z*K5*_xj(jv>nU%3m!=^&?Zd~JHFS}&H~ z4P`jIUW_N@Xj@6$7AkD5l5U1~V!-)=8gb;dk0%Jy;(!arEtaOzLUO6(=iH`M$2{8NR`#q~!u4DGQ8M+MXz{NP zf*dTHexgjIq54GNiQghK)nNA?|)&m1S}d`sdu0q zH@)xK6Q5BIpYI{@rz46X zwo2gFkJlTlKKhIT)st|pNI-ykaw{}EJPk0GdhWdm&4jQ3fMF00%S^w~(KQ%Xb^wG} z;Xp79b8|;g_xMu?;Q+*|Zr?vJVx}@{0n>_gR!-^q{@rZ&*fJB6s?n2)#g@JEnAQ(eSD>$cg5F#}eS-QIOmG12fMtw~P{$S}1K^zG4 zg}@Rp?>RSV@dMUFGfHxYTdo@zxXr$Q@&lg(KRZ4MFb6<&G&=BTh`rmZO?78o;o&6Z zhJY(-I(09ge}8qstXaqd`XbFBsOisfAD=yeEmRNtS_0E92%rf@&YBG`aQOZmz${`m zXGAa;|BX}p8>jd;PVsM?;@>z0Fqk_y&A)Mqf8!MY#wq@dQ~aHP=O6g`yEw(x`o{IO zxzo8qOQ#?=FaL(&yN~aCvvT6R9J~aUPQBM;1F7F#Ou};CK?e9N>Tc<Cf{yBCdMfn_cS;jg)et^l9oHS0lpJp!~|N^Y}gB z2#8ok6gKYVO0BfsTbr!QOPSqf+t~bmsC*h8DLs{B#Y!_Gr}|@JWpc~2dn}-r_OSqA zN6PbL?%{IS5P-;X=WGZ(?YJs+;e}ij z*IJz&Z|X*>90cHFxm<#fLVd9-v?_VGTNABuk${#ujv+`!%4oJ`MRR3s=6A)^?g~;9 zLJrqX#8ila6xw+qBV2o6^?GVw0dH#dhwX`8Wkl}qBP)+wnYrvL`Gx0D{+GRS>27VX zS_jV%X6_*Q5lGs*cX;YqgY>0xZ7`-nn+^efwOd=8uQ|m8z{QPWl{2BCL%CraF6&z8 zhr_p_gd9xNVZE5}@Pmk$^{Q5+{Msl4gsf-@jQ)_9PgsiI%0FkSbL>4KL%+TuPxyS`h+NG#l$vN4*;T6 zA;Vs|;iJ7meh>g4X@=y6YCL{m0Nk+uLN!X0<`U~#&nQ&hvqjBRu%ru>Hs#WpTR`Td z3Cfp3=&;|@t^6Lc{nru36z?nwz-`{NMqQSJrW4xJV|&xOu^LMLpNt;yL(Vnc20(Up z?f4Z?7{V*0Qwxy%W+`9JkQV__mZLLXN?@#w*ZjM?Dv0&~SD{K#vT0qiP07r|EmMp$ zZ>NHJt)PT|vA2KK_tm@hd@p-q-Yo3q9?QspG%90jOQ%GY@w+t1N5&Kih2>J zg*(eLK_a&=4rwir$u`l4BLo8rp^9CUUiOtjljGKg0x-rG5BdB(q~zj&i&oJ?^IUoI zA}CaAy}cqEqca3w?JJj&R~SS#NTZyuV(vrbKby0Gj77^YqPbw}+mnki?vLh-6>RT& zP2wd%@~T9e48gniVdXYpe$^;)#{18r)7zV?p;9aZ|FB3o_dD1z{E?3F0KGdlK{tak znO3ouC4nY|ASq@~60su|o>D+q)(3G!f;?9$R8zC?s(ISuel4^-MC9iiH92v1{$Zdd zVXm)6#No@?>L3MffKHY)7-8Kd(5>lJ^Uafs1gj1R3W17O8sfzGOeFiuR7t)rwcIDS zSIZm=i7RdsI+zTmz2T)KXF78pRGZW0Sz!}hy_|kKP z45FA}f-L=gwK8f4$PWt%eL~us1)qxg2RO)J{>)#hays%r(aT_;vLWu`aQxMCu!Yz- zBcakfly3SIIf-pe>`SZP6K1u>V|H2?K*5xPQ-ZdR#@qfs3Yf9KNKjzYoO@rzBtSS~@F(@LnE z4S-K(mZdJqEK{P%99!{Fhk7oHz-lAASz>7&9DiDvV-eO@-$2y&G{LaVe^VlqCzKBX z0;{sR`@{N+n}Qd}G{#h`rRH)Hz3+0SKYcBbSSu{i0P ziNQpzSGxxs8%(Ivr#C(p!LX$c9>$c>8j!F5T#CBlzR~U=G+(}cG1=!|gT8!Ku(7)D zhx%)%CExC^p_V^a4$;>39h~ShXe&+jY)NzvTG9fgIv|r<(O=WqaYz{$F~{PWK!414 z!752}pW$fzm4$D$%+*p&i7j#Ao5oHtnRfzMWEfp)x9o+bmR4VHHE5L#u}M0_fkYAn zh+k+w6=4_5J0A7KhRa}qWZhQM2MG5+KOm2SNi^}9mop1R%C$%<&UTJH5~L8bk4DC4 zup>Typ2A&EtGcrAoLgECnn&Beo8R{C zn)dAPgi4ljuTp!e)&>Qk!XSdrd;(MlFo3QdHpm(6l7n4^0WF6&v_Ls7HHC*R!w4tG z7siO|5MBdS(@Yf%uqgZFf^r8XsD=^(55l*nLtZ28ZgJGm!;HVzC3PoMov-Bd`9xkO z0niQl0aUcLK)Apx>{=T-RN6AExV2LqRjaVHD!V;Na_?BuvjY1OUJ4CX-~O0o|JFFE z3n$~M(zoc6ykh_nCHE?ew7xKMiT9Ko*mSpna;B@&A^4u2o{-G7>x@DNlmR{OdT0qm zP&!jcEnJG*5SlBfaaG%z$u(XGJ|=3joXrKJg&K~h7qu%V)vM4pw1Qfy)W{QouAJ7x z1U2`i0fjaO96^0!fT?s)7=TzJC+U+LteaoOrt*6Sm5p)9*7rayXb>e@Pf<2}3E><| z+d{yhnhFK_L=1tJCP?j>1(jP;6Vs%N%Vz`g+`iX4q`0pvfs#D@LL>W4O|TPScj`G7 zTk`U%IZ4>Ktmg+=z{cOv0t%-Lk)#^1{F@4Fn45Kx(;&rx4i_rar0Td2x54r4^UKt{ z316gRv~E^*)CXqy*pnPcTfcQ7{Ywfsllct-iWnLRN|K^*neEFP93jAym#@&hAA0rRKdDgy&dsOS{J~-)o$t9sLaf0km$&GXGV4= zD7upC0$9*?X94p+1+eVe@c_FP1I2RUcD=G$o|LMdbK_mh$}Do~V@0%wcrb27%nEND z;vW^Lw(%BOTYrqu07Z$}9@D*YIYT98lyY6JkJq3HqkD?-%X!k#HX|Jfyxt(pY_IT_ zK$j$bpKPH(i-J4K!I!=P61Ll#)%}Q$_y=E{6x1TR5yXS(M zJ58Bw3?*y5Emn+Ifm*B9J78OM2G#3ba{VjR;BWK3-Q^nye^jY!xAoLKQ7GI_7_{PX z3iqLl>3;#7a%Sh9)Ir4n7zTlrtu;z(+-14wnKESnmWS1nytL%E$VjRAO~5q{ojAC5 z)cn3V%8OLxq8vog8Qg)bN5xzE{s}ylb>cDj;d|T*kmi4`%=xIW=fXu$;AS)isJUQR z=y3tjSCkDHn5ht?T4VC;$#;p3>5LEHAA3kw!gd%3|DVsi{Db&rHxzSTqbCqn!W}@Z zKlD&g9u{%wJEH90vrM;Y7wrq%qi&ww7EioOJ>x%=s8JW^dfQ8LL$5-__L>!y#$c z_3Zra(fP0YzloZMRDyZ|MIalacV-mm27rS!c|Jerko14GI@@dlPPPE;wJbrFKTZOU z(#E$JS}sGScwoWc6X5Q zjtn#!;^dG5;^IBeZtZ-zN9VNF*?aCQM7Y@kl0ii0=qO@PlA2op=TL@MCm-x!GvjEzy1Wk zwH9uWbpVq-(_i#YsX!Om)tCinJLbsDmeq~@Y)g3C^d4?V*`w$}!fS$S13M&RCjssb zgvJHL#JZ>N@-YBOS0KAAF0-=|_k`)%HNOWc4bTsERomL9Pw`zSC@27OESLZ4qH8e9 z_C1UQrMD4$&v3e?KOyjfOfx<}urDid>Sk&*`)+5YgUX_JDmU@zGFJC>j-!>!2(pCM zz}@fEQwo?4p4fZ)?9Pi^8%X*|V|o!N`|bSSp8VgMukC(%e;zb-_oIKp|G9~ndz7At{;AJ$ExM)T5#>RVMa1ma`lp6l^=?+FR#891c)w;kS~Ff7?-_l< zAsvq#o<5Z-^C!$TWmA*HM)ccE7R|Ql9Yk@@9=%(vFCE-k(v5PbJny_acTz9>bj)st z@8Q1As(*Vdrm6E=_(_NDb$YXWWX^Zh5sGYSQPhgl>?yI8`SL>V!JZYr8pG9(H`oae z)pM`tfsVfmul5QDBo^Y^k!wl=Zxc7<%hk5tZK=9{QhlXU|MrmF)s^c)F@@^o2IXvW zsA4X=4`ch9TiUxH+b~Z0;xOtTala$1<^-|HJ#rcA%WE#4Tq7?RiWW}v%HBiqENM_b z(UECrMzrE0=`*g^3mHJKIedp2lCdqew3HKGSzY)U4&wcs!0B=#rC`m>;3C&b;dRWz0lC$@P$Q3@aV_-ezis^@u z4~Dc(HYc-U6C&s2Y{ckm`fGF~&3}+EaL1X$S-Na0q1`IR{ft5>EuuX&4)1$GzNa9z z!9$A*!fr;iR6-=YBg8B}zGApfhr^UZX;WS!p0MG!UsmQCeXA?)(|~ZGOt9M>r0(2X z_+uQ~ify(K$+5GhWTSYE+NNX}n>>Mv7EE`3<0VYWSn;jS<2OpX>UCkbvE+c=@x;+u zc0Di%Pw zOt;Q@edA{A?3BmA8Q$es^`hZ6kGX~wfnwM6Lv>&NusPW{Vb?uMQpg2#1u)Gyg{9us z$w%Tptt3%oUf`3uUuD~6*(E680!WOxxbn`{*F*oE{&Ql-u#2^;T|9D zd3Nh@VOQaqyrIJT{(J5BrQ3C+>b1Hzn9Zr%j;g~n7fLO;U6gf4IRe;kCvcUQS5Ad# z;L^|5UV~*}m11Vj>q+V@p|f$Lt}`qp+^_%_r)dw^!zAYbP&i6l7Bvvu8JNe-FJhmB z68uoSKx7VZ4lE9=cEkjZTTduqyO>jD;<8vYRc7O6RY$8BfUJ}EfH(G)^Wj3;M(p6$ z+zbU_rUwnzNSiO7+=0q{L}RYKuK>O9{MDIjQOVmOZnb(iPh#TchFXX~G}oH%T1#79 z+x+S`2ejE$C9J=ZM(;tg$niFv{XfZlBJ;)au`TV5VO#^UP7pJ}`aEEr*_d2D;J`9* zYHoHMU9JVY)46?Xn1de>4mg>V>0m>?tNg*0(LLE>cK#yl3NzVRYQs{aGy!q_bG>P! zpunWD$u^BwAM1!6n;P|QI;U5u>}v3n{>kcN`+KF_J5SlL$!}i%DIiv&cS_&<`n!d~ zg|U$tjZF7JStp2y>zhK5p4Xky(u@6x-;F-{6GK-r2Ph}4AglM5em*vkebA$kALGsE z|C9gd9^gM}dj8C}QKVlDtZ>~AYkSiy6sqV?6K?pG{$Gci!|d`3`6lhNuq#)b!ZHjPWx3@lL@WH zZQizlSRA(?{aS@?n|{i;i?CjIoXUw|{;D3b)0cNofNio}+Q=FonR$2R=1|D`8NgIV znZL%&?V_ej&~laD8gh;8hu=_66T+9yl>!6jt2uY2io>cv7f;m<=gQf3tPbQl!|&g} zyOJdjh5xSi3os|TTjf4N2-mkqYlPO(3VP956|ZL!pL8&L5SVbq(t(rd+5GepZCFSRNAd>AFfSxHw`*nt*e5(q65y;7tfA*DiMP- z{YWYo|NO*f{^qAri3iR}OB-T=b|4&I034)>l~E+#%XX+RP{(fEYMkF2SqSE@cBQw| zzTV%(Ww1AO4a2a#_0U1hocufDLq^{_fyxs7{hXqc7PF2(Sk z1kFrstNYW*`e4hF__^_`^Xp`a=99YVJ!MFXB8xqIDSzY8n74iGNk?7)U+w-0Ju3N= z{^)4w_r-vGVYn^#-3yZx0nad=~!M)R5EA=KEfRak0&v#7Da| z!Nj^kI_xZ)KJIc+&)RUxRIgJA<(Z+YV)nC(9=qLSeVHZw!>uV=F0$S^cnso8&xAvt zn9Pr}Su2)Zth;1+>VzccPD8GF7eun}xxJi2Ov^cl6R`8dwv%J@ZjpbJnC?|KsrP)F zHH48Ky70T=;;4**fVWYhy?FY&FnDo@zrS(;E!2a6ha8x{@gK^#_&wD!YfxK8?69ke zo7p~qR7EQuhU!5)q-u81fVI}HyzL}G8NRE~v1x$poz^J%YEQXZn1oR1H4@IYb<1frZ)O& z^cI#a@qyVnfE=c9r&?Onz~X{d`7x$oj)i zzJ=ZcJ@)E?gsHGQ&E$h%o8EX+9T>%ZzEc5of^q-J50{nd!f4q@Z^Wet}~V zuSE0r_KRZtjh%84<}C*UKhBoIa8u(~L>$N~GVIU0B|A_TqSI8=au7<_m;T^_Rr4%& zQE@f1yZ!QicpgCZ((wEAUa(^#Ha8kh{E0aQP#Hp?X5^pHe4(qUe5GNMeFvG&jS+o1vZdXW4puhaOtK)h`{seSP`Gpu0v?+UtphIp1l?-|G&;^btpkdj%F8Njy9p|T zn@iRuwUyPQk6;rmQn43cfv2BsngX|LbY=Q=qGu5}F-@7Rl9#_OZ_i2GdU2w3r?q>+ zZ?M`HW-ne8imJGame^d}oFI=+k*BVfe#eWwkeZKE`b4zouFRlR}22~A%>i4Mm*e5(Po{J%1aRD+hH-Tu?v>4syz~c?02q$V& z8&L&`9v)p>2fiTamM0A^1UBxacEA&kVDn8~lFEa#BOkJ_`M!i|lM0n1<3IW1p4K{@4v zaVBRo#^zLVq8*y>6NZj*svOc8atd@Z>bKvS--3|PE02%^0$)`N%%crPxzk(Ohh3zM z3qS8>Ys&VAJmJY2C`*+^$%O(7>4mPm9w+yC0A2d$upVfk7^$O3uJl z?eo;COM@r4iBN2*_E61G>fy`mN+@1)nCO8+bcfd3V${6X0DP!qddX_tySKw$ra0$( z#jTJ>IhoJ3wR)cMu@8CQqs2v}CH<~g0xf3r{Ze3NDP9M0a--p`qYVl?`vf+8Uh&bq?p?@K)jX0dABc< zHLG~DZQOpaCjrc20sLdcequgUw;#`_h+_ z$7;S5=XNjl`r*WZFFbBT7DRhhUGdQ2qs%{5hLyHZxFDh+lBR53)KwS~6LB%xSEb~F zLksU0a#TCL?+!+%x^!t#op2K}Re{cOnP5;4I2dleU`sK`+H_x&2EJicrZ+#=`Q(1$ zAIgd1G}vuP?((w3wD`!p*cLDQ=3ahP3lQ=_6A*;BnZ&Bnz0_aBr1h*s`E9WeEG$kO zJLdk`6)+5SzdUlm@>|0$Ihc;Y85st_-+*5u;b-sGMBd&IE_vd)S6Q3G#Lk$Yr6+Qq zJ7e@^p#7JECtUnzvsa1U^c$N!*?lRsocZGJn{j|}I%?SwF)*KEAjr(gthnXiS+dqn zj_Z4UPE5Pj0>y;1ib(pcRUtf&CAkw!qy$<*c9RmqL&- zO1;rG!buS!vDN5t>R%7ou#M5JD!Bn7P-tB*e4qcKlH=0FS-XBgX3tI{>)P|{tS=9Y zM0$2au#Z$cT0`1>SAHs?|1=bNMK2B==jJ+YeRijD2@l>K`qlZ82u7p`Du4lhjImh* zKXRFSLYV(G8P5Dj_5|?-QCi5~Mn7K(4dkdF+1fqn#R4jE#$o3^;D?aGLGeL--A->6QEY`Dp-=iS8h-08WMBsdAvS zN&zSFc+z%_QMxpQxBz`sPyA(|;tFYWA3t-(_^{vZjLZhP?(x~&%2}+hZ-Y)l0J}`- znXcWh^cex%5?fnayZ_Lsho9&6dcfg;UtriEFk|=oO!?+(5f54;E(`GSm5p5n!pY$D zQ)5R*-hYH!60ScT8y$6Tve>t8AEaX2EcHWRVh65y07bk_OG|_Oyg(RSkg66hf8f9YH^eMZ=w$<{?64oNmI?j5 z9*9OKDF#8}>Z9dAp8|*hM@IZWS11s=0(vH8bJGWQJN&mGb{SoO3GwV4h5~p{6okzH zGO~0amW+KRe1t>FExGCE&7ns(gY@iZeLyf5KF3IpUB*lN&#z`BCUXte7IC?5`Zqp{ zw{PDfw&^QvfloxcYB3lI$Nn$VJC&$=^bJ$^8F*D=_!L|?y%A;6?E9?Vg9g`Tn`EiX z&s!^AXT9n@@fz4){#^*ZPuQO)4P`!Sp z7V-~y{Z*RiB_x6cHqxKznNL<9--x5@qwO_tPbx@+8)D%iy-_B92bNd*8lB~q`rp5E z^K^B6ZL%?>DnuwV&4N2?siJG7CCd$nQ4)P+aj5Z$^>}Bg`NnhA-|=aKZujFCfZCfS zaetJo>>ZS$$yVXlxhtO?CJHfv0);7YtXC4XX*042{XC(U_~kmr69#!T`#mSK2k}oA zo0`WF)GgoRV|prBj|t;a)4+`zRQOz1Q}#Vi`yOoz5|GHHP#_bT+8&(*jMBQCMiMXkHO8W!IAz3qi3;wv2o$z3)u9u~y= zP=mdumNPm8&>b!mLUQC@l+V)QbkO4BG=@^CGT+?P5zNKjay28J9N$GK$-=5tXE4i`eqTHSv3 z0*2iZf0!<xT;7Zp0FB(ZQ*Pk^8_8rJ;(`SoK9$XxRx`mj!NtG*ao@hx zRRJ+cMQkzs`!C!aQprzdoL!GU!S!(hSrw1y z53bEp@qHSZ#NEIR#>|WCj639;Yad?smjwC&{){_~ zQ-=XAUTpHqd_%W2BAh`95CLnN{xUEEb>E7a8!(phrEM?W<5WVnhoICOQyg>vE7w+m;LRBddgz?>8>M5+k^gJjl_=PGTA zT~m#3!E!Y)-9YChlmyGUKdt)YX8(yu`=9~$~x9*NmD60$u2)ExL zs`}j(yVQUmkI&3mY%Pjn+bFkRWwfPHyMgxV%&C>$k86P>>ht3!nU&t#rc84L;*^hj zU#{*ub6bxfQ*81TCN|1b->qcO_l7l|MRC!z0z`>z4jX<`S>EiS3aslzbtS&mi7{=h zlv93kwfg$qPVSZ?<|X3Pk%rj%6USKu1SJ)RIpXw;cy?O$SScX($s4X29A_W8(*4}( zc!wZ2Oo;lNYKw?Ez;yGejE9@Rgr;a$$!H@+3FkEuFg8B&E*?JfP+Ct@sgQ$iThTi= zJoP9y3Wo#XC}cFW6a%#o69#fJWDA(UypJSFTkVRZWR5CfLY(yOSjM^?*{fmn`Mi;W?O0s4-lYsa=RG`Oc`V-{t0;E`Ce=Pd@6qO?8QFjV`ZqcesOYQ$x2_4pnh7i#N`S@$Al0d_SkY z{|40*H8lNO-)B;TqDa6g!GKd)MdbFTjv=F5!jA%xgj3DewE4H><*gNWZA*H-aogeD zk3e_nzspPiFT-2F;R&g4W;Y6r^REBl`|Y`wqMgzF-bFIEuj`qLb;FOw38X0L?62AI zRowOP00#BfZrw6YD(z=2z@No`FFyU(SIzugpHEx*^a>Bq9W@i2X|*d#+I=H!0Wc#x zz5#y(zh>5io(LJx*!@D_MDw+fDDzXIhw6TI2_JX#z%T0FTwupNS+lNfba+~8>-oV~ z*+bWYeH+r%vb?a#P4U~x1K#BbuhqNN)wWM+h-Ico8=vzqX?I~Gkp|SerAqjerEaRU zgned#+(tTZE>+#L=}u`Z&c=Co85}>af>~@dA=|kv%E7p!#?9RmpB!Xk@A>TttjmJ4 zO`Xgm=`h%@5MuvhvSdB5Ijdwj<06BHQ!Hns!V)iQpgiE=9D_D$os+mUyU}4#?EerQ>^anbX4~$@D~-;8nhfZ%-SwN-XU?-p1#DAIlFr=znlXJ5 z#a?>{m_#WhY1l&kfOxJA=QwbRKS!hAa>+^G&qv3sM?q-A=E0N4zm8_L5@K2=1lx9K z(e~`o1-kFX_z&8ZTp5t>6G$(Z+mqJt0l$e~hL3-(8RZu}baSA*oK0a=C9NP*>h?a; zhhw+NF2-4bqJ^SBbt2 z;Wqt+*u!b@Wz#V!kx=9gvmk)(~Vcc zf`Pwfr5G?P%z9)eNh_;BfLF)Yl02kv_1!lt@cBr`DI)xAe6r zlQs)gNxYkx^ySfYw>E*LjUo5UiJ7En!kHNv^gXP$WSMQ+H=8V7zLEDeM>~GZEl*>f z*GIiU)By%}PRJl#O;0GC>-m1&KHp7xDd z;@mE~hYaMVR@!XqChl-Efl}r6&zBC>Wj7@yC2_-m5|ti16%?2TaiTPD%6JM<^~?VF z5BqhPtRweaaFLrkh#;<8ktZ^}iHOW&$BiBD0Wa9=jC}f+_Y`}>4IV|m@VB8E-2(%! zeq(ZfT@v5?CLV|Dw))H+$uKTrOE7sA5TJAKUh9d5wrEMU3wK_>@#9q9nNr$Tt|0r9 zvHA#+X_1uLxA=1-er55;W+uNtA)Y&)c_C|J(YT*`*4cQMVG6v(tH10T{|mL2e|Oi| zd7V9fUFNxyvP6RkyVxoQHpXTZun8>W>tte5cdkv?n6H zHpDr{8W!8WyY0^s$$oI>O6?mVPx#;=Px`FCzJCJgk;TiS!~v&$0H|e-jEuBy*p0+P ze}fWoL}xu)+w_6U0x1-Hmc#P&07%8g)ogDeOB@F@6}DEB1`2j=dJWW=K$kl?K>2^(r#m1hs%%M-%%h#Y42)>p)_(xVL*wIJyu^3M^f2^;X)jQHz3)rHbq4zPI_9X$7B8UtFV74) zw1f%$90IG?AO*}LBVz=Jl#O|euWzEm?>~H)FYQVBBz%46W|>!krxrch5Ttw*-0m0$ zM{cN~o;|2VkCI1K0D0i_XU~2u+Ev1PEP4&hIt$zox5j?mTeT$aQ>RW%_H#el+Vn0; z+Og)3PrrnoCn2E&YS=Ojic$wO9LUb{{)_Y zx{?S^;@CG5iNtBXvb}nGdU8M|1CVV6%gV{&%iV3Nmj@j{D*BQPM7g$SPnjxMRFZ%X z)A+oxpSa7gF|{|RTUn$!`J|bT1|Tu(13D^gKOT_F048hz(K+ozi(}w3d~Xt zeArtNtRwc_8m7&Y;6neN9&PAC1JHX*Eh4p(1q~eC!D{jufK`k-*_rfn64ioYF*kGE zkX6xucG4ZDqNIdF;HB-r@EBHl8^J6VPFf=fIc&@#=9(b9~GR zk6D5N5TK5_Wc~+-_t+7pQflih#u~9ZX(QCT@ zoSK@Ngk9ICX+Azi1^Qw)cRXMfs|FG-@+m1P8OW7cpb_mT#mk^RU@}XCrsOrhPXoDR zybOF+TS@5^U}J+P?S1!Zu*4zUzArZiylDi;HEuwZH#~88aPxmYD&JWt6OUAqQsD73$F+J^)`Q z@hNhNk^)S$4gf_E&r7) z1r`mIVdNZeDqe&0H$PU`wNNm%MZ?3wQvdj4Z}YX|qriQFCIhe%-7lC<_axpv*cPuq z`Z2?`4ZzBB2P+E;W^oH;2$u(*{tl3_3xFiZ<;$0q)YK$@Og5dvV3cE^o#6S47f~_4 zo`?i^5~P8iq8tt_AtA%!FOno5iyvk#GlW_Z{k`O6P*7j4Nu{{$mxKd)hlLCZZ`pLj z;lWWK5gBW!Q>zh6+(F<>jhc{AsU;%;Q!q5Mu&|(c|9--NhWhV|)IK#oKR?9!RGNg( z>ciWLid0}K)dSchposx~avZ4wtTw&|g#?H3l3Hn@WA2HCt6*S+fQf03l@SybE$p8! zap)7^=ZDE5m$ku>qu9V$S#M1edLw6Zb5s1mrF-4s*z+%cIuQf%V2F*ZDF- z`|)5UJg=~*=map6LoT%@I1(D^Cq{bd9+CpAjstiyIN3y{8ok>>gTuG!RaLUUZ27LQFxleeyfT1gi&tUak#A^vH=5Rs@f36}>ZJHtlC_cAdaT!)G1) zXO`WROy+^IdmHGjCtwRb40i1REIK=6jAqe<<-QXu?fE)C|I#REiv&=jM|{R0u_Nj-jDp!6-nsz*p~|FuX87Rj;tUrdHR|((=X<_z&H`F)?{oxqyx7ay!rz z!?&hLDH`Mq;ag**Xqnrb8aQX7TiYTs+%O#YB&SA3MpCT=GP-w0%aV~ku$y~v~q^l0S*_{VPsuIA4M*2dludf&2C+Zj<=Zy6AwdH7NV@D^hV7tgnV_u1;q9wi^6!5+p{{+60Kcu} zdQ+b$yD;mB7u*8Q<Gwy$Cj`iF||L*=RP~I4#D6K*HS9{KKCJub$ft4=mcbetWmfX0SmHpRxkj^hQup zj!}IP*9uHfn`W+IfAePK`4@Xkno^<&vZe{>>gtksqJ-4u&8wr0&t2652S$5&d1=OH z<3y-SvCh)jjRD}-Y}g=EZ-K8qtoi%zOIB7%?8d);es}2hGuq0`9E(5^b>-9JX!X=+ z$k?NQ|NgBRe>DG^5t~&(hlh9^4ys;NR#A>EWi!#QL=)I6`AjiZJryaPgIVO}SHAhG zFew6r`)i{1va+(m&Q;p9J}y-~4FK6;AU1s~q7J|VwxQHU(edX5?m;;(TxXk}*I%om zqa-2WwlFtVVO_d`@!0v3Cr|pgd{ahQ`4uPTpfCzqN4~QyFF9*USrLpoJQ(%k1nU9dZ#tG@H*XeWh2 zMT*Os2)MXE7pTr*veC$H42ZfEGnCoEU&Lqg!+Oq@iP+}p&vfv#ec~N`24*douq5_?Oz_GNl zQVy~0Yk*~*vGI=W+rtS8Kv`F#AS)V%C*Ma*L9uudnt3QQh-l4*M&?J5-({s-4QQe?))Ip`S`=VwNlhqj>Cd@nlgA_wOFcLW77LQ9VYrQuHahX z9*DTIp7Ge%AD^DoBGZy!hSJC3$D;$sZggKPO#&qG+|H%?8kfLv4lPqlwimCE$Yj0uu8;D%pqlJOngt-RVI1=)>c8(_g`EZ8+MUsoGqJ(`Tq)KQKX z9K_N=FuDMtYc2L{b*l8*aSLa~6GNN7zkbb@YT2%BR32Cffh*S-d}&quF77YY`@DXa zfEPQtIBbT#+Z8~)S5X3R$h|!i&TnMI?8ls*leayIGd+LT@TBw0#_Nq)HZ%eACVe$o z<@orRI)1}1JEen!lt{`)O&x-}F+qO9qc2I-TaCBMwrjY{{f<4q+j$d{=*k4$(;;Ub z{qwo6I)b};%a$$f@7_riG&VM#u+hIEw=<_kaX7)G; z1NS$wT4ZEo?A*btosyF2GM^=vWo=wgUtiD1Exh94QzpTxhK%bWT_c68uE)Zq0}PLu z)_u8VZGEh}Jjhfn+;*r*xtVel_55ininsLzhb{6xYEgdyO78^dL8?0k(6@Afs)<+Z zVg_84-=10B0Z^Q1!%#$@De6`F>u>+7eb`S}3>hnO`c zof~RO<*p`od7f{-?5n^yXGKM?-NpD|@7&Jm8j+WF7gSVKM5iGT zsIK~YP(3BG5@A^nf#)+`9bfE(a-&w^P+{3&t3~yq^G$SnkO$5IeOJx5IxyWB%ic|E zf55=(t3ki1xO~FfJL{i+{#lpYXeKh<>8U)69L#mPiaEZv`G^}SuAXxaS1dbY3&|jA z8;-+=FX0=Gi5HItlrf6jO!#NZmQ$KQi#8Rxrtg`W1|{?pbEaITARI6R7B*MMazlOd}b!5@hT}^d{=n> zcZWxxNs)<8SATyrP_7ATGl=q&QjpmFP|7CTJWy0l_w$<=s2{*fWITM?%>cz&tG+d&k z)vH&e5QF=BC>T%0h@05Ob}b{0T9Vp2Ytv`PD#Q^LUkzI#=h|rr?6sSgdC_zC7&@>qK^>cGN@mO0A8DIXIYx> z>FK#`+qSTMQa-%^toT5{yd?@m&gOxM$w}omr|!jsZ{yY0BqBK95OwoDv~&2@-8sC; zJ{%Q-N#)yn=R8vF7JLqBCy5Y{VrLf0!= znPwkT8=|ovI&nf(%7?3Rw5ya~_rqTV?B_2oWMePvO>PYAnlCFCIGS|r`zcBldPK(z zO173-V^sh{Uj%k_fJb_)k1+rW_7+WiaZUjl;}U{|k{iR;t)VN|Za*j{Ce}K;y!z$E z{RDR4_q?o=EmhIo+Rm#TgG?5AqoNTa{{82uJEeEpN8@y4 zyPP_8Y8>fmc36&&$$^-(RiQ=4s!xP;d z9U)z%zEV^lULBr-!!`M?i)KTnwrTd+QocL{1M>M@^gn9!3_;9nTKGoyQ+9(B&x>M5 z*3W%^z^aJQMN5bTMkB3MRn?cs*f?hl%JTELIckbk*9Ali)!1ZQA)|oVOHx-%^i@l$ z#A#My%kaer147oW@=~CXf)DC0PPlkujKvvllwgn6>L7Zky*%VpgF&N)2cnWtZ&W+x4_OWWX!7;aUO?tWfrgayO^U|EXt&d6| zoqCY^8!N}8G0A|yD>PePm<{QXr;CoGI%&ZO+r^O_%Fl}e7%P`x_87I~&cQ)5zxpg2 zQ{Vyf#-y|2b3g9@aTsq~POipGfYi3Y>MGUY(p}x%`zcmdR#jksDyEsK(Kf%b+{VNd zgR)=!+~s^Ni#f zLY7ww04Okpbqwhg!eE~lk2D&%NBQJ8;5HQbaUxOq$m7(|f%M`@CVP3%Q<}8XzrgW8CN+-Nu z;J2c0{`x^`VFP~;506)_3-*-b?I)KInrkEM4`Fe_4`8`&(FtUkFGnj5^6|+7G*sp} z&sz6>-7Sp@y406f1@{=O5+5fYc7c3gqAA(PgHUbJur|dIgfxu;0|SrYn;lqL3Six? zS-rXvnboGVXfp{tc-4`Qw{rvFWN7A}=H<4&n9Ui72uq5;Z}0D5{t`Yt)PeY_ozH_LRV@S)vPn5%{ zuWBD9X&s9d4a>E3uDN~337{G5XL5drq15^me296dEmwn6B~B5Kvdv}IoUCf|<}^Q+ z2ZM1PlO5fIgE4p_eJHk3a@L^;yD{;<-sZW&E?dFeJicnc8hb~_0kVw{3wd<%Vvv1Q zR~gyQ{VGh(jVCE$xH9+Y*_IH=jfie6jp|6)ua2;{{=)7r&bZ!j{|MT9)?;{k1_Yia z<)K>F`|!HYYoPmwnFSDWwK zw~qlZ>wa=_@^yTMiDuvKJQcLNIP%h)Q};`U8xjki>t|$TF`_lu*KOID^yw-xdzL@& zquDwZk%oRv{r$oNMZFme_jNH_YCecG7N@zEVH)8D56BfC6v-ZzwQxPI-!*$Uq@yb1 zx?TwG%SS*hH!n%O9zAGxK7$g~SQ<|E8I9%D zQMs-{!UPJPeLHsSKqIw+@51+?9~~X3MingExYZT$uhJdby{*T(exVNL(8@|dJ~C(6 zHwub<*3DmU;lJc#H2b_8pZXATo=HJVw8~w6`ZXd~#Krxdh*9QRYer9EOo^{x28F%1 z7!6s>sKvy9I1m9vNRf^MJ*yKN&R*(-<~vsQ0QN}zrzbm7oo8*mxiyp#naitYlABbV z$I6cP*Tn{)LG#9(>}-wPVK!QGj?T{eZ%9%Gg3NN7PAc)`jR(TlcZj%x z923%Ei!KTo*YNO=+FNOQv@6ypv)*)fw$*I*cYrDR&)6rK_RiRK(vFjZl|Xl$eSJ|l zL{W3Iu*N`<%p&Q}u_=@e&mP{iQUm=H#$#7sBNQ}eZSA}>E4a?fViko5qwcQWqWI?2 zN)o&fLw)p_>AJ!o956;M1H7ktRyzyc_iQbVH(l0vdg;@UosSKsy6E=Yy!P?q$HUaa z!>H{jK!(&cE80WA8AW*+XDInpj2E)kiDR1knbCrIRsyrx=f1IrGY#ek#tougczmn{0kJ z?#-+ARD3KfDlzR7$dvLIW!(jb~}l?Iv<^=R*Q_bseuUbh@Pea{`oAbKh%@I}|AzP}?cL-C4 z4I4I$PEHzr{i~%OIP)#?s~YJ7l5&BxS)Od?rUnNG$4V_t+t^f|e_>PcLVJe1Vf1uS zN*VC^c?AY?0b8aRD^QM)b zXx)@0LT;XJc(upsy($3p1ONf2VQ*(4V4_%$X~Me1-j2jvSRneJqAWCj`|5REYfa;G z!vQ>K$jU-De|}^31uZ~YOlgUKBvB4%S5L-w0DRTcSP$MDO7ciUqlp}BJ;;}2oOcz& z z1k!nJLUX2z=RHCv8Km)v@7{e9I7tPam(L!o@yOE63FJ#&baZ@@9CtJo(k-;MaVm3F zPlL5ki{|85;vTlC(AIq!*sbW%s}KKkQChl|bY&wW4>H`}jpv%{XoqsTv2NeeJ9xf9 zm(9w(SbvE~z@+~kXc2944piXlIaXN7qP3o4GoUQGI5*~iZzXk(jKrhk-)^Ns@GM%+ zA&?d|WXLaFQPtjcXW~%m*xVmkZ+-3BSIltCE^U53`^7L>+jZUsQ`SDT-uvRF(e6k4 z?gFdpT%tS25*L6CF6c!{47ta@;{52!qQjRWrzFvaVMQ%s-chDY9}lm??v+9PmT=Nw z>2&%}4{ypuf$h-Ow(BL8`W7vr#>hQvmUkaK7{EL;r{lHYecebP${g{fO?j=I4cCVP z1^7CwEB6+F9IjOXMT@y6!*fSOS@h1^*RejzSHiu4n86R!e8 zp)cClZUuly6_7zW9B3I2;$=i#j~Z zx-W@GJLmO4{T6G)0$jEdpjH~s=Az?i!{X#}RZG(k|Mb-n7^u9@uasw9JlDi-)@u^= z*6N6_qTk!7G@g~2Z2Hm}3k$QB0p|IaCs{YhKl(v^EEOF(aQvSMyFr5>QCGDKC*t|q zKACH=J?AxRbtQ*Q@Uhf=&2D2}f9B*em+4GO!};;XwWAzr7Yz?cYWUvP+?2GxrEcF> zSGtuo^Z0N4^E=pS{R&<25-$$jfj`n%x@)^u{f9No;9t6I)`@kQgN&Hdx0Ns2?_2~Mv*`3I-gHzt*e#gMU+)a1QazP&Vml^lAdsQ{rZyHweQfCkt6dN2I*<%SnAwAvhzo3sksj*D|x&Q z$LdQkD{LGL{hr&@+ghP)TEEbNicGOXNNM~>WE_WPQDah`v!qP_SeLlzVxU;${B2L|5th=N;1NUS9T3;?&E}7hgqO8IJ3pQQh+C(}Anbzggx7%2v)e&Frj= zSE9U_^)RiWjVb253~;c$dFEbV(AG5b{Liu)F%fs}97~cFrc}KV53v!Lq`EZQsGeVdTfmz8`M)WoH!Id0(>Admd8pX;15kJeMP) zG8}iWCedJp{Z^Y=%j^xNC54>U9NjseH}>f_mVe^>)s$fqT;jv=W|Eun>d2cd>Z!X= zU9bOVd_n3CS7CQweRt`n#ux#QI`2Ehp&mXlp4l3@y8;h=TR6RVa&c)#!dLdz3zMA= zj>GAE0YNF3XRaz8bROgJd*7-NRoZK}HEF-5OWrnq=W-4~{J1m7s8i`}B)jqXsX9h} z`&ZpAxhIEK^wX}W!I5(C6K$s6ZIZDqJC;4_^pjPLV@*zH2L;2Gf<-Kir$*^@1{PhK z;yf!8eT=8F=B5JnbanQ=P94+n${r|VZ8`1uS9C{0dP?8aybtRbb*uB_?fa8gV-xQP zl_1O496Ri+(djpej?21Ae@t2c@3NC%9A3x8CKLF`n_VehQK&U1FUX%lt(#AhqSpKlX01q% z){2c`?~Cuf;bYvk4`%+ISEtjHr@OnZnQme$9$knH4c$uXn@V!(vTDn+p77u9oJ`5y zxDeKxyu*5rkTOsfu^&4npzr8xAl2FuG34kGRkqhAraMT?ad*gBy6jVn zF9c^clDIZ`-wtQ9G4b+!lC0Kj%bM!%GWby~ckblFw2@hb)My{?#$q`QJ7$@5r;lxV z2q>45@AHq*Ixn7Gbe4;{{|5Uhb!Io~7Do#G;?wfU8J1{m85x;pO=*QQ4WXf-CPU#x zj&VB<_Tu&=|J+{NCY*4eXbb=A8*U^J2e8I}WtEgMnfN^9@O_9EjjGq$%q%Zd9=+{v zj>_D?A{=`tIHsmD|H@XI{y?$mQTo_L8xNsWi^w$wA|rW;BFa&L>5sQsRc;KhGpbtueyOckkt>5y`HkTu|*^?Nf@ziTZ!eyU?v% zJulh)C$}_g@x*vdX2&opNp@p2DDB#x7l+zu?|l85E!(s-GQy^#5iF?LwQgbiz z>;4dSf?0%x+bB82M3F;}L3GT;s;{4)+_d(-7Nsz!Lu7n8&Uw9F@s0K5$f?JVazppZ z87blirNUpYjr802zeXDmQ?r3*LnvLz9Ks+>7= zyi76UdGeeSXIrB5>W#-mn_iz2Q}Q~>GJJOY+I&BQ%+xhrYKYCuO^2ew>JDwG6)dXb ztbQR&`StR~n?`J8LH_@X1^%4ze*iD;D!5n-=IhKy#@EmJ2M7H1)9v-r_0xT8>oVQh z+tK%C)ZjvQ&;Dhdx0z=bQfyX_vq~(p5g%tIXIeO*Q}8a%gJ15Fn~sacKN38Dt8K9u z_@`IgZF#@TpZ7es@jrR!{|zYfKfe6`KHxe5C_^XFCTRO3#$~}gm`YVRmesD8##8vC zr+wkl75Svw%je9rmc)Jy+lJTu#(-MIO%0<>B9`op!>3w1&WVd^9MNrWSxvUp2Ly_?M z17!Di&Q4bATd$k3H3{D>s@@zEe*2wIJ&|I#(eLnBz2e=9{tGSq6!}<_u2BO=LJ+#R zzgnwVtTFr-b58{0eWs*@S2MzsscKwwfImvO%oP=EGpgTqjPft7%;@y&PvhpFV+rqsm@CT8Z^44G}Y zC!%RzfjS{6|N`>_-{IkXuK}bz;Sz z1uDv0B}J=ngdCY?Q5gH2=f`rV?}qZBg_-raf^q2vqOPw*Xm;WnvCa=<`yD>4SE8w^ z%T8R4O)KkHYtm1R%MJ<<&a7TI?>FV4CK^xO3K;0plBA)rcjLghD^$jsxR}p2v}nmo z3W6t2w2Y-LRkVI6mF{m~6%T2UQW+mN9kV&n|I|c@o0HPyI_@Q};BGx6d!~@XYVmp) z-pxMdnWO4E;y;>p)H!o#L}O}u^>yw8nX9$;|Nf*U^>vnaeMyh#a;x?lS>@hEf#I)P zZn@)w(J?ZgKAKEtwcD`wKlRC(K(W z3DM0Bs=$upK-4q!18a}zHh8WXW@4kvIM;6;5_h<;R(URFzYQ%)a`pj%p@uw~AzLbY z#+#CJ`C`jxXO4LPyym$;b;=P@{G6Q;+OoJ#5}5UfXHV42Y+9CB-I?pZR;#n-NiI6v zxFr@Ww!d00VfLF&mi4~2r1_%xLjL|*dy<(&g+<<;DwkQ)oTh`MQ&;`|Bx7Slke7Sb zu&o8(jr92_9h!XjyqF7v4&Q9_VnGU$gjqWQ2dhnkiDzhy)GSE`ttlf_Qu z3W-HG`H3uVSuF16kZ&O1qSlRvyXLiRm)S^1t5^ZhLwb)3C*bSaa1Z5suFGlv1!!0A zA20p6vjw9(>EPrG<%S2i?RyyPy$^zmRh651PWn|urjF&MS+|*J)jCco8-uQv(AWWG zwZ`C?R1Qi>hU=Gp`*|0I&_E{U6x)R-c#>Qc#-_cOm>=YZN#~E$|K1&B)wL9V+@2nG z`+K+-7k*>wUoaj{RtuS3sn(X>b(t^fd0K-*Crkf+saBqYwCVA)(J~XGK8Gc#Uu2&3ZyrxT%p6zvtzBwuSN}oT&Qb8? zMaDXs`?`WlH#>E)_7Q>9=P`8aE)i=RI_mTjctnUL_T7U4)!RoY(VFENTO@7ef*7~) zX{34so)5fBtTvI4_umIQL(np9EIQ&tJGYvRoTgd}vD>pMmdCkuQ9Zheo-C`2!i-XxuM%Tl;k)>_Jzc`zgN6CWR$Gu)2yB6z}>t_*Qw_nQZxpjlt?F~0=2wPA6K4_-=>usDvj?!t`w7~`MqDjnsjynAsEptAk zl%$%LZby5v$mE$a#n{Gix+A69Wrb$IfM zxnXy!RhN^6*jx2CPgSxuJL7ZS+g#`GugRRV3~lpVY>Bm{_2%KIH@08h-Asg6zkqp)^FMFGZq*(QRONA~yUi(ndv)U+;8u`F#8r^(wrsMblYE_KWbOK7Y`Z9&}q#5 z6J_@z%KBvSa~?b-c$T+fPJOeZhjG`&eAB^zwOz`g&(TaDRhn<|R{BsN9n7N@|1h_3 zE6Ut?N!h-kOd30dd(Mx9X^BZePEcR{?>;vgeWgxKIq+6*V(9+g>tFtS)ABf@*WuLLw=zMcKQsO_p!|G!ILY8ry>gTD zPu|$|PimSH&fEKPg&$bHi$WFCNUQu@yKK5JHQ`CPY;Qbegtu0KccGrI^;09IV#DC& z)qyM)mrhSOadKFjXFLlQp;d&tE2kIwAN+pqt=R85;rRsnboOH0P%EoxwCno8%lwzm zE99Gu@~B;`$W1*TwV1Q9BHuzunz33`(xzw9=rN3nQ2#>y2-N1XP3Bxp3Or_3M?uu1`KD@#RjHD|^fNs_z|@ zLN|t~k9XK8r9Kwp`2O7|t~GP5iehNOliH9szv?Y}za>^@p=nXe>FwhY@t~HEdnPaX zfr;W*R~^=H?4yao7oK=dTq>&`6%AGVoVa@iYh`cx9-in+pQQ{V##d}IuwG|j_gy}4 z1B-`Ga-$qShj-%Be*(9ti5?eh4gS(Hlyjjfj8<=r)MW2{O~rdOjH~BcRjp~(KX%gF zE-a476a{JuPkH|DWgweX=*=U?xAFRJey=3IA$z)b!M(MDrM&co-)#4Zw2WWn88T9?kUp1`B<@#SVn1<>_w9oL zm5OUw4;(ksjHeuy0{y$#|9jQ=O{`1pZ6uypstp5qA#U`4lLEonq^od?nbwjB@!PV3 zpj+f;`04+sZT#nV{s*Pv|68pUG1t=I>Kmt0)D(+eI^D*<|8!*NhPCqU-NFE?I9~ew zDwp5i>8GcsiL|pWjBVqw)i*CjD)+r^s+m9FIaB^J;M{BtA!hj0Fa#GlLf|L}SKpFHHXBpZ~Mvxx?! z+riHd4al<_(e?)2!$jS4zqBHRjB_08$qf%K*JYz)Yj*-K61*PDapw}RL;idQkGtB> zKHgdh;^AoWl1w#Wt;qg$bVbL()yp;Ab>P4Oo4%^E_$$Q^9+Sho%OmrUQRxR8m^+7u zT?z#-UINvnJmSCp)E#^aLZt^y(>MUqiYQD&r&X6e3DvOig9M#K9?>s=_s17#BoI&= zuahr){~kHITi{;IIy%AkP%kbm(YrpZkA{rmR}ZSy@~B!V~B z_?mB?zP6zI|&w zGj1FNZuzL)^r*q7!a^x3NJ9rG5G$!vyb{Go9E0($@8920_erTQKrIh`OL_*q`;Xt)vb0fze+7kfiu8Wjwq zd&!MFi=;u&PzcfR=FJSGpSKSm{@t2!{WJ!x;IJy=k#B|C9Ugj1h;PHG}^fjnS)3_ZNmB!5im5)vN+*cpM6uK9JBbyd9mGPy(w1RKn8I79FT_)!7_yWb#YoW(5Z?JuTzTCR3qvR*M%`wLTSUe+{;s;JWqaj zJL)Yw1CzR#D6mVz19oiL5(2{K)%HB+CQ$i(xYXW4Kdls_Du+@0C9uJ3!=-m+W@dIp zijhZt^EtWa7<8Ghf>=qYm|&O@TPJdd;C~T<1}G_Pc6N3ezQ*4>I*2X^9}zo0(;iZq zKR+2DVA&RfEmu-pOu^P5*_WuLSq_m^aygy6Pikl@l!*&-)8y#jsdDS+=u8YYDnM_l zg2!p5rS0QaW?1U@*Mp6$afE>jIxvyX5P9W#RxwTJh=_Cuq#3UcVMas#|*y|3|i=ON;mL3pnPc>ts} zQyAHSpsRXojR-j?TLdol-3G`dd#h!|Yr*~A1_C}P3}w*b6V44r&^Gq=#PSpGL=ij* zo8guv*$wz0RZkg?*1S1QPp12L45>DLI3nmP+w*lmE>Ssu{vjT2I8k)sHVNr7 zR;ttvTuH*hpP3qohmex!g}{ai0^2_tY@uinw>MfJCLO|eVuB4z-9%x?grLt;0&C(P zSOR!bgcV3W?v}l7xnNz(P@ub7DnH?8Zv=Y2zx+^8Yn&xtcYboq}mNx zOEI=+8)DT@x-7_G2!|FD`m5k0IP``2j(7R$;K;UC*Ip0g2M{+2(A{UCP3goHf+j<_ zl0=zcV@w{^HQC>1aY-++fru`dh|P{&en&@MFV-#FUtew!iVKvsU$aN@Kz{ri+{z`Y z^r=&~{}6H9n7;e&rW0Yia^(tPhY;-=(Wn6%LZHV6ZbNfc%JO*dVP<|j;XP*6dEWWw zd-~C_v5rbzasey<(Pg6(p%Yu2yMY)YMFiC#C6AYI#)IZmU1uPA2@IJU=n|c$yL`b8 zQbuGy<}iL$E7zg6E{rT72_J-5iDlcdc@S@|f&fC+9yZ_}S65dJ-w6p$5RC{IJ{Lq` z>;?EK1gBZFnhYe7^Rr1|t-N+?O3=Ln!dr@Yvw-vO?+=KoO$m&14PS%fpa4hk=FiDP z*@@aE+;+2LJwgRek!{H5R|5(s(TGDm zhK~&H_E=ArQikQyO(!}|S&%h`_)-lrp^oUUk6Y4sR;Nco$w9BXxi~ou+z$#nKL<8U-Mui`C5JYaIu{_bPtb#}(ENFXi}eF+sRlAvzUkSn(>>SKEOC9#WDGrRWw? zsVQhjjKa1t%U7(JfHaf%g8X?E33{NXnGH0jk7zZY5E3FheqY!Ziz)skqKZxtz7I)+ zNe~7;VoPwNsV2S^n;%#B^!al%p?rZY`RDOo1{8QmT#l&j3_L~nWym~bTGdOpcOSQ>jO9l*lVP%8wE(iL2{MbA z@=B4NgsnrIPW**zqCsRc)7oJ50#aPD>CP=!mjoygXXSsd=K-!~@BGY!0#@k#F_5Xi z-QLsyWKXgi%}z%A>?QvpEhPOb?%0WhM4bER#x^yw;H~vyOJDYbUEl^ zACa+u49kAX>4GB)7n-V7O3PMOBZ-c;+^8^yF?7z}uQK2i%RmuNv{2`sF&Ph7EnUK9 zT!Jc|Lv?HA5AdYn{7h(jksPltHzw*m07?pj_%HCJ-QYzexN#602`k+JR8w=UCHE`P za~?7|;(03(^@1^oeT1^^&ar%r4^9>dIwk^BHA(gwzAJX!?0g|fL}-Nj&Z!)eV6$Wz zHAsh>_@d8e43{8fk2Ss^AVOs>5VPjsP9n0woDngP5VOBos0%~fcbuSK(E6&tl>KuV zZuCsLY{SK|9%};14A--Ws6zhtS!vl+{;!TgQ|x5qAb;#cyjU|c(CLT_jBpsi8E?pf zJ=j7p6yed)nk^?5n7ZFD`-@)D>o59}BDJcRi`;TYxqFsvD`P6D3N}>p^AB$gEP2Dn zS)FvPq~s&l`m!V2R$f1~_sseOXP6|GeV=;Zdt>jVlefxljTmv)xr^pjh}GYFQfN1_ zgYME(Sf;j~aisztdq!6WY8 z650M700c^Wk|%%Bh3NHHMJ#)q26LU>dpoo*LC{tU%}Ged>%XHey}3>3i4ZLr6KjJ% zzkjiDYn=qbo~M_#YQdXLNN8GAYt)4Nmcy3U7`LwTY_m6d{Oab9)ggCOQ^v=wA*_?d zE+HB4>9ad~A_<0S%gu%yn*(V8p|`qs$?oZ~-zm-&>E_LGZgn<_$@d*~*FQR?N6SzM zt4&fn>wU=Fv8O=yx|n9dSrEyO3*G+|0DBz*Mnb1&4+ca=%iyXxMZ2iaJ88m?IX^<)QRaO5?EnBF~Q zwp=hVHMJIMkytDa5}F*4?IcSDt0NX3D_8K!6xA#G$G1hIG7J4vzV0 zqE-iUHx6JTq=W2^gXa#COvbI6AQ2#74k~$vzMewlmL%Qxz&N}}Ltqkr={-pP>f_BN z$XXJ=e*Gf!SCU|AqZA5zPb9FVrKOSZSQoDo57^DfOpm`UafE!_ZRJ(0#iX>fI=5vj zo<3W?wzH=vf&^u3SgcO}%IoC7TUWq5^O6KH(11SfYVX%KASDV2ZQ!bbNZSUwSt36B z_ZpUwYhWW%c>K$4n>Jmx-m01ah9C{%bO*>$Y*5$yxwC|83Hp2q7A~kW>~V?aJ$!WT#@b|9XudZkXdSp+&JX*&V=-I71)S~E&SGl1hA_21i4hM zod#@loyo**1&>P}X8VHWE!2_^XI9IMt_z zv9YnlccnsX_9Ds*KB1-Xvi2;V)AUs&zNRFDCrB~w&u(8Kk~LDVK(?y)?-3u9BUX|y z12h#4rsX-wlSlMTO_%Ts06WU9WB(rU3^^h@h}h_}DWfnWM~neDB?%+%_m8z9A>-xl#Two*OID&A$ayL@(CUi5HZbg+wHXkCTN+x!ooi%a?qH`|SloJs9H zEZbAAqjY)dK8MEVlj4?7tEzLw>Yq|%2fTN4E0#v&?AaKse9`Q-$#h0(PhNn*4@->& z#ekmek!B3^6Ap2CEv143zS44xLJr<<^>}*4w=_v*sa2(p{PJUBGMg}aKyP5c*B2Wl zUK$p2+Q+fDY+{&y?WM2EahiwF9wGF3tlZT^^I_0*hFDe6rA&KTbO)>u8)AbkGTyh{Dk$Ng6eD|n-f z3rwh?kMx4&C8q8%3ilmYX=@o-S-@KDzN@b(Zvf7))2nM~8fw(>vXb1S#I&m5Y#mQ2 zHAzZUgRe&PxjRuazk2R;FCX^4E?(5ucFs{CXHUFI95u9(S1jDYqwI?nYi4F!uHmvZ z>R_0{vv0Sgq$HxmT_{k!Ngg6d03D}4dFb-8o6o9FscC2!e|h2&knh@?zp`-)x6_xf zV0O{A77eQP&msm&)xEpy+*+9`?gNhgPb4U?391{rCGAbMI!&t^%6mH+Ip zuO-!sO5N|9v?0rS+MnIHaJXZQ_|NqFEBTM^xJ5-a%AT<`ec3b6Dz*Lf>mV^_b>if( zMOH-DdaIWNf~03qP*C3Nu<*Z|e$$tADVMxCy$DgE%@ml2=5mTLdgxBB53L_Zi^4 zT)HNTyay=JeM52aV#9t0?N}#SPF-1nOgDHKD?aJentIkn=V%;aZsk`FlbVm8gG20{ z>HLw|k(~i)XK6QPv((}?ZwYw*=ee_yrW1VDE-fxB-R>ZQQeBF1bqczpsm9fx|0Ivc zCk$8l92T3(q~!!8yR_{LtEt(gHYpvHD;L++F(IjH_N<3R98QqML1Son+%eT;h2 zv|~DbdiLBzbV*&E`Yu;-pRT!-6l-oR2A_Y=OniW#(~a4)?lfx8lSHDsdi4<@(uAN} zi%;5pXlPV3?bjHx?i=NwV3=|Lut%n^L8pgbZ(+amtHhJ$Eb#^|U(&LA%07v2cgj4S zJ^AXAdGIJlms$L`+V5mj2r)WqWA+?txa@{fcuH=ZrgSA1NvGPUvjy}5)eCp7GyWds zyLUFQs%<`^#^BwRecChaA6PPTmtI9xcUb`1c>7Q?G9r*`!afVnV}FPqxx55Ak^P#> z*AG^op%h0s*nD8*)?ntLBqb(Ry%v+^?{ZJ$^RvjaODNMz?QDkME+$_sLIY<}F=e2~3B0#KpyNNd(5DIb)5Es5g%mG8VbW$A=Zpojdor z^;Ju%sS2eEP-8>{Jj8GJoNHpMPumGIjW%|1)~DJn9@5J6e@pMoS)BY>>RI8u^9=QZ zZ_;t`XS-Z?Qyiy)sgj@gvhr-v!~gX zlarI!6+vnSy*0=4L|3TU!@|N2v9Z0Dh;vOx|00@rRNSIxAr@#!Lx6WGkTDV!8~Snj zjlS>&1$rjlIna-k+YS%6W^dcEBLcXcCLb2CmWzZJZ0uU7mb?Q4V@RWb;<+Ci%=mrF zH)aUOXz1h4Ow{Y3E5mu3Z)-}=R>U^xdaG^WE67EaZDhA&Gh*Qzr5CtCNP~quuWy;t^-)oC*EXabzQKy`uoTRK=w3sME* z5C!ED*IBNG1o}5Y>$IF z(5S;Hl&RiGUDxH1%Z*ngpX_BLpTM*^O^YZSzB(VCe_=x9%%T~k_JtpEjfJ=#$d zXp|bG-ADRd@DSsNt%_4_eha$WwU8OEV<6w|bL)ByYGU>#I7bnGO~AH9$;Rx@CoxHcE9qK0cHtdnw0vxRdz=Ff|M;OvK`#_Jh_pkV82p z606aWAtd3ZqhpV%E$K+T6PH`H{GHa{umb}&tUO``cP4&HmETb#b}UHgDm0;zn2Ear zOo~2gYKDb|l0LB#k#YBictm3jVxnohqRl4(`?!)(nXE5X+{x>{%w5DXcr`ThNB4ON9xo_cYc;zn zWML2~_{}SxSwDK8)vMTa>!Gi(3J%|3N2*p$r zCE`p2yU9`Rqlnxk=%QDZCGp)**20+O_U+qSF}5IjRx~rPX-LsHj>dNTFKySEz5^6_ z2xHNSsDafi*TNN)~+oTSn>1GfmdH~OG@Dv>!t#>uR?doKwlpY14&;Cm*y3MsuF+EjUI$Y1e()j z*KJjGh0xsdQL}(0cT=uoIt-^mF&OE|;2VxbPt62t-=r-kv(0HtdTbK$?kYApJhk~N zPF#eO0Ng1^SAOjJ3i=Jl@*Q~(m;?eAhu+;}KHlEmuDPA@ zJtk|Q_XuIkfJ-z~T`DQh?mD_+8Jh>OxS%Ws*c*9~PBc1iM_reu28mbhA#z8c42!}Z z%gj^mUYDMbkifN45BnnpIzFzE!ej))%a{HX5~DhXqlQE-OT3NAFb=Sh=DN5*Ql6@+YMl6;ITTKv z7*E1j?eu5TZn(_w5=E~)R_0#M;!tF9M*j|&vItZddRq8cV{-Yz*VlJ&aB$R`o1UJ2 zJ=2Lx!A_%((Az!0%gc+;uwB|HK9>NnYG5#t?ghUsL3fP2&nhu(6hm zI~peq$`eXO_8;)`_&XRALmj7I_w0#L3${cUl(Pfx=~=&6$)J^x>M&t~wl1+ZB0i0o z>%2nI#x=o9hHAHbGVdM8sucXcqZN7C*S_B-9V09qGAF|Hu@=o#rA(_XP1o(ypI~RD zsZ4gigbeyMY#x}Kbi!8!w{rRh%n;CuSGIFr$lu}Li;w0Z9fx2mnV{`D1Ls;|a)X~s zqt$jie{95dYQEyr@nPbvQdPEmpdIer`wmI1o|Yw<5|@W3n}UV~lLt^`vGOVb3U2^e}QOy5F^7eK{EKj?yiMV7(Pa* zoj*7F1aC4NajbJDxri6TKKS19j9(}4Ky?GlmQ`*dLiVGf-;F0GNzxG8MnnASh!j4d0)uXedVOmLxGTF-an;=8=QO zJ+bg^1Tq+7=M=IyUc{EHLJK8%5MQY!9eXnS!z@UkbngeETMH!^1d$C!Uqz!26N>fW zvltKC*Be}ikKQn93)l?DL~A&1pZ>fQGZ~gvPkFF-aKtCUr?&8uZ&&Ag_T)&$}@Cl}X6vNn(+U?0RqDeJ_{0 zON&qDV*AKiCVmqyJO%zMAO9vE3dC~})`ajokA|bmlEoTsl3;@evildk;1B`225Bem z?jT#_;0&Qib;`lW9Xe%Q>!+FnwKQMEapD{otCO#d0HlH(#ZD~v(WZVF069FFL*NDj zDpHe^tKmIagNB!5d1WYN3;CcU#)tAq{U~N4h_|t*s20#UaVei}0Byn)BN{>N12?`u zca7UKX9@J<_2kJDg0>MjX|Px5b_U$8g8MHGfvE%pljclP3?pQAp$mdHXxP{@G!zR& z2WEg88M`J8CZqj`wmx|VV(5Xdpdcdg?=?ndWt$zsmC6vETGHL_Sq@b^WoW)=)41LN*Vun1P!0 z2YJ~0wEWU~fMiBInbH`pPZzhgiZN_WF*x@jJM{#ub0YY$%FlQKa-!f#*Wyf@PvlkOW};$Y;eveoUac{A&g zbCPt=*ZMs%RaHCki!jk7%Gac>8*ESBOk2HTd+%w(aYJ28%XnC=o?;-beZ*@IR>Dzu zZiEFw=X9NbfheFTTr6JK1ODY8GbCSh&ejfB1Kb9XScd>@P zB6FDp3!eA33k+B`;wJEvMZSE2tn4L=M|QLlur_n4>~X`L}5C5E(uH#^;uw!+7r-_%C&4@ZOMHySH;Z6MG=6 zAug>HFC72aog%3kPFjX#;dUK^wiVv&=Xl|>8&0T#>Cub5SZQs8)APpS_y9gs6f#>1$$xT#4U zV?u9q;($0J05vY*-J{VVhrB=xw7AaJBfw1{v!OaWeL-5<8+Kcy&?2t2=$4oMy5WcV z1Ea!mT!Mc*Oi=amWaQ86;REAsDcZr4k+(ianSsxyu6{l4S_N z)K$zWFsM||Ypq7GwZTlrtXVej#G9W#ufZ)FC;n;Pnx%xY3NHIG1jZnaTm?Y-zu0@v zs4BCqT@a=dj2^vzDLcGrt-0o!^9c}?rcUOdeFkmZ(?t(=HefRs z&lcpMh@ep7fQlG!eEuRwrCMnfzh?4~Nl4(Xh6*ekS3Zm-($9=?Add(TE77<{<3E zJ5Q>;Xx*38*Q>&sg_uB0iu}`#26LWAEfYCnDn!Tx%-fEXmB0O`y^~JtF7%mVFmqoe zeMz(-6Hxeq;X?R8Z8s_DBcN80rddOjE&&}xGT=`dYLI)>!8xv4y*j&Vl4LlT7G3@Q zresfn6Csc-3>q+SE1G2>_JcB%e||c4B>7IWqzmO#J8r$0Z1?=K5<>ch=&)-Dx=nO2z2hPILV*lA&|v6HTH=GLBn*~x_a2RS360Jrk2Yi`h+%W}C9 z1kwPLEbk$i4d5r`)YM%_SDqM2k@AD2+!GNxI$I(My)<-%&qt%DYP9aOaqc&Awmo-+DzSRP$tZ$CYgmAW0L z$?b+tONYPjG_~Yn>F^UXr$t~X%CE@o{W=ajjr}rbx%;v6qRM#+ zs(UEi8Z=H2WFA&uC`1L@YFxg2*|$iK)fgdx>TT{CiG2~C=&R2Ao}QI;7mDY*`}?`j z_84ePW-A+q#IX!gdpX~_zsw`Cs=i81tr*I^hY;z57Xgg)YeEJ~0adFMq{bsKH5DS_ zr&LtHyEJvU65$F$dXk0!odP_@v#_vYPXsp?2ybN-yx;t9=IPuND05L+h}jbmYmZ23 zfBDmXfbmp_FU#Y-r-<4*K@XCxG-CZ8)Z<5P^!D}=Q~@dDphQQ&OT-lkS$8ojpT%rK(l&C|X z@<4_XX(}j-lA#vpG-XUGhS*XPb$FImhu<<0?YADhc-nUvNm0VkfiT5VhCIWMh13xV zZzA0ukj?+q7RNTP4Hg&R{sJzCoxKS~s4N#~)n{t*mA#ge>HoT+v9-sZSiqvQ_=o|j z9@PLakO>B4Z%a-ql4q$qYf8r1Eu5xsGe~U*RcnS%n{nB6+(8_BBbg3&DFVpAO2Nh_ ze?)Sh*I!KI3h8p|TL-QdFgxzFkZeedY#pjP|L*4D*|e@FOS6TjvF;akhbL?9n>YYC zzbXo!SL|D$9Sv0I#P;^p6Xv#)lL`#Ev*XPK?3y>(?Z1?n4N;WsY+x-e0##BK()L1? zH2t}(4K}*Sa}Z%ma(DnX!;Vy1TAGSoGs+TAx}GB)D+P8Qp}LwBp`cJo5}|?}Lb?Cg z0|i~?+T8vCx%w=2zPJO_$jC@i31arD=@~XhNrVXaA&-OLX?g5#?GLbf=>^OWl}C?K zUmh+LDKS*q#>Ol<5HqU-#q1fjY&jp?v6+=mLWkU@mTC*b z!tRxUS*c_TBL-W*;cLfm7rk!XlDhfe0`idF-lzSiR4mmQKb4elG(4O_??D~=h!Umu zIjE35c6XP_)2c#A9hJ7+fdj&hyz||Cea|8%=-WJ@V~H-3eUauQrmRR=XgfKHM}NZD zhl_8WNkXS-^vNR0l^-A>Nm`hwHj+Bw1J7FW0yoVn(}b-)*L*M_I?V z<|xp**c`?t*#-_c>J2g=Iq`kIzoz%V0y1xWV>A0KX@OY;Ro-3=+`^Be*BS5;yg zc6@NGD~mqWW7m|)rmDl;8vahEA;m%=7+mr8`#5-DWk7yR!H=~nX&$#o<2TaYF=2~W zU_Fvw0nSX6;s*@jGKVC{uc^OUxRF{6a2#6|mLR7|B5MHQXXW?p1o(>`wQ4 z^w&fj$6qx~%$xyor?w!`jm7xvFG_f%k$H>f$)FlvX6L69UE|u8rqc->Zf8#)bl#aY zcJ{x*W?X)q+p(7Q6!7?Cuo)v+qpIpm^D2)&H*9!;GGJ+CW#uB~>Tqen9?pG&M=zX8 zsjW?4iUOf`VBm}Q@e&{IRGg|P$`Le<*Vq3urK88MHboCzn0HBMIfAE;JqQ@}!VFi= zuD2SU5;aXV)=0@*(~`oY4)atp5Khe6$^p~C^`o4X*-a#t!#t(pK-Iu@VQaawVhS3s zS5HUDRjgWOk!aKZFQ5z|wc+ivUoXT=d@8rk6a4VXKn5GVK4jrp^K;W=uPRyR-!o28~JfSy`!G zwL(w@ScrJ_WF4pVH=^85oi}US#6!t?^eAk0#aYwu_k2+_dhQkWAb+P9OR97AM8a_1 z$dr+Jp6;twoGZpUu~O;Q6?9`?tik{tu&Z0b-+)T}O;~ z2D{_lkBJR-3$nIO;!MAULiK&D&oDo7MW1q3fB(|?QP`sxV*x;+ogk2ir(hYHKWn!Q zKU@HQ{qVpUdWYT`;F&PESUjVB#9nk;W79dKNth?A4)U%ocU8mFzAnZpD?0Jcq$cf> zn0#aiXNI3*>?y}`etMzH;xRx_{@{$wb^IaPwR&YqV_Iu3&ct=z>12I<&kE`etf}b#LPo9b1*;XV}#kqzLvZzzK#cyR=WDq7iu6kA{aY##bdDypDa4M(sJnQ z32cuE=6Dgrw)@FR-4_l&EY)m1?4$50E z9Q>2s8K=}n4fB0?qpY+vY)Hx37f>AAn~!Hccb%6zz{Y#A#QO8kEzHwg;_dG6&!WoJiu95h3GYkF_QvH+Tfh`wYbMwERVyatFf>9tNZ115HOSd08T(=A5C6CTZ1N&!>6ij4=?O$xpbxS@LOTWh(#6L*wZm9R&X7o6r)=xx=6?U4x!dowDAktRd-&QPVH$EIWY{LA;I8RrzhK zz1X#O0OtWr9rJ0k0 zRCF?6#XnbF-|}eYS!e<2fjP@z_aU8nBsOC+bIEtj8I=GxnOdp!H!gCpt#1Aoz{f6& z{|(Y(y$rF6h3VWktid=3>3h&bCM*lo0jRg6zEorhN^3jOY|_HJ0BKqdz)3^s`iA24 z{8LXMw~?dTY&JwSruQ!dLwP>OL-j5xDp*|KGT51vEi z5)bUa81&hrU<5`qj{!BtM+=q$Bp@kHRAjN6OvI=I2C47^wz@<+86HxKf`U1{WP2vr zw(v=^5w@%nizGJ%rA0ti4N>MZKs6VQ3w)Ox-gV*dg2ZO#)G2<^KPHI}6ER1GBs0-! zUWB&Eqe=g$ilErXzI~Je>ZTTl2gY3gE7A&IYzwrp$zPnSx(h&MBI{0?g#fAuiL7$F z2?6x@b4kY%bH|@a`|+ABB5w(w)yBJ5~y^`U&~gbBN(Cg_Rim^=1|mWMyOPk4j}i00Xt~a^D3V5p8`c; z5k$2B+m=KfY6?h^RMfeV(3_tuy19&k);mlA4}m_2;!N@4$5#@Rx9&rUxeVg!2UJHN zc0KsG>k|P}Un2hu@(yh{NFE@?WLI6(!wJkBKSj>rrB%P`EcWK4l1r>#Xb*+`cVO{^?zJCV68s04Bc`aYg9Y1w+9EQbJuE0Yf=@_xyX2}CuRa3h{oqHq z%cseEKr9e17ChM#l7LtdR0JWSkplc8D*u9O77Up_N5GGxACH2Ze;2w`xX>-kE0<>i zOAebk$h-&>aX#F9L6MDQg6JbhpwoSXW>ySp4t8U8YOtHWbbb5gi+Dz)ih=eMD=d2# z&#D_|Rj>`94|DE_u0Wkyu zh&Cy_UZzbmR@iIPF)TqBqMmf^Py{7x#{DAeR>@zy7)EF;#H|U~>_y(m8+inp4Pdhy zk$xqK2DrVL?Mts>k4XXzY6+X zFOeH|dGYJPZ**=^sF0Vy$BKCA;gJ#^BiK`0nOEEX0S^~p%kE2H@4zI6*OiwX=*iC|KsCFt5MMbNX1wD*Jt`%?Br3C7C*6z9Ba?b_A_+B z_e_D*1HIuXPE_?R56fJ8%X!VVzc!p@mk z(h=?e4t5K5L|8zdMqLHd3H;kuj|GYh-FZAK7gu(eD-p)_feY`y!I6;o@mis27<%DL zKDzuw=N*QyQZU9M1^tDAO?OcWk!I67^m2&T5@tV%V?aZR$;FH$ z7%#{5MgrPH_?D!QREvR&yu9Yyxk72QA4vE>k6W5J`k|myoA2z5y;f)_sfZC||5C{V zdMFoRP?MGj>J$qC@woZyf`awmE^R^>IZvG7N#Pfi+#shZ3OelE8yA-GJqzDgg5jnk zx{(B4O5!t^*_TK}vH9`*<4103^b3eB1d>csfKLgflo+KZXaV<5PNG1IEQk7}w@N%; zh$#!Mh5!LcBv>Gfr0n70;gKa+7Yvp|sM-rfV9|uC%8eOhErN?6?Cn3VvfctXoZf>uictNxDNawwT1f{~1hFxv?zhERt|(FrX>Rn%x7c>Ye# zZ`m7gXXsqs7#SFZ8wEM4g0>Wgv6}+xGvdQc@KXRxjlKKxY!m|3LNafXM?w1&=bx(g zb~%!181%^ih7^yGAC56r_Wn-(M)EWmoM*b}=~$lwWFkKEQ;JxE5;O(D)MD8scw$UD z;$uYE*x<5nV-~1F#3ckeL{e14&8lI}(1lhH8A4d4QJg;@vWJ{C1L^;|p!4+y$^aS! z#D~=l4TX5Z6hwOx(9SF@Vc{ro5E7Lgc;)V$aj((hA$c&NL<0;#%Kzx47FSiBg$)OR zV=8A`n}uFjn91NQEum{f=3oN+BT0FwUS0&7f>GVu6XaYE+?IiX0V_&@fawt`Bx(Pl z69dxOxl3EX9kxQX8!fIg7+We>PDt+o;{~l%f)zxi_a3o329TRUeIcf@my!df6R3!b z3muVjpb~BrZiF#b8sgr7hy^4o3*g^q)vLgGX87k%d31A#fCqvnacD#H&GwlrF)eUs z;v*?50LG*niD+py*N*x+>1x#1*OPO_Qzaw~;uS{ZH%cyYMm_wE+*|HVSVnf8(3KW> z^sa$}n#@Kc9=9*qgP75vuMiD15J6<^xbb<3*sssG>^|RIfyR*hX`2P@X5R;xGFQ(6 zE!9aAw-B5Do_v0soI60Dgux6?1)>Cj4j?;xEUGKM@*%R|x(Jz$B^t;rj8k8KfB(r? z^u|#<!Oti6E;?00%^H99n1MG3qn`;jN})9qseWcqg%))7XB4{ z```cL%L5{GE>yIlC}?2|i7cmEeHEL`(7oh`3YX&6zKzq$R)xgG$s*cszr%0q-Xk-D z6dM-*c>D$a_&d-#lfQm#W+o?3=d#gt+3|(1;GZO_EWFrvlW{{{|KIuJWsh}qWh3a` z_aljEOdEt)2gZ5Xt9YPRIWdOxPAuFIx&kwKIN6XygOL53$3a*yMUve;$jrUgvq5OunPlXr{r1?@WY^U@S;b0 zOm2bUj&kygHSe^DnPHmGEq($4!iQa-FGS%5xXit5eKWIo#MPc(Q7_S=-{Cl_K>zpo zS%X8_fUf@20R=q`4GrQy1K@tJGVS~_uza@&AdC_ePgxrJmsr+nf)|HN1~YjV`4&`2 z8k?G)rK1}igFWQ(2ombBY{tuj(v2SMDWv4oE8=^O=oZjnWxu6FyE(hC8fLJ;iz8-a z{(GyE*aBJY_jC)LISN>>QWXQ9qA^P>j``t_g8N$(% zg36IjM)1i8J6~D~WH576_nnhhqk?6c`F%~@A} zthe{0-Sw+gyVty9W;GnKa}@4c%lcf*Z1E1#{Sdx=^QGSAJ8Rx%*yhK@?TPBH^e4;` zI=atSx-SkcmES*YbEW3Z*|vt8TB4mAjXJVBSNU8lspp%lK2c$38lo?FqF?b?j)~yq zkaZjAYolaW_&ePl|6c7u=}2)dQZ>+|_&Gml>f<*_*?K1>eO%aDH^}hJ*4SCmG!7nC zicf5v;^3Fi(a(Fe;v6ILMbwR>OZuiC9#H@JG23lQ?{W9#Rz7GTOL^DNW}ItoNKmv> zk1-Lv5+fq}EcQaDr0`)8lcpyMO`B4ssiUvmtD;mxBestBbsl?4y9B!1H*Lb(g^u^8 zzTRMQ%}OcRGr-{5)ep}(%!}W~-j>a`j67>{{ah3;>vJXlz{kVvKf>cpRZYG=61SV_ zy5A>qBf~0Z<*hf$Io|0a$4AvHzOtT}PdT2rzBksnYWAqWR`KB0e6tU+566A|Y_^EC zm_O3gY)$EHFnBgo8`?L~viZc3k+T#v9es%uI8y)kY}TF ziT#~^@YuZV0IP6zMqexIyi%i^4=Y9N&VvuhFXAsxRoGenRk=eVCOu!{g~^1FH_O;; z(WP7wc1@i#fx&`ed%Jp7Xy4Av=zC!9Tbw8z%o5%6^08f(l+x_sp8S}$;3RSX{UmFEjlV>&2*F+gFH*cO&bHrsEjXLtY99G;yW)ZMXDL+YG4;mx z{fDl?c4dI-N(~c{^*8GOQ+c{CK7(yNb4r@ePu|ea(|ygOv6~s<-i@qOA8kA|(3(cU zv#u$TTH!|mxnCtT;wt3cnI?qk1_NopG^0{pysm zSbbbt-qR`Hg9#U-v`6ag#s`dJ8ly_q{?C_Ps*<(UZJ=p$ydkq_^LA1DnW(WKN3Ka3 z>dXxNSW9+L(`Z>~xw44`RnwiZCQZUT=|OAW37+iHl-_iUFG2@jy;4rlXQU{WM~s=Y zq=m``;pyuHB))P^5YRbuIP$5voczkc>Z$8>oBFRDr#zDw2-uc5v8ji)&==@@+H%8h zWnN<5_JHT9{B{py?F|j@sPbv+&VKLN8N9L0AjPag+Q3jVsaJHq!=^4$+hu&k7@j+tS+BW!|G6tacDPM_5jC5#dMdX=JW7YjA)LwH+B@6mS~x3P zKgK3Sv2m3T3#BYSFnU%rz4h;T_G@+e%68WsvkhEX@;?d8>cBYEB|b*63VZ?yyB zZ03f-Qg!a$o6CCc8^SjyrmY~uM4@hH4j7g1q#JYAi{swp&JsOu$w#^K!nG#Sq)|=h zG*#M%Cz7cx=!)Z?Lj_Da3WkQhWRY@u#{X8NXz%Qfnh!(sd3w2pM~;kr*Vx%N>+JcV zUusI4>gLBCUQ{GY)i|&|cjC>}2RFVQb+q{$_rrWcNZ;7D-bi8o>oW!7vQtyj8qXGE zxpUIyJ9-SGo!C=Ss-YriF8;VMgh}H0i%kJpOxHzbDl(PWuHP8Eq7tJn+r_C;lB$D| z`BmSVQ$YCqoPbHRhmPy^>G1HdZ$H~MXWDO4n{<&B7B+I;QdIa$f+5}B@41#{r%q_v z5gG4t%iQ}te=?*`+xQG~L`uJ|i`TpOb@De>t*l-7oHy(9RvO90K8cL1Fe^+yD0D@( zfzF%=!of^j=(`XQmw;G&+ccL-TY0YpO@3dKK+Cz$MF%3CZo&$w4w#_NFej_}!nY6sE00c4Gn5_BFCK`)- zk^e`j%X2r2SB>VfwKg@8o4UMdr%N}d9HysR7*nUb`=yD3+{)lTzPD#qw6QL{>vBvU zZ~E`pVDd5Ae&J2-smsrDWoTb@bun51vf+RG{^7XxiR0N2e zxX-k)HblurRLw8>ulk9J5es$Eh3~j*`D6Jr!hk`+8o>ZowYyO1vSOG*$W5SKpE{w{n8W8sg~rXZrvO0*qr{&Z&HU- zc!>Jm;fPWfeQaNGo|p)hFV5e7E2)x@q^8*P8?tJje(A}KdH?ln(1&rW*v>C1$E^HQ zYnyiF)g-J{P|v+_Gc8do{EnGO)MZ!crkr7;2mK9F^XId*HrUVU3Ojk%wNLSl zaBA-slFBt@8HVp$*Be@-6Bw0n>uyhfQ>XHOab-8<-Z9JHQZ*aXQQvBDS56QqAi9|GYl_OCfi* zb(KnX{tK4qL5Y+7bw|HU=@0*gQ;+`2D$`}}IC;Lmy~}7pvYL)=V(SWfCYR@9_H$iJ zm))dxe(jDB4?S6?pLCka4Z05=pW;To-j-gk$*pCtKPrd?-FQDG9Bjx;sdikS_Ueuk zRXN?*l2V;CYtKr#&s1qT_I24RtmFd37s80i%42&Jk}dW8nY&Bl2kl%lRcSNBo>`QB zP?9yLj2*kNW#ai4jdx~dWHqJO`m%FSq-EqP8_u;EI3nfIpZz}5lM!Av)ii2(^}08U zfoA5O-o76v-|qMJrN+%~JMbyOTv{=NrFZnP)MN}R+dmya?=y38V_{c!z?}E>FUQgZ znS|0cr?=gCoT#3Vd%I&WL5yj?P($Fyp6_2d*^G?*FKCD7FjkA5!9Jx}jYpWHd@-uG z^dkx1h=+AvI7hPcc#D&$vZZC7&kQTYY3^oYo6B?Ly0x7wnVYx_@0KoH9bLIQl3dnO z)wnqRqK6#(9d4-u-|})rY=708r{*#vqf&m8?U5FL-dHJ_rDFb<)>t%Cu%l>NQ#63< zMO^%Fme*TMt31R)Y$ZQ6;g^Z50G(I~Nmw&{~p7B!7D@INV)z?`}2|G5faz$1x zM}=atCgWB_c-S3VVKeX{dha!oZ&vt=oR~=eU7_NR>?+5$oQzh}6${76P`c*-Q!8Zp z(FQW&kZc9lxidwjBqGzh_lSk(*RNcMhc?J)R1(`yOH@34(XnFOxVoX8$-1QPkB>Tj zHHxE!`9)kNy?ZFsP-Hc3+k@x$JW}(PWJEC*E?9e@wsL$^$ELPd=zQ%%%I7>+!S?Fh zD5e5}<-}idD=18VR&vaZaxANjQ)1UsfAh44v%5DMtEQ{GR;8ks7fFwzOG<_l6=V!% zI9t*(heO%xof5xo{Y=<+@PnVZGfP;1!2fBtEkw(L0Aq2CtAiLkfu>gyCz`tK`xXM{t(w!rq;4U%!l0EYH zhLZAfZ@^5E&E?9_eoT_rjeCs)0Nd^+oS$B!lU<9>C^%THZQ-8TxMWZGf=Thf95u0wx8J?84f7IZMs zKD|=YjpjZPK84z1N*a-)1iFW`tK72|i-nyo`~R3!MH>#-4mJ&Y^i?W)(9hlY8!e4! zIhrB6d9pt8?9TJtzT3vHSzFhmpY0a0i*|jTEA@5_^PpD)HNJekT~E;J|M~Ohp14)C zw}qrO-&?xn@YUe$%*>WY43o~)f_n>%04B2qKoJ^myWdDMoDLFq^agpM;{bRFVDZ1a zw^#=(sQ6=fh$JA`M8mx+L-fP`HRxoOty7_m$j|V{QP8DdzNWao8+{Z26A0VTI%=`R zDs&77?{Cdchk%*$F?cwGmSP?A&u@+Z8btb05T-Zv2ZD)^c8g+Yr~YW$8_5}QF?HV<5Xv|;B5Q!7tz@qNB6v) z{725s6I*_~>S5`$zNnUtsk*XI1(+OjoVG7umabqe>@OyJ8cSm26)~R(I z;s+FD(l0AB23hnqrnshUZ%O29h^r0vW~kUcc~)g?tR}GIl+|d<`DK&7YLdb_lCs&6wn5dMmvx;hv^pT$a*2An( z+fkvGIva4dQx|!n+`e~Qqhm=Tem4{z^Bn6`Dc9n1Zch*&%z1I&afR`XA_y>9EM5&i zu0~;nT!3=d(6t+M{{Lha2|iGC7$+5XxqGw$lS_V5-xeo4r{ z(7<@)=D7!b)lybmD>_B$8{L^kEe7-K@=W1voM_0yFgMO}VxTpAY~JbY(Y!8J^tpp# z5<^vcKRi{cPg^;HsFInie!%G!ril#o;XfeM?2}(Ee#s%P&t-RSt)47Dt3g0QsP%LEZ}QSG0!|aJEml5buB4Pr|RPu6G2DT zy;9WcpKDj5DA}2EXloMTqrWSu76G&8 zQ+~#@8Mov5w{i|qRiA7Zi#$%H78#5UL*%`=*%x0d4cs0Te{Fg|nbyWFXuZ4t!*M($ za;L~Mj1oKWEVqwo%-Q@^PK(&8L)zg7ffs)|ukpg{+KN@l$|$%q3`^f|W{rsGyMMN= z#`EnpH&ILV8mB33rA>VDrvvOvHfn}`=%L+|47F?iFYkNY2#XaQY>!;l-(d23Ds`Bv zEB;=0JS$~z_NP`}(duZWvkziaYdm!POxsI3rku)6UA10KrC*8k9xdBCU%}1Ow3&xu zVR60IrG7v_FfMV-tcZ5AGKAe9x2GS~WCKcwddPWh#{cZaq)YE>s$%WER;(6bUR9F$ zPDaE^T-%Csl1ciB^_0V3N8A~`A~lkZMe!O@jH8>dgQM0|E5|AtsQl!$Z0llW;?RI%bh>?g@ z`pP`s2o}??GklD>HtAozHr9oiSgrltIC0(B)^pPyfwMZSI&w8n0w3GilpNz}Vu+uu z5pU??QXBn2H5cHsdw|6tR=@Y4$@$oXedD&ufmaISdiI5f_q~0wm!&tN$K%=1ebs@c z-4wg-WMdQc_=tV$ZyeF-Z?_tK`QCBXHSImKyxqw=CThbC-vYmwIJfJVHrWhkymt~% zYF@`IuZWhQN?e=*`cg6-5%uC#mA}p<$W_`U$eevJZ5Sc=qnV5C)G2{P1r76coA=m2 zokUxyHgGdf8$-Ox0a5Q~fe8_HCRo?iw{EUJ>bY*6ZNlDk;o)Is+Bj+9_?>>Wu59^W z(=~RvoiiDyc}(j+hMI|dbIvz!xNU2B{bxCcg{se$Zli`(tkCy) zVp?Lpt#+cdWa>-!eY;&5StJc;GnZ9<`jzz%y0e*?877Ts6?WJAG`7}oDb9b#KD^1Q z^U>Eo|4hAJc2Z@CBQj7{<5Jn{*(r+(X)S5W$MVPXIpG5Ft;sTC?ay~5>i_7Q)s?Ze zw==FwV78lc?-g+v_t?WtaA(;6{W44qqmPt}F>9R`xbSXd)p%@h*%q}!=B*}T*PW&M6C^0Z zd>V2Wy30Sju3=;B)M$%{$a&7cC&m9@@~f0d$+(kNy)UwS=!bK1(AFKNE^LdpzV(JbVR<6ku5jJhipbYbLnGO6oUVohApkn@q((9d& z8i_S$-ql{&&*okj!u?i0dx+0u>_?ylaoZ)B@?p`QpUso{-{l9o{N2bv z*gqxy#lpeGB|~LMHS)9GA$)OwKSe6a(|O!yrp&PYquL|O;R`*F4-bggoeS7{L>FlW z2fngz!dY|e(pZxsymNSaq;G1U`YlO{h4ppiAYU`@z{fbcZFW2hM#iPM6M^6JxZP#D4kv8t!08o&3vul~D>W zs9~|$%hx{_*^Q#ybKeMrmrtWIXJk{yvm>xNktAqVv%FA+Z~Weeuw^)Re1I-Yn5b!yBhGvu7ti_bSy3 z?o8ZhVP&1xt5)4%a>LI-Shg!yhUtdvYbQqUN8VA!5{`+oEA*|tKa{|1r64hA`pw5S zdM{i0v>mO>%OF_owse^1Jg0_#cC0^3NkwDf)OHCqHX~tCk;WNYG>bmnidPQRH|Wnq z{)EEfWI}FA*MRYqQTqN&bm2#Q$`~grx0whUdRJ=aJi4H@OqeaPQY(vvGCZyE$U67e zyV(AO?;TzPpAZyTDC-(rFJ7tS&GfVyFe_Z9Ra3i@DRw+rYOMb2WNC4^a=*xgannnq zT9%BHX?Y9hIr5M5v|heRzU8S>W0Pl~C>JHBPfdGHT+MvL+R-t1Ac>pS@bPoWAaCI8 z{`#!{SMQcB(z5w+r3R%YXA)08=w*yfX5zb4Imge!S zn20s1*(6v*f)NHdW6&#V*jSQ%?adqfD7pNK+JBOP8!-?}Te6`5ni1YaiM2U z-d8V5CKO!Q&-ZUzPR)LMvAwD!vDq<~7Ti~D)6-potwi+vwAa{M-v>?hU!Sz8AJGY# z%TaFiKN4kFXIhFpJ@NL08nWrYU!7uP0%h<=3JgRJ3K$G6b*+u|>llb>I25faC9cgn zS|4KT*7Rp6_5@8%zk;=&@k>!h<#X>W6OUAU8Y?)jD^2|}%yY{DN0Hfvi zdx6t4N@k+h<=ctt~*#;&Do30w?;#)5yt$bsBd*ALqG*(nN_0?HIhFa>w9p3x))cN*jGf3AB zww#~3IH_3i%PMoMrKrieHh+43<%ni_R>b=(4#wV95A&$H&$ENe`aD2?Z8T7S~Dx zx#q9+hFCL;aWk>8)g>sOz#eWytV&>+meMiB5GM(X(_PZOietl!@k=h_54D zSJt=8re9fv&2*Ejsdd|6(;}D-j*{Dvh|*pzA^(l-+%L-BUWII0)k&7l4FA%k{w!hS z6ye;L9P*r2NTVwVLUFDIay$yM%S11M>hibuQ8q)%KCm6ec^6Z}5LCte4;d>eE4`M3 z6msbVEu+7>qGQ=&Rq+3>hX0>>;)ffvF^F7d&VgzM@d{Ew^jh#vy56f#N}TNb-$+6S zQ0*yrON5?q(4h2FHUr&0OXxEJ9Dg4m&Z}yB30Bi40GN0p@&(|hO9^7d0`T+7ma2aj z$oZK(-Gb}nC|G=npJNQX{wk6nvV;aK?%1QXl7X0U>MxX&Lfx?uCQ31=5@i=bAYl!A%O2(lT_#Gow5 zLjLHLa2{On2-OWlmpX8P%{(1p$3t`&wr}5F3my)fWZrlGg6tL})T2RYp;Qy^xd)pM z90P$D*lhv_+Qh3F1!`v$1j*ng{S>5$E9cCe-rs{#0(AKV2=9$c?T0F#N?^QpuKHQH zm#v)!>9*vGo~~R&7l=V>4h}QSj89|(L{wyaxl5GVq1M;6b$Q!;Qv?#5b>vy9sEHFfO)mSPDtBf4x$Q6S4NE)dLGoUxHxV?zjd4%P1+#}VgV7fD=3fz{em6F zI*=pq21q*r22U|ylTZ_W@H#+8P5@G8gv0A23|pZ75fc?yuROBv6F zurlG(z^m#MaO!=H$&p0Q-+u^`XY~Cxb9NCCb%-$z4-F~cwvB;q$q61!!`2bv=+1F>Kal@=F2Vp+>9q6|vV2mFl)FOnIDS-JJdjhCRYm{qpIQ4ajFMA+t*!bJ%p zJs+Xz%R(?)b ziLx?t&U6%JTpSnrGBr=YMhlC*=?(o{kcNor7xUq(1rhD3l7mwku8FOCTA=Q6ut}kPL>pfb>JL@h9tcipN39ju0-BJ;y+VHvu7wurnc!4Y8|z5E6c{Lrfj? zwd^le6%R2;FM;9I*fcTNo`@)D`ZEI@XrdFg{>V=kqETC1d>YIzN|5thCMW=x#LPjV zAa*ZMs*Z)JpC(w4Do|L1VnfWgZ{IW%%fK2T!@vJ9@x_EDJ$h-OY=Jk9VO05qhK9oD z&}`&;aZWvqR?iwk%@cFmhp?Q%Wh4A|&^Ir<&?+N5NaCbW8|_7WDvpMCuFsj+gD{Oc%qZKj%mcXEg20;sxzj_Y@dUq1cnDG_l8 zWlCVp+x;H6*ez5Kp>~Iw{XpNtNcl^5w_$CpT2&L#NhQ97YbbYE-j{l0ds0FsZ3 zXo0exJv(~=x{Wkl0D&GG4~*<>R1dJr&Nt(38J#{Tc=?mASvGzTQtTRUp$2ddLc};A zQ6)Hika(o}o7Tgj^sRo@kI?ai(SGv0QQdNfAYxEJb=_BZe~ z^jG6cdw1S~fyj9W3eLMr4#P7x zaC*zhHfN>GIvb&md1ru`Tx`FSB|YM$e&H&@nnYLmWg6j%LNSRrb;`DzraNgjZ@B?- z7ZC|1%JM8ksA9>ICH1X;LEIQ}anGzyy|_MRYYqlx9Y~UdZ%3@w2=`apY0`iw(j$D= zBi>#)Hxlpp7=%|CZc5{Cst}Ef8vvh=tgY2GHL2iZn%aX9OvEkwjSx_CN2_IEW-2j1 zGqTpyCk$&XNO?xn8t6L9R#i}v^b1G9;8hP=@(q%UK)3W`@fU%|1W~I6FEI)Nfr3@d zu`vB01S+wg1@yMhjKJw5R6{6u#)49)(F_eVBDxtK8EIPR#%P;B>wRmvH?*6GL$eyg zor9e{O0r?E5by-$L#h^(dec$D6~%3?A4mI~WH0Tr!_)p*=N;i(_^olCLVQe5e1<>8 zBNk>cJLSauxvbP&3{s*RKnzWwFrnVu3};y$Lgx^Uxt>Dru7wx++B96v2tbfg>;sxOVN@)sSUw-~pv_)XT!cSo|eVQOC}^ zLh>vn4b)vD`$=MLkUUmaF%P1AFU*v4jPr8{$1E-V4?j$9Dkzs1b91v1b{i|159l8c z>}kw*&R;NmX!O*|R>o z(2S*+(|~;}rcK&~hP>vP(Xv7FK0|c!TtbQ}Nfuf~ zCD7FOgWN~;c2i!D0|~i5U)R;CU_=W#M?BKqcE#KVODn6~FGGMW$Yf0|@5|3lyLCX~p9V-0KT^+@5oiEzgEFQuf+3j! z6-h{di=pZVA+s6~1NoSkm?(-wwkhcxAyY%>FBUnf;BI-O7DT8QD>svgU{mtEE!p~T8AbkG+yxY^Fz#n9j9;F+^!S0 zhXNOLU+)pmq`zLC;}*^fI1D|}_6u682Z`Sp7AOA;WZ0>y`r^r7uep{m5Cf`yJSgjz zEx$MMR6_Y&I}GBa$a|m%Ph2%en@kgXLAN7GA7Mt5L>BYCbmT_**Cqj&{RSY_(YO3Q zrSW8>i7!7N#I376dPci|yyzk32r%JMfkQ^x1w1+8zPfSa#!<+Ht2aA3I1r^t!Xn4Q zTs6Mw6DEBu@@x{ZKqoe}bbnc>Pvn{j{~qQ9-REjj({mB~_ZM}z9yp1Gb!>s*$6)aG z{TF*L*+ItB-*Rn!`z`D=pEQ5%uEe#4e=TDeD_qJM9OCeaaaq!?C0y*oe=#sV-jln0 z>F(W2Si&A1TUFxZd|cI+LGiSpo^yk+@F~MU$L_X9vBsgw?`NRMPt<6Tce`5=6?3FV z2uOz7n|AEDGFSTD0GkXd3sE6%H)Bx4)MVVU#rSa#WWpvlJ#MIeA+;hiBZK%XY${t1 zBT;BWs<$Yl??)>67z=x$)e^pDB9b*Q;BLkA84#qhqM}m>z5;x)>ZvgxgYWBYPySgd zg;8X1XzR$4bj&SNtnr0A)LJG0u8F!q$g~A(2Ow`i;vlq{xy{W79f~yb#?L$HFv^`d z1Rrzqm4uLMgx$nHmxFApXF)P*EUOOh`GjbjQC| zUYmTsey7u~7evVd*e2p*j}-U=BwSJ<1xx7SWPr=?<%TU<(svI)-@_C#y;G1gv4VUA z+4K<&1xV8%%Bg;H;*CL$#EHZo5x5r9j;s_Ue+9$1FfbNz*bb(~sn`t?CDkTmuCPB3 z0inM5(xGRGIyc6gGSy+mMEquOyJd*z1u#xl*j|cZWg)7)Bv-}AW)NObQH59)lzL1M z#~`{{4eKg{JmE5e0Grj6A-Swl#F>xa)(b#FgzVmMsvlgI&`Tdm$QZ;hmS5u2t^E!u z`yp&1hecZIHv0sy)IbR|j+tH+&L{_!IGp3@3IrG@esv%iAEF#2S@SZ_5+nhLHU5B& z$s&w)cOlLlq#Pu=H|<5xjnUah*#}))B|}5Qz_I-Kx$J1jtL~$0lE4ZYL1a*2hT&W> zQ}}c*FK;DOK-j!klpgL7D=%Z>KBr4W4J7JuJBW~zd(qV7qCo=e>^|^FV918nh0G82@C`L$y-#e9eKu{P}uFfONqa1zz*d1~+PMM0J5U zA3)3-zEg=vcens&dpDxB9tY{}Az(Wq7y{#p!NEt$$R*f_=4}EAA9~4==eD>bpO8d)*I*`&u?F3t%nt`92mE5e z#f~v=&ud`yf>8suRAG3CT7Hw#UlDxqNc_m3NW!l96QpnA3)F3Tbdyz!kK0Hf4$Zj2 z0D+zE>MbJjiX9u-h!C+L=#D_)`UCWYh!Gb`5SvV)&KNbbK=%*EmW6mjH64&&o;Ncr zlRI3>MJy;b@o978+)~Y&Rk2{E98OM*>Hxa0>m)>N=;)9VRW)spKZ%+MQ9dF3BY3Jn zM&eOGfYyCD1-O(F=>-T`Z$R{Dup-BE_@zX{G|}nqJS!Md?;joN?R{ZltIFO&h@{mv16k1js z!2BX!+*p0fb}vGI+@NhxC}`1n8d|KHmMySZDBVqCvC_H09=VVpSP`O_?06(0=GO{f zao3YYcxl72E%!EVfw&cXR!6jq4s8{H#18}~d@MO}Rd_K#0D|=((r_SOd$M^&MK!Pj z@bw(0y_>PARMPt%9wLN&0tZv+wKO$}02@~r9ur=Y&;(J?$0kw*YklECRSvrpylCHsJS4-` zt(+7{dnzJ9i~n0)U>QgHpk*Gc!n#s9ypzjp7_MMjj9FsTb%>oSz17+uIzyKRHq*&> z!@xHh@%fzSa+i&y0CQ)}7Ja-yr+ltUb-5h;L7eh_SXf=-Oq31TE`0b?|7WD8b{8L} zh6b-WUi4fIqnm?%$*0>8*Sl0#y0f#ROK*Uk&W&R?$N5`#TgWh!++8{*w|+2BM0vNA zhELBv1-ZQ0Yx#H9np_#!%)WQ|ym7rJ9bH$9#`ODV6QjSG7SYL{Ia{u+KY#(Od_$A} zkA`I@b}d}7WFZl;TX=mU%3lkwf|C$_qWx3w9~v1iH$cs2;pOt+A3`3Ne_NP$__OZb zza=#2wh=q?h1b9Flk95t>z~)pJO1?FxRFm?6i-Rgb;Ax@b)A`?{=8vVJ@8;kp{25F zv?%R#?jiGl%0QXCE0%M4W4CuWj;{!q%)VjidjeC!#ZABA=AU5SPq@is=;C(KeO`WQ z%e|$u<2@@Yp5&g4tWwW+i9LOuc4C}d3)kws#jAwHtK@pcHQ^JRQTBAH<_B5KZSeo% z?mdH|THAI(Ot@8)ts;sN6p11rS+a_XNR*r#L1GI?me?vPN>rlcoI^{d$%uf&mYj3W zp=okB_j2#|n=@1O&D5N!IzOhX-o1Cd^jf{v6Yl3u*L4lmsj~18{EcK5GX)O|@1wiu zp1Btv6r%@ZuVN&%%2t=KJNh+pkLnoGbmr$Y?>MJi?I0F@yU8^h7v*T0?>1DY)1^ zBYsI|Zf@weZ%*N1A!?|cO%q$5{|vJzm{^>@$7`#{Lq+yc1V61w(DAzsk)R8W;Q93K z%UK<7<69ewZKckdoittRDWB^r&vtBDoym3D9}nuIqs{c1rAkAkhb__eDwUPw@%lNA zW=?|gVpjrJ8L@ti*jGyUWOm|&iAnUis{^@_!7hbT;ubxrJXTU>nia)SQM}50?t9q& zaYobK^1t?%H%0q$*tDraHO$)ceESMhlE3Z?Asoo1G?FLdm(Jo;u$WTC67r}oWo2c3 z0@(EAWjdzI#*ze=UBAcZR?o9SuNNRAOjN?QGzV?|bqe0&r2?tygObgf8ReJ$UFJf! zEIBU|7c^0g<^7pH;ePXtSO?R^+E*};z@@!4Q#IK)kgyRAPN*f3V}b+nWtKvG`vp$N4gru{(Cw24bsrP;d(x7Z%=YO9ksf`!v+g&VRI6f^OtLb|pxd^xr=T7<0xb3mfeS7+JpEVH@4DdAWoh}PTWU)!*4 zk1nYuwh5!=J3QI*2Qmzml&zqKnx zcbdq7a&r!gqE%T8gND;oayzy8B};V?zqlr~K4cs0O?;DaB33U?j_7qT+t4aBo8d4& z!|%pN*-^tRXXdoCS{&%HjH7YiSp^zx#?`y5P8h-&Xqs|ZqufC0&zMjeu@k0 zwpB(=Cl2Gf4?hT%hd+t zLPd)aD5i+Dsy{{amn6=Lp~Q3MX5T;&-P$&M61Tb2|8GULa_H@xMUIkd^pW5Y8 zibhk6j?RMf53GwBX7M!4`P26@n1=0gaeoCSklO8Ey4sf2LkMCz0|h*k$4!0PiH8|Z zc4ig`A-5dyo@_L)a`UyWj@VCEXgoh5i!&U4kNfJP-EJmLpdG80mi>qDkpA9Y)3Hxc zQy77?aOZ*cY!z3pHmro#z@B-Vt! z4N_K6krx!|3qM?3c)`nA;SVg?RVGG81q`#lvTZ;xzltz>``Aq%nlsTQ4lbnB(@rq8W#&}vy6qXlGiF@0gLNv@$ zW(p;mQFaET4qqAe_vS666n8NVg=~D8bho91sGM6$Z~yLAI#-MFbEc-T_H)jxWemRrp*F>6d=lKAWuseS9t3Eq*yx(^*H_<>^HZX355CG#*oa5FQ*A zdolSSn1L@VeDg}GZu2e2U|0!(CtAod5(czs2s&=t#cxgsMt@;~W@p}pDih1q@#*m- ztDF5(tP5G|ef`%|%4%*mJGn;YRPD5bJjdYabehJ%bkjC_<`=pltwh?k%grND4eTadwNrLa2A1E=_%0YN1? zmweR_4#RY%dD1Xqk4e6%rZ8TeS<>U^G#3o?Qz^DrB_BlLz9UX!P4})T6{;?&DPHX$ zbWbn)WvhooM#fJBlHrOcFqJM#E@?^lJ*gvZOG3r_td2#(q1|PBwVYFjSp+;k|Gt%W zH7OsQqujS*|7%HxoGCApluJ8g&t&h))UJ>a6l6MaNVwD+c~UNAS5}t2c0rJWRVq!l z6T48YEAc?lbBgoenV9a9Cb88Phud>5U2V;=tBK+)db(uOm(BU0(7r(Av8UiQRPlFk z1s?qjeaoNZMuR!Ag)?p5SrI|ShKgd!T;@cR&4uIFTo>wRj!#Z}<8^suv#PqCW zvo7p#u-wiL+&1*SW!-IB%&eks`JR(;_VmAl8zzu(11ybszkwt5-{Tccp}7XmQyXps-O=5El4WR@hqH=Q23vepB zw5e=0L1st3YzGGsIR$r2?ok&}KBHG^G`^o&rI+2}4VdvPYaJjNU2`>auvE?6xj?1) z`XSmf!@Dz!mPO5&Idj__bclshQhM&XR#=ogNG@k|P-E7S!7ERs#?e_S6F0kN?hDMD zOqhxuVx9|8oR6{{{dUuwbU?4$i+ocrn zQS+uftp$f%*tW^1sBpR@4`VlaDex#d12%`4E;CDt?oO#)-55dl5%<#{MG;y*yW4V! zDxt8wX_nm<0StC8JMXnKpY7x!1P9j>U`-eKS3%CTAoQjvr|2raFlEr03#&iOEY zi91h4O2A|OwjiTK$wmzeBcBp`n6iqG9wg^#qHx#A9heE-R`|MP)9&&sD(PdXNj^uL zCr80*GBPBlxuwepi9Wv-9JWhPf)!DwslkWpj4r4mUFY!eHBsmgT?9jw4uRk6I9dF z^S8uNYF>tX@5;{Hrjq7HndX4&zcx9xkfQ2?z9dxJ-LE51XsRyv`Dtx`wMwMswCK>v z#;c%0$$DRbv>oU4G|Xt-6Bud3x9I!QDnDgViwWU?Ml$k(ekgCMGGvV6aBt^Xt) zhgR~o7bZ_L=^e#1F)alTRzz7Y@Gawde>5|%!N%1{G3WhrMUDD$6KPL&3+N3Y-+0%a z_HJyU_@T!1SmPnVusimN;R_>E@HB~wB}$F1yCIW>aW^?leoSBk0}R>_C*q3u^dUbI_T`P1>>Bd+%|d+stG-niFJhmbBv$w| zyfE@h>yYkXNk^xa&Buo3&g_S~Yn7w9VFb*4{z0Jugb!(Ig*#5@76sg0;cs;eOgbGr zoS9u%Enl`JmoSU+)ZnF4iz#o*u4y%?sP5V~@7xUA-Vs_Hx*Q+Y>B?*L8ZAob$poi2 z%enT)5OAeXFx(U!`Tp}iy#NG%G2H_%Rr8+IT&vG%lq2!snw{Mj9mv0idj;sapm7BY zVz;E6mJWkc&D({-#8@{8jDp0tgXQ+l9rU3U(4nZzGv)J^q9pCm!#_n1^Kj9r&mynq zTRxLVvLFRZf=uF;4rgaObuUV;v*g_luh^2@FWv_txJAZvVtZ|*(`b}m7-6_h`>x8zoPA%T?*sp6Fuh_ea;!5RDH=G1Rbh)9@MJ8&)C;0ZRbnFfD z&ZI}*>FM$F@r+ve%fC(cvv+RrWzfa!d*E5PZ{0sTznczDoK)M#;hP=CKRwpgY`c2S zn@=pyC*_C#AXau*+KO=NV$1J5@z2|=G(|#M_ga^XvXgS@=8Ypqr*Eh2O$8=YY(yx^ z`Lg1-;_L9Uo8t1{x{(!7w59C2*?y!mbuzuRS`$5wdZxKPY0s$hq`TXTnZj8-N;o<$ zs<46?auz<`q!_sYqnLk!#UM@BN1a zqjRnCn+|O4uDiR@%8v1AFVR=X@`OEf}x_6%M zqpr54M_%nd+XdMf%~gUbauDKmG3xp6j@d3uP(jeCwirXwnG(FaZOzXi5{V? zr^p3Gy#f*x=eARcG;?Cwk)hM4*S|sp8hpt0MEPZeJ&i~#b%4}{zc00)%8w7qnZr4# zDixQ`!(jeC2ZpD^nf+OTdA4P%O_BZyi@Wr^+uMEQ@S+fBuYJHvHuv{Hf33SjbX8P<{vrjO0;w+JP1wSy zawYd2-qyIngZ+7QS)W_jTGQrfzq`q)%d>AE0V1D$NaR$#v zo^DVQ=AUpmdM^1$Jmp+!sJPjW6#F%z6??9$^VISS_Cp5co0g5D7e{cZWu4+LVuM{v zB>m*(lH?ZAI^EtU%Aj|N<%v-jV`g8GQlYJhzLk+WG3@|SrVfY;aM5E^LC#mo`CN2| zj>Ed*crh-TlW_Skr6+fFM}O$1Su4vX&&#zUH)L$rr{YwQg;;tZ$a8yV-BGq=n*YPO zvPS^9faJ;M!pR-`Mf(qAr5dtlwob1U^%pmE);#OqpjMjstp!njq`r6^+_o6~LS7bU zqqoHMsXl?rucbTM+1l^EwG30)&giD~t2=W6g5)Vv;*-5U&x`V|zwORjs1xUPp6hYh ztO|k>;FQZsBxF%`7V~c4*w~t_z)g$RBgVaLJQq_`W%a5R$M86$aJydeQXHB zqbQ-|@P7U_bi0H~veVEwE`lwpT8%)*7lL&-H=ot&*TT7RH}&;$0D2M*zDslhf8(Co z+b^NM3t0I6&T{3Nnj{}=f5iHXGhNrb;watj)`V7`>)_2P`X~O}Je>9YjT)Ez+X>#$ zMB9oBLG~Y5nymQT$u_w-VABpR3_W+Sx+G66u6|9f4Mozs3WOT~_nLAN8d zhoOc!VHQ%IsS@E=r~GAs{%_H4`MocYU~~zjZ%ExLaqR~q2x%JzFzFx2%o(M2Cc6y` z!_Gs~OOcEHps(wBA?nrT?D}svbn@(K*wUV{9~uNCOzZ?q!kga8% zS1fSbHi&k_d4zjyliiWZ;kjZf|0@#`Cp!jeMKA>rcjawC!v~H7-&)^#pWJ(KbD8T? zRSduEJcJE8#^e6Bcj+HX%Kc1JOP-pdSMZ-xChu66lISdb-uy19;1vt~V-I?LTcySw zef97@{$Fi3&sV)Cy7yphJdp=@CqFr{~ zrA=4sOc%uC8gHshFDkw}vu*0SKCI@Z+8;&eOn4sSWS@zB>v}aEg?~JSZl6rHa=#bp zC*NM`AYj_K5#Z_l^mpsHRq>aIteCVYN`qs210AulKJ#H^c{Ze^MUB1nYwV-rk-KUm zfgRrZ_{-YO(sI7R1RahBMMWM8ngFQIB?NYm`4msvcv*!s=6?D`@Ncjf<~RMed9MIm z3IBwP&sK%37qmlP94CoMiHM|Luly$FG@l!NiF;DEGxOOa8qF>x$lF4Me-o0_n3or* z0(Pl_gQHSSZNzicL;QnxakQ>erH&ez*AGPmCQ{ULJsD;@X74TvUr79&YN?SCEo>e^ zYHO)9H$5}G7&ZN^s$F#R+YNS6iD${<43m@^YS(y*q`wwkyc8}hze`D6%Rp<2Z?a*T z4e*x))h=PM7s~iQtgc$C5lBza)jf=Eacwr6wyAwy7N+7Q3Vj6sS>(o~Q{%mLmYvx& z?R*Aimbl`(bNe+~ysw*>&AKM-;H1P=)|G8+JXW#flFrpdC1+$%6PJeWc@5Jha?qfs zspcna_nmvjny8{*ap7~=Ea$BFl!l}aw3VI<&~Sw&5KiG7Xl?(`Ijb~{@d5YFoW8I&vNW>Va|Rf5wkBQ^k~r2XKajwuAoH9e!p z8DBQ(SLl&WN;E-(l5+j9z%pGH$1p>6YwLZnD2uBd@tHO93YUiU#X}M%@!5S8Ud}!SRYqeqRNr_0xSC*LeMihnC+J4p(qQ-E*-}t zqi6q@mH5KQh;30n$*g;&wP|*r*TzZwPky{&)oVz%NW;=KPXqD{;`s5FNcqn7P;u5u zypCj{I3xKcW|?+#?OH|Pw9MDVp9>P$-{CTPvdH1Z5`;uCD9^xcrePb6P>%9MX~GfE@2?I(TsF^ zIyCte>EyRHH=n!QQ`%w%XB=>){mv5!HEP<7rjt7~;7`tO4AEo>%CCq|PPQCl?$}RC zqMYBL_6%HKc4PDOFXvCR+7?_8`nD-ZT|AcF@Q4OlfS{n0vUitf<<>k#m?_!`Z#pYb z(%hIv>SO{5Z3d}RpB)dw*rk}`FCy;!N>wwvXUxpEhl@jWT1la2luABg>Ep+Ph9 z<1F~_LiS@WH~enV5$CtqD)F-DE)T7VfP`K`cVqI9OWuE}v9w`=mp;6ZBdmUu&Dy6vpG*4u-nu4CM z3MXer!HCJw2&!(Oeqf^rPBT4ON!&cSe5x)1i{^^|n$}N3x`Z-K=_v%x30&(zD?!sfb@LY#-g0(cL{8 zz*&I)`GwM~HBU42#ci){Dw0>@W`lXYE@gwngSPxW7y;H)5VTqd)ZJeZmTF^%T)8Yq zv}objw0QoZNglQLMws>Vy~KpX`B_KQoWeuzIJvsnc1wZTYQy;soUBB?g<^kcWCV3U zQfc0kikH1un1Y>=^n4^Q(TUY!%Qsiu6Fy#OqO7D5l0zKcMj?et*Ic9Yq$bVV4&7sP z?puz|EbAKB3~WjXL=vgaP8_1Buc6ONuekAs^Sxn5A}JZ|~^i zH^H?GWnkZNiVwyP(X$o~g!9;CY;zQCFS@78hTc`^6%^bV^BAVsXs4T+ym3ruK2=J> zD=0MuE9>UzpMTv?jvA>txgP9ecSb{KaNQg8y{eFQb;8VQ;!5sbMr&IQF@wLc#`8^v z5y$s1>%YmKs%5ot7@01=Reu!uG_WE%JWL=p?H*=RWLzJw{l%jsU^_i_db~Xkjq*-Y zSygU-{C3JcnXqaKlukwLZ^)``E*P2ttnf7O}=#18{{+oO5Diz!Sm>hlTzvY zYomoH>Gr01+Xq`Pis?x{X)KkT%O46F=?yeWTMTPP#AN~Mq4ea*li2YeKYs8+FMH5f z_h}MsB`aU?r$U&v*;mcZP7gwxP>Qou8mGn2r?*dEm>i!?LhMerFgS>~tuxSl;ve2v z5zU!;nFsOYOlPT})D@PtqviH-kb6py53R1WT=v?JBS9N!E`T^ar^xD|6y$+t7Pilr zDNpZEyK6&k{`G_@u0$x+NZ}2&x1*)W*I}_DO&5L7bJ3rI;@!)xcAeuMW-Max+<~kB zi^dt78x4Rdpvge~5OYw~#oUlz>6+Et`}@~}fSv+M>s8h*z>Sjb z}rkXF7NhH3Fok*dymDg{GXA`qLd9h0=GMKUf|*8Q*4sg6FoCy|qGPZGK12DxVmE zfMfd(37bAX&OM$5Zh42xb{6g5A5sy<$0s@HuyL0R9*rUyV#c16Y6PNvcXRiwBPB7v zUjA8*F6&JmX$Nsaxh8*FD)VyLp!N#*&mB{dk{v~I3fZ+fsdtwzLa6EZj8s)mw4{r( zt;kHJAz3-i#1WElFMR^y-Ae~_h?5Nf*O%>P7vSss5a_kzHpsv4DkjA#c0n=23hJI4 z&?KSfFfi2CcmB3=o(SYCcUCrufqurb<@evL@i_F|b*0$wDp)wzy;-pU(ERwY4)U&k z{Ghp&5$YNArIGX_`i~@fRVY56x}uE425?7@)Ea-BuOac!N*7a@4!6)!|Fu?L#C z;L&3ItsQL2v&P}qeef;+@`yrVlKjtK;S2x!I)!T226`I!PF}hcp#E=OI)H0BZ!6rV z%lAmkH^^rzHJY7;95u8fNY zN1YPdu7Cb~$d>$1t6-&J;9HggXyVqk@dk*FMHx{Qx}!rR=>HibsIvakgbDQBrBg+} zfB&|A0%-yGyork=dzcNVVlz-R`=EQ~cU9GW(7tpQOac!MK9%{s+ZOx7<7s0QAfSi^v z2h;*^(4UwJ;GxH%p`rfP74+25CJF7MY{~HAUjXFt!q{Q4S_fqMprS9c>%ri#|V17D-aHp;bN^urO?Vd@9hX4AN0w5#$Xb5r?$i zySTW3+%JNdbc9D`VflnT2lJSIMbJ8dNj@s$&m3JNBLxl0j0i-lNuAvA`r0MH!vKDtBESSKzUl<|3K@iyX9DD*e}F~+Lhl5u+bvL?BZ6k3!?0^1 zKxr8-Uk>5M>%pV|ngfx1gzfMQe)>msH8gSaL*KtDoy*EwM3?i_MJ5)&x&a`Sl^_F0 zXzk?OTpBzK8}I}IFx#I28j=xcK0w1%<)99sP+k=N-3LL{1Ep;owA6#HGg2%=ut133 zE+T^r;8rCZ#oO)d+*~Dqbnrd@aolyI*H{yH5=_(9(v zChnwXd4U8P;SE5rP>5zX%tkt}>F&ThWzv8Ipz;Gm!xo@H+ruFh2&I}Hn&p_K0zjI& z0h-$hSZZWk+LL4vtqEVq5nBqj2oxIZ$+5%mXYix(=3W z(5F_D4q!Y0kwq3{uxNx;!2osnf@xA%f?v1;zzaml6Img^oiGQ_(~ZD3U$2Zjb*=2r zF_cD-RR}~0Fl7KmP5~e~>#bW2eXU?%7`ebcKvOmV%cTu!y|eA1d!8gc<@NCSN1x4m zzTXFAjq1O$!otg?xr&$HY@0%VwGlol6G0jStRKK4*-QW z{Rec6Vdd~J3h-n844{BZ0|3^B&kB?xqM~4nUi{Oa9r#w@4akWjNY2w|&L{y2i4U6m z0i2*ItQv~wa{@RZgR^|=0m6&`kPfl^btSu*-&M%d2MxgD&I70`y6zqox+^0ToGfo| zSWFNd`v@SAcK}110s>64^mTUg5;@7eS;yj{Gz}lkF5kD@ zmGe#p1Y4_o`{R(15V5$fc&etZ4h^T3)7WQb+hNyDgt~(tNL^~SrKJu_V#@VvfOa_bdk%1f;(L*%f#rIa+^KZfu zg^%);z_&ra5SETa>C2#0Jp+(rHCQbfG^r2PHa5Co2W0_X$OL4d5y|ol8n9jPiFq&* z_BQAzkUsOh8o7sPdm?J^VDB>FI)MAM1Jyw);5`tey65N5t)Ty_x2XQ-QWDiiRO+WP zvPYy{4_2DNSQh~p1i_Ud8`G%i z+YLAi(UNJlkA10G!9ta7jd>xD3J{rx+DcmY~T5xxsR#$?+kT&o3lH4|3251qI!Kp<4QvvmHCUxFFC z2QVu?L^Kt4OalPm`>ZVPBGd|FV`B)x#9)a7fP#Maef?+X8a_nkAFFJGy%=vs zJR-N9K{NdkkpV8JY6L;xz{p7R#V7xKfASso_oKalF-nq)OaoYpw7{TkK>ql^?}5QV zM28%~#v-OQfBc{C2c9Ws(!QUNkTBybyrcjK(e>!Vy}CZDmk$8y$?LfGLKGes4v}0Z z;+!G!BNUB?_J@fTz>`_-Cc$?u>v{GPenmpaUPx6Zj8`eLGL3S6Ov#_vj)D%ag^cGk zB@>~W-R`>-c1xCamR*W`d&d~0TCJwnJEwev+@|u{xVe>it}B#j=BeVRGCdqF;_g;pSd0RwUaG;KAI*?VMtgI)IGu$#_oXz10R4JOxv870d3 z%_x!B*=W~{)Iz@wo}0Dm8M{_>sbTpsxb3S4~0n&dJFI@$6>63%TsQ{>)@_%H&mY11NC1-JDxb^#o2(z z<6%ZTtMq;Y0*x08ID)DV1FdLePb)ACTMTBbH_14{HX0m?^puwh#+G+&y=T&Qq}7kYssGcb-!$ls7OS&XK6Q9Hqy?BKRniF`^zCR z*Ihy|d0WkGcQQ)sSWr~x^L#k{W7b9~c-f#r+yw_|GT4rE;OXe{H3$Hg5wUZ#J_19C zMt26yNf2J&UMoC%dS$&$sroT1p^eeCkVx)D^l?YC{BG0|ruUs|yCl%q&{E%;bj-Ok zSw#ycBsd%5T5_AVaB=25=ztEkRZB7t5BoCg6SsFrXaB*^8WW(ssA!F(rpN zJ++-#b<~roZk=XxeF+-n;*0YFw+V~kGEAW|N%TH6?fD)D?dhnu^ERG$;fViE6G3HS z2twxth%ho#EVJ=FNmbVv7hu7ONA^+r4|Jl{kEE zDWoDf`LP2qGMzjdG-$WujTv!2(ZChDUbU}uI~b25W*!voa_+CJdvW*mqZ^o)>+5G? z>O9E0y|H4QutnkjO_v@w3cKdyryUv5?-EV=&J~4E3|yZ zqGmA{$cmrf5Umg1p2dFj`t<3^PD1EgN`R;}00q%ZSLQWvR)c*BPa6YSF9MH>9ypq} zQRM?|Ru3OOjG9X>6FxGdTte$e9hst2OfC7Y<<(MO=kf8mAfM<#hdV16?CIsL=}XV? zYeBE$T@156432grdQA~Lj>;L_xsn28Nwt^GRFGDFS)R`6K2(nDbyvz^enJpKds4fgXYF0FyQB6KlC4g<8B*97TB5>PQo99T~Ha~3JIPc51nv9)1u6-)jx$I*ertI8(HHjIa zd8KnFqa4R<1TWZrbWN5Iou?*Ssk8a9dt4hQgC+I3vaNa&7|I^vIg@!@GyNR36>WU9 zMf^f5GGmSExn&j>9>sgAsNL&xck86=Bm^mIr%F zKY)AZ#X?WE?OewxGuzP0W7}uO{QIe3^xtfeJ{Ro%@$+Ym)w=d|L!E(N`}ib!oq4C6(V6FN zhm9esN_7FE(v)U_EV9J0#Mki;3~l0%-J9v6rOfMS@q>dipAJR$ zSL#mm^!9vTs;-|>OZ6x)k@X?}$c5Q8D!+DTAsatAEW>osAegL^ggAm#b`N5#5^tR{qNQvJ2;pj^e6=xm$%Y*`r|R$W^89{~ zg{o))ikN%ANb@T~FgJ1hmpD*cNI;s$=%nYTC&l^m6IKtF2a=RBccu*3)4M8b8$4BZ z?U}F4`}c>^JFbPI_Rl*1?K_OSepNMv43_M+l}(F*h0DI(D0ORg_7oHS8=s&{Y{R=> zf|gw$`t6*p4;p-HUgx^~nrMC1o37^C#UW$tA#2psBlLjd-z{^;oKyBRxomF!UD?%l z6}xlx^Jg!E{jcs*J~hH8E>`sLlCTjrN!|FdxT}hiyUo-(;9*A3ka|UTQi%y(Ibg=xXi}I!AWY!Nd8(;9E2PCIbO73j^b+;noZxx$ZaZ z4|_lKu%cB-Xdg;i9YlARG!`-F@+YCKcpxXQ~TU19@i+ppt>U_1*0R;c38ze0)ey(aCJe_imC*|ewBSz z0#YS^ojyG_9gQilr8s~7>+z+>B6`)W60i{{7ke6##j$q|iz)3-z6shrc(M}ebhvQZ zg=K}eU!Ji zYgfRzWJ`dHyT3oT!u{8A2J$6I^)JSz%A83i4!d8LPg~8q?b!fJvz#r55I^Ph zDsrLdpfe+7s*87p3^?M4=H5V%bjIOjEMYo&W#`+dcEz=g2ad2>esaU6J6gU)@Ahm8 z0zA#TpGD+iAAB(R9dm>Zv&RKY7ws)S=;{3}_FP_Z!v1Mm;vvEA_*{taf;Ur?*jbs6 zb(L8hI@youD$mqOq9%2J>we^5;jWdpW@3YX?*>LTrr59nhNEWVq8s%&5s9EU#$P-5 z3rt<1+*~F@Artto0oCp0E-#aO`R22yDs?S}N_Kk)tk2@V+w9G!U?&@+O*V%-9^V{& z_Vq>f{xG>ZUgykY#XC#glQHTCfg0m2Z;r&*f17!&Hxu-t+o_{e*ops30iPN3-9r6k z4lrRl^SrN(A|-Qgkw>g1OC_eAY7up+!F{(GVwm>)>~)iy>1y|YTz@xJ@@}k*N+4`I;BdJE>J=>0Mcd_}NWjJCVh-J; zCBcMoX_Xr@+jsGos$?3)+3>5X=A~#x3X}inwIAqq5pl3T*+5}E*)6e)v)-k&+?&7j z{LRbI+>vJ)4Cxz-Q?+F!9c;T5#G_|Pj|F*AFtV(lKH0J*&fgv%`I3#&aqlrbjLMTC zmx-38O5claKM#zV$28{Ml75kJy5LWQ`9+pp4I8mje=}L)Kq6v4?lV0)7vUi-fMxz= zBQ$Ygny&qm3*{z#&99f}#-$2*hP!S|6Ws0Y>e>vjRSMKN^_F zoKBYsy--SjaNGGn?Ecpyx*38+-7dHN;x8`qTb>nhqf9^!ON<$5QuZ+BFcL!OFab-8 zS(Y^l0JJB(ULCuwQ{2(^3{6)NfSm-KBDQ3mYlnu+U~pzZ$FT zmU)HPsOqA>G12v^o<>~BVztG5EAq^buD-BUni3mvZvTVfmZ_}|HW4NELAY=?gDGf6 z=2ZI%<~J0(`P(F6H$q+mv;KFo1P7OECi$@W;Jx`dKF6{bv&mx932BOXcfs6}MQt*I z=aFsOy=)u3&w5UNtIYALu_QAoziEA{^ku5-72&;>XhZM<*7tX8GI$KRv8b)#={3~0 z^+bYYwGr57k(-zC)rZ!1?#HTatxsRtv!5WLL3bsZlu@gO} z`|x-jN5o{c;TaMW?d+#H%PKH>x8<>?lCyzxN-_4q32hynXi@dsX;J}9w>5I!Lki<7 zqco*-9hfV%fp{MYp{|V0pIrIUAve|RPZm(`27Es`de^X-^j;bFdVh~t|J zNq^F*<0^~@194nqGoDO>C&>3ObJ8TIV@RJp_4hz1mFDClgufcP8ZNIj}GlB78)(Fip zZ23|8wwmhhvb3fwjis1{u|ajuhK{MWRP#L;9q(UyPI*ErHju_SMj|FA#!5M32t-P9 z0p2)Z6xzwM!{3DZ zLcFA3TdAW>4$x9e;1`E0qJb&xHzY!g$>olKN^t^Y@-x)4-$A^YT+hc-3|W{~Ak)zu z@w;^XtMSvtWpT(F-)pM*o3c!pTV>8znJM72O?Q^{k|xV0Jw3}ew#$iUaAH=$`cFHG zoTmAEb6(ugnmXK9;1IdxU%?SIeP5ZvP+Obz8;`>~JjJyRuLkU;YU2s@AGGgn7WOpO zrhh%EfA={&hIMpm-}(Iz?vY@~57wNPR}XD@lR0#*J5xIAHii$EZj4umF5&jNeViQx zsa%d|q0tS+MWNTK@n9Cil&UV{T;`bsbB&X2Z>YgKjV{qOngqhceY*l#R;7d?$j^V{G9cyR;sT1!WA0UzPytdecM+Uy2r>ZD$QR9F z>z2PUJG0Edc&AuK^%@8~A-5|F>GOb2GVOwwR5xziFzq61iWC-rVKW3OKQmI#Prumk zdUq6fQ3o%|jCSJq)@R0LL)^`8c}99yp}NBe)H}7Ut2G#ZCtxv7l8^TF@;U>H z z611_)VlCigKw3Hx7_TU9tygsG0w>cp@axov3vfbeWqcH4Kj6NlX1jjXxDEJ8++l&i zr3SMsFfRVb9V4y3uxYgoi~Nn;yK{zbl-C+kNbt85MMgtP&2sz2t0WKpBj2$*>hI$WR-LLNXpTYr7AKcpLl5v`iWM+(%vk^&&MY=fx=?ST+WxA9W24(G5WczB)p~*PADyEM)-nDtW-r{K(7`*2`S)X8mnx zN#v!Zx_No-_$&{%No`N_q+i+G@MI1zO4O$;-QKw3FfJ)8d0E#(;ChMF?E@ut<}1x$2LU*8(=aY390Urq+@wG55ikH95)>=jU; z0T*y1)Mv~O_TZn*ke+G+0x@Fu`;Q;jfXYyz;H3r$@|92LZ+-(VW#E#juBv+5n`>-f zU{DQ2NWY@R-C|c@-;GU92GQfMrwZx@mXrv?q8=!93;=Fb@!eyvlKy;V1C28r|)}i+mu6Nvq2Zc9LN#Ho}&BMZ~+27s7x$O%9`<4L|4FWlI`Uc({`<|U03+w7< zZEX%vUdcs?Na^Wa2S1ZX;OG%3dIA&t?3rPVqD{hasOFN99s^ETxvx(DdD`GOFgGJW z3HVS_lG#Kv@wj`{J$U`*`SHPW*KlAg%rR|e@}(1R0q#RQ&<+E)DmMvmYRM|cL#gak zIdo~(*48prWdo1qk-w>^>Y*3|2=|ANA9LF;^c)|j6?si^4}wicp9&zQVlJLSLybYC zF#O)DSFil76W+sRz$5!HI+_m71lg})|Mgrk$a6J?^1%9k7IW_{MJf<4*1-&{Y;MZG z5;1OxfC8T<e$BeusrOtzlUd>gW)pq)d=wC!D8IoBNIWR=Z{x_wSYp# zsYjj^qfq20@f+N#?&{)~+T2d59cSQ^d}?K9#|X5YHFKZA6GJ}g?~sY)F>X;vQHXuW zvlIms1(zU&{-d{d`|~wJUh}>{cxJ2V78(M&GFdJ%4vN^PYT@iV72o=8BGQ_Uj~95H zE-W5E@f+@0+0x<*6L@T7dOF19VY8R(*qlJ`W`L z2|Woc$|-+Cjh$M^_ES+&(eY`h9Lc~o-GXUkRY{}h>gw8qBIBuA5tpqmP;q3s8!V1- z1KOvH9YFaUu}z%1T>n*(wwcVOt)=!qb3*kt?4Py5+DCT$3YY$Ipra(C5efue!M8v$ zcj3yFU&sV)Y#0HB8?4T&;K!`D3j-^y2>huOSWE6uWOs&e=!{jlpPZ-<@BqstW~Hk0 z;spb^RW(CHv{3WEtXX%5p8g$}8<HQTIcX$2Z$^N$CdjDwIS0-qO z-rvc|Rq&Wr;k(qG>0>j?;&1}TZo^u>Rf0JbBiY#4NL3uxyE8I2_Rh;o+P~Jg-v5L3 zWJBVw{Kdt^ROQq*p1MhJ2&B#mfgo-~M1&$coD$a7V?Z31>PGX=n4JIG5yy8-sW+au zs+dm2Vp`YQ`nA8%QcLLLzq94Ds;s zISU2XU|(AJtazBujY$&j2V0XIvq0Z$|cG`g?5d7$chGBPuP&oJAT z4CdvSupD*#&P*(FIX>34UO(9PI=pLED%H0J+UCDxusjXGO5Ow>kw0(2eRop%fyBFl zP}uz(8A$`wWREzrhUv$m1#9qcaG12g7FG$~kOvyaK(+Wn{+ipt@D1osX4 z`iYoCM>tcXq~IP?BO@dU!$u=e?wDy$_S>F^W0DJ6FQ63%zz+Ktz1h{j>uJ_Ws{t3{ zpKPJQzhn7HL;%KG!zr3d9gN{|l$KWAA=~K4$UW$8VL#702zCFPK>qt@6#tluwSf*Y z8ACWhPqta|+!2J9vfgZilfY-o1O#-Rm)I1c3kV?`g#CQ{Cq7E6^fec7ACCeh_(XGf z2<(=%rIKYaJIP6CDEbbq93WKk6mswC0$ms^2>}-5zpq_uL{_g#Iwe%H`7i}~RqvX@_`IN+ z4^imFj=>zmGjO)VflK4EGn?Gg(~~gk%Ge2kDhbQ8xc46d{%ayCWP;DF++qPE4((yV zzN-&_viAqr%8Lt3)Yq;_1D7dujT|8X>u`Jw#eD9szwY>*GG!lP2%!dyrJ|IUcGK?@%3=9m%;U%T(qzb_I1-e2s7@d{XRjKc6D_}&M z4v~?ywdIATzGPH|8&JjjfuoiUUbre&*T>I~97x(p#-i(iP8a8}OpCbg;HE+c+F54i zvuA5ecoA!U+@}&OGNbyn z73dm5y49<0SHN~}Epv9b?#4fI6H*m5U_WPOeto$i@zqKu zrDT<)VUxWYRwzZKB0D1qQ8sBRD>EY_L`n%MvKn?qlB=iQr1;$2$$B2&4r9a2iIKi;#=Xv&bhykgp@iNIv?IO7c<PvGJ zZFoz*if^XNU0j$wrK2;^XcmKY4JdY)w#OYlI&gD!pReZMk6J-^&okyQC1lg#f`(`o zy=?>UsZX`G(T@SE;q$iE*dj4{ayTi5oIt55kXKZuQd}Gz;cyQgJ^B|^ zAE=fKz*wd^=;$ajnT4*KGeg+VTF-SQF^uMZ=$K{^5*bOC zNnT9GctJ3925%`=$6Ku)F|lUzwScE0Kp z(=)y>RjixdiU{VgTT)lPm!idT7MwOr`D=UDL)XCEL9(C)=@l;<@NR`#hM=Eor-ALYwA!cN=KI?}HhuV5kfV@!-f zK*Y&W_bX9SM@-b|L{)+>;kvM(-D4~sMYmWI?G=~Gv?W_MVYjNz+Z$?Y* zx>he*^~srl`J~0ZlhbZbHt`$+VP$c=hV5z{?c@uCStg)zrL)hZ!{lGjjxbi78agVjFS__9I=>RUUeTfYA) zc=4;@m^+I|K!A6F$M-ys7xr=h+g4I_t(zaoO`> zC3HHYYk_q8IA-v5pXBX3`qm=lS<2%RPk28&|14^&;y+zY;7h%W&W9z|Lc#h86EVcB z+b#_53S1s3Zp{tp9Aeh@O9-)MU1`xo*i7@5^P#*S3aj+?J~?$0EK6*wSgLihzgE*y&k04SYvfLC z-*497@H==$1V0&jFe-;t{QPwPcBT9}PxWx&nu(8t>iR>*><;5yPOsRd%URmuS7l%J z25X@uXm-)PCGWatyIVj?KN$%jAYVzRZcKbLOz3i2+U{@P&JRkAZ$PZ@5b{Jgp`t%) zc-UT-_vz&P{%+M1>nl`v9_$nV8{6@{j)%@-fr;K%$!~d-;dG?cu8}j#uZq%68by_@ zZV%V*HjmkO6~ct0gL$48CTD~0MCDcdx^^J71BBU{Te-?(b7PUovd>L55C1>O=AiAC zV)=ve!NrX79jzRF=`Q8!x!U}_wUsQc;s%=$Dj&JTr#zeDI^9k6{lbM)Q=KMyvq_%f zFBL7%y9)Y{2ldRcjO(07?ZljTP+4Kvxiv2`UKvhg?e6&)@Yyx*|KyP=EAySA4EdS` z^ACyxA5aRqBhH;$&Ec^a^1n+4`^%{hUa9G2S?#f&-SFuvSFVgj^KY0JM7&RYl(v&& zIbL5h(bxSSg<2*uNh9tO^n)=eoW+v@4nU&Gtk?axN!%Dm=8)0F<@sKqb56S%(1+;r zm|kpK)^J9r`$WMuyfH<7eSw8e(fd@P^vkqV2M-R+6xlOMpW^Owh1 zcU~9x^gb^ln_=Zq8IS#PKNiG>Q#ZPF&i;P=e=xpO4W~}mJUT*+gWE7u{j``8Ze=wS z3su3ETj6vxPc{h@<$0}y?For%#3AD{{tSA`^xU~3xSB9gOD1fWw@?_?2J{GFe3~i% zvBwylwq!^Xp76f9lG~ohsC7~xX(1~hc-K)csYf4c9>+;X_wt0-&$~Q;TH%uPoSdEb zd~s*2i0&KP>lyXWZRCZ8_fxOg3Jz;e($NpRyY?=}gS!5B*YAp`@Q#-G(rNCVnIY}U zl7-ZBI->c3ZjzP42fpez%{IVE1Yw)O-ezf`U7CUzjmdQw}I0c_Z@v9t_)3tzn~r zU9WvQbtbcWz$e?s`6>0}t1G-MX*a84Jl+o(-?E#(*uW@1)mxky%=XoP(9+GzW%{0T z?;gRaD2HUd+~aX+uNEzi=Xt>)EyK2ZS6nXjF7-iU`satHXH_?20czaRn@b*^ef&5LW&1&h5OXVu&d+TM1SJ;|Kt5u*4-uT8M>yL)NYx!*s_ppwnAH;4wm4)92HTjjb#ZoS`q~?x81lR^5hu$!3Es@$VB;+Wweo^d#R-0`5QM~j-Tlm9rSE)d(4r#O-2F3ww) z`G75Ib3?)o@H-}j??*ksK48L37-d>UN9*RQ`dkhtirZ4@>$#HOQrFty`ZqMta&>x?wR3$xGnKHtzzm!kii z^MrSZ!MjO5=(Q@Wjk(88w7;X9G2}mMc6h~B<*ygi(JeFON;p`(HRs(W9t^F~mK|&I ze;R(gbtrGYg8E*@lf0ev2gfqz5PA$8OWkHHZI3KNu&iI1>bb6|edLi~-cu{RtkVzU zbGEg66n&b}&oC;yedx)GQNtX5*pvL0tWZVi*HriFchgSLtP>wmkomumlTX$=7mZYy z2FZ=C0Uuvq3>0y>qPpo)QR6$InyjY|7lZiJm?oaSd;MX=uB*#wT&g$L#PdMk;_bF| z1sV<4X>&ux+eK36+at;lx|QL%V$0nX=6}ASB3!yrG*E|!Qz}C1KL0&gZ`st|O8+0m z?p}@)3$oBw#XV5#Omf!`2zDv39pQ(|f-d0j=}qDf(oNMCD)Tu1Z&2Cqpqp!%g>L|9 z9|KMcRFqY%#Od7p_(y>Ua&B8t1|>6MD&>Ese*ru znO}x)C9eQF0DAVE2<0%G&<52?c!&9*MCx*|ZGBZyAE6p|f|Q(>O&JD?ytwLs)A?Zl zo+lF93=GuqW0f=-ln}@Sa|euC1Dw_>Iyx?1-iq3QKfj@%pcxh6IaqzO;LqJ6iJLYn}1HHn_*8gY% zPuOm>V?Pws=54tUjQ!e##-NWd%mbt6EGw1%+#Q3H1+I1-1~{anJ`2NSnSsCu1O$Zo z{j3FQx2#~{HxJ5W;;;N@qV!ZnMS?MhO`${4fGRS3toPlULH`2aGPsgtsuior!?c_K zeHo>cB_D3n^ReyzpaUoj5)@e!4#W|PXNeD8%Tf%^il7)MRWJg)!lX)~Y4Qq`0@7D< zcauOBMAd}IA%l2^o(dRXP>Xtu9GkVZbsMDOJN=$&K3WYV0FYc5xKCk>egeu^E0~Nu zMqVo4j)@3HVJsXC=tB`yPZ?E&34xW;kk>*XV24^QZR(e5NzoKsmphPPTdFdbp{`nhH0L1v)@HfaZ-le81(Bj z>P{BsCNxm4@fg<#=*f7`Ba&~E*C7^s(qE`U1cWY2_B*h3vSMn=s`ZF@FMIz>R21Jo|H(UMfvTMFgpLgS7!V^`W@<7S`1c zMThvj`+(gyi=N-WB7PAdj415d;>~Z3jg3M${q8fpn=tiD1(>8H7Gm|IfA3Tp*Cz;i z=zH^PdN6#b4lDJSU+^k3Nw-h2?*<}8t_0Oo8DIy2xIWwPbRMOQo&EjjZ!LnMeg>dZ zoG1T0OQ7_RnZuU#R)hzE#}mQ!0cKf(J+fGyy4_qj{1n)I0I2Uie4s`#7%AZFs<*YZ zC5!CCE*k;jn+(;ugge_%;P=lBpMUYnp5i12B^#>NfaL85AB5?Y;vi{`gwULFcKG$} z1bH6h5dfYnCIj&d1vhK{aE8&r3D7EVcOx20aO*_&02u8^rX{4SO$4DA+*?>Hq`(yp zO?a8b;9t0n!3PnZ%(_@V$t&MCgK#rLsGB=h%tDwd8K?16QfHoc`}+1K#kdo2F)KSe z4NrEj)vd3o`Z0iJF*DN|^LXAWbGQ`(a(t1Vu9`tdcKMYD5=y^n4)Os(j6OIUSj5Y= z9~R`SmdNOidpz3w00JukX%Q+5BlF?_i~aoG$jAcRutaU6ySqDKe=*ut$)CLvYK|8f z8EO-(C|2u3{Zuzb5*qv*sQKQO7m0&U=+3>>xRc$+!0dcj#H(tBxIzG#t-cB~r6X}C zfeG$z$wnTk|3c?_$I;pO()R`-vUaGdY|OUX3BywS`E!tZxSk){dQefF0(FBE6$jde zP*s3By-l5emIFVzJb-FaOu#k4fL;P1#AO2-oymIh23+L%!OelHpiP$ncfIxpfj(9I zv1Z%)4B%8EpbcQ56oF@j0aXLieGX;@=I^vro1HXZV$5<$fCLDbe4610ny=5#T8=4o z6rd{G?&rX<2r;X2C=bd&M0gP{h4;^4V_*;wrPkNPsV2|oeS13xkU89!61L2#JZDkt zo>0I0TuSi)I8(x?<%03O4f-Edc|a(MfJm*xB?X5%17{qeRN=K1YaXA@;0ClG>bK+X zJx=wQdAgS8Sln_1qgw?9?}3IBLA@f;q@c(XXsGp@2Yj11Y$&|1?|umIF|o5izsz9I zFfcR*CnD+Fk$1a`AcMdbHrl(z7WxG+o}qqA-VWaqXWH@fePyLDo-c{y37$;sH7Hut z{V~C?cA-KB6!{sh$QpC5ty`}G=7+LC5re@bY&s|?ROQ4EQ71=6mU^4GFziqb#<*bW zpTn&zuvM6%BwUfA{`ZnONm@_`X!~5hP7Nu+Lx^JtrjKnj26IGQB_@{c(tgf5JUo0H zh05SD6U#3aLLRkd6Tc0f1VGENU%v>aLYxLpZ~7m58nVrlBIga73NcqFYIamD zSB^sYp{9&nlDq=6??rf~3Svua1_WUP*<0ZK zV0^$&OGrq_q+J0)2~9&cR`$k?8#ZHIiU6O9uh}G^$F*nAYE;akSkU|RYw_sl=wJc; zg-7NTxg$V;C2+!#F@e|Kq8=&c3&J-4?WYp;DHDgi<8T;~7+aHwmXqEIx z$KW8@yo(WO zjUhZwif%SY_E3~59)|qof8NDO-X(%225+mY>A{ZzpMMXX)&G3kdh(8gU^`5)E<`Q^ zX<;W$k)vbQBjdkgE)|+2(^wU$bH>Kkv3&rPnu4PP>9PXuORZ%6-&+`Q0%d!|S7USZ!GQd?Ow2wJo}A1#-WeD) zyfB0%x{akL2L|bh)!?Z=&xL!kOPJc40HPk^lLqtK_5Z z#m8^QNHr5Ut2wCMzC6>p51EIKROsLSm9W!yO$;zMjaAMy43X;|?dLuyVSlv!pEZz5 zBp0p?krQ}Ob%Zg4P*&mn!~*^IF$eK6xy#c{%HBVav;+5Lh%*Cf!iK~q|E`q15?1QI z3eNQy7(HmD5P6aCL{0ghwX|0t?@-6pHs-mABXW>Kc~va7;u1Og??qFSZ@U31eGGCD zY-qAlI9((R{`cD$)@X?w?b-1a$p#ciM?gqI9>otJ4l=p_dDAiSruzt~QA}Nkd}5|2 zNCT>;-Ecmc|2md53U;sFvOhsc8qPe*b=442G+@}~e~)+z`NEBw&(4U#EGFw0$U*0J z4$l$ds{dIj2J%rtU~Y&am1Jl{LY8!d4Wk~d|9iwVU+6YCDPjhkEyxZ$_s2N7c#i5p zLkl3?TrtC1{egtg;pg{Hf(d!Iw6(j_kk5QTEKY)qrC04T@?D1K%7Z)MG7wP&p>Ibw zHaY(F$=MMcKVp+HV0I-3N6hgYza1pZ&_Gp6wj7ie?s5oaA;}4SpWKDo>=WOeW@2Ht`CTYFxfkE ziHX8zIF%(c@8YCDm$MFg7<}nth;`sB4vkqNOs1tcI85#I6WZdt+r2dyvlSph{#P9^ zu8Z$x5P{Did0d;vq8qZI{n%8i)~rzrXhPqKWpI*I2B;82@cPlUs=b}KUG#b+>c6qk zQ3Hf`tK{Sn0JOQ!T7iDAK*Vi(<%;{aUnGV`42I-uJK*YOgpE)Hi5FCEp9?%Zz(Oy6=|_q6d33sf?U~Xl`Ewjg zKPo~(edujd(-v5D0`egNfaGO8?K)P+&NnMoZ`i64Ekl85tD~=v<}H*SxL>qjh{4-L zu6#cQ77tjbEK*1@j|I^9Mo>i%wG#e!i?(+CPQ4-cf#c70!--YJNL~ySt%QwqM0HL$ z%8Q*^uz@UK+tDT?7P@yLWF;aHz^Y(gjoDu#OThHKcZf8dFaUM1DU%0UJ0dV7!_sAj zu{rmef4T~JMa?rEaR@nz;q?YvbEX(GLU$dzhQCBEW<$yk{H6HeMah5 z!VB=h;RqVfr@wk7b~sG%-Otg^4@rLd*_R#-T6m-&$Y@_4M%2FO#>*XFV-88FII@_} zq(=w)CPqf%@t;5UHbN~0mqXr@h}!pXiWc&Bb+oq7s3bSTXg~&m&z!Qif0^9}yAI=D zM#k+1q0!MGR;|w#*>)niLufJ2kq?2KC1TIX6mxVt+KYA<5hY|&ENX{|iW9$V@&vPg zR|%qn&dyR|Q?vjtL`O^^E)!kGjWaWmmSEzI$&!r?4Pj_A5fa-5%d38MrV*psi@zZB z!$$~%Z>=ynckUeebY!FD$G7Bcg<2lB#ccv{cm|08$4*JzGCBHt0+@y#)ziHqM-`%X zVFM9)3y!4y}}YsN;};BVEvp-M4JQ7Rd>gK~Y_u4o*>RG#q%X7ToO4 zueolsV#pXvK76>2{x6aK%}q_USO}!+(}6R(I5tM zbd*3Qq-ryKjPVB?(LUFMBk<|Ol%kP~P3QX+2u+~th|ZQ|-mqc$zM1h}#x1>=`KRlqc?2U0e{n-7C0LNF3zWBteYfAM9zbhB*Y?4VVntlDzs zPz#xmq0PR3iOGUN(*mJ_eg9tDT3?m-51_njN_nNUiw8 zYJoblHPbMW8PNSM?`M~DEk%Y->MCJZTu0y0N=QpHZr-edlZ>KQ^>Cwwer4ZsVr-B~ zE&m0<%C#DE^%o+SR8`KuLV*OX1gj#5V{8QVyl$Q|PEO)zowoiMCKX4NXeVplgc-D% zowUNF@wYG;7A;vq_^2Z_-~Zf1V7`TEtQS`#?hiY98nJFS46f1PS>TL(s$^Elk7<&wLw z=r7;a=(1i!$?L@SYj*l zynleIKN4^w6uuJ~$%2k#5Hts)j)4JHMaPeCfKxaJ{S6}4+H5;Bq_hLguSd4`=EPuc zQwaj6LjD!rc>!R=dz8)~kSCY}DSd`vT18J!8w>9|#g+=X8h-sYAlM8XnVnSve|ACR zZFWYcfD))9-=Z7QW5j(#V+K&czXfJ>IqV!{1~;qa7h7C&F>e`v=}o%?37R+T{20=z z7Z8F`$VCCw!gDV&5$=SN8=7K8JRFPGH=-0g9UMqz^mjouj}ng2Ayri)#1{#PiNY`z zq_~8F6!|U6ig~Z_y6fVka-%8=U-AGZ2uq*bVCK0aE}Z-`3UD>4tqYe}y$;*tn5+ z9jEc0hj_#Q>rKv!1(@A&U%0o^cEl{ig@fsi%<4Q+4z=#ZDG_Q#F0Bo_HE%#97X!7s zASNr5NFV4-cxPNR#CL#s%f6@s0!3~#_0S3xH0Wd zeX0?WD%9wlyLJsEpZkMh1U%SpNpn{vG40hW8Uuui7f_vsTx0@mleRn{Vkq(&J|=Wj zLHH~%E~p~q5njgQEc>uZ1_lRji2o~Y>hbGKA0068KF>apqJrbw6t)BSGXpX@I? z;;tv>_!sbHXxLT=Pjko5j~jpwf-L5L(qyiBO?+SU8?afZiDL)vq5#1iiEN=gR{AF& zE4zFgrhpd}6vfy%VIpSh2x<=o;DL-&?SJ2Q7vC57h~P4rmYeYV%V}r`$`k)jhS;+O zm#q@Yqm@WU)boGW&!9Nw(nR0k|8C9^@ABG)ofjs2ZZ&c?W9$!T9DPyi^52`x$(tTF zy|Vg9Lf6Sj5^3SU6b-2SIcuDfp|~2OEVu@2YiEM!lM*DLlUY}OMAW4HS#5<&tQ)Bn zd%_J+e*+VF8k_`VV>_`(2+X$pyGY7uLQ+?~dX6M%#N>~Bmxt{pYC*|MTsWNZ$p4TH z((>@QUD+1Mf_vUV5L;lj%34@jI&yGF(b#F;GKU08=$yu@!o^vA4b3s*2^a6>)w6Uj zh9vCR8RI(sNem%+2cWR6a&pmVsc;Y3m)T(nhB8e+QAWV$lpx{8t85Gph2aBSnHI9? z+pl(?PUj-(MF>Z8e}8+Li(%O6VS>w~rE_>ycXOWJhBQTKsep^2Y~#z7_q}8$pa>2} z84_VTk?&wAy*EIF&HvtYGueviq|FwdG!Y3x*t!~+fVfoP(sg^Oyoxi5^)lf>+GvQ- zj#O0xddBky^pyDT5~r`g5_2`%+SnMB2k$0q3c>a=&!0dkjrBi?bAVa27=-B;w)6A* zk&GhO(H0?v3UZ`|*VY_RATM9BLg-?P1a`80yQK$|8K_xsxKGzUF~jNpyqP3ejrEL-K@aow+-J_o$jC%XdKT% z{?aX}T!|-l#NshGMhkE|5#K49=z>^$~s7>boZ=ve3J;n&9R*K_c38bL5&d5rf5tUJAt}a=2Ns*T{0ak**0Bpu?C5Shv?c$kfBAa1;kewHdWFuzM$<;$QIvgl46g-bN@8XnHg4=4uqahME zcnVUCNXH?^YH4W+&oh}?ZgNGhK{Q0>XRqw^`*o_MK@A{%6=Xf4(2^w+m#97 zC~;T76@DfA?B>i?#!0tD?FH-?D)&ko+-e0W&vjI-(Tw+@)+~gQF^YUbo z3g%8WSr(R*$iEyJ$)EL;y8I2=^3tThvkE?_c6iH^nrpNW+v$G6%F3xXL>o11rSDCM zT`R6Z3@2OJT!?-Uje1hhozI7lrM7$Hee z7a*aB+-CJaH{pd7ij3}-$@z;6LV8e`d-?ddcFZIsaA99(2nHu*AEqGKQ zA`#-xHw&Ar1lxv?Qs&?a0kB8zD2(&UZ$)Ybqh(#FMp0aXg|7r-tpr!GY`Wt* zJ^B?ggWKG;ckkZT#9jWJqM{)#OV+I^UvcWPKd}Rs%2>Miy|R<#0$8tpJ~N8gaIcwE%tK68qa%^ zj=BKe52>r0V0)5S5W51KSjYR<9Q?@#xXkw`il&!|a+yGX(~-9B_aucJY0K-of^nxP zxj2fY@Rr-YeLJ@6Bdi!)kowJi0j%Y~X<$guoA&)Jc33crmTObG_1tf|D zFyB)_;8iFQ0mL}_E=~+IBHLie|J0eB1~UPEP5>}2l1MdpMMxa7oR~P?f~(@Ns6d1b zi$QWt^vaBdv196=pPQqg5||XoGxej<@g0BvlCrYFfa19%m!cBnA8634=52@IN2EVX zo3|z3O&Z2H-Fy=>GcvE$d}m&kzH|v_EL2q;R@&jL2&#_ijmkugnFs4$eBMlwa1@2L~$hYC8I z)&6b*O6ktzf*b?wECd5 z@5SYzleg$O{QgVn!E?ylKL9EwjGz%%Jcy=F&?he?7F+D9Yw*zUH6)FO`=DbMCPi69 z?F(r7Anros=kyQcRG%w(bInjq3`S|JMwz1qa64*r{Smkz2qriLv8O0Crv3}y0rGcb zvyVvW5a=du2k|A1$ijrCk091{j`WTdmv#Ly|w|s!t7)_>aEaA zqY|Jm8YEPGat`DSoF_yuZiC1euy$9F%&$E{v;~2`h%?hoM#MvcVgxzg>Qk!$SdzZ0 zm^@+tOOHKu8?B0o6RLf4*>bBDQzrsZFc*oqWLRRkR`q-j?2JwKUM*E=)<)5)z!4t4llG zM0I}l+i(BJ_&y3R0-w)RhbF=K5ZMVBTtTV3n}Oph5~t~t`%H#s1W3aeEvzm!4)*az z5(~m}GJ^m`IJ^^v0Wm(osPRKH{_>*YVj`nZ4do3_cIxWxhD1XERv*t{nZACz`VWo{ zJPsm%Lc&kr_%^4Rn2T+>hkzSlUt=RAHQS1#R$Xo6uu@6Rkv$wxzNWI$l$yPi36)ankf?G=OB8Z z;X4$}qtfjdN>EY6ei!&VJ3Bj#$2n{R&TzG&eGCBw=*2)_Sp>45;9p1?vqqVu6qZgJmKo6}tmbI7dT0 zf)>x0naUhoyLT^XT5t;_AO)F0T5Le-z`A?)ZW3%G>LQhEdXAkdo{13kVk5ytIO$P^ zPRu*1P2<4n0hqptu<&c{&~q;@8CeuH8^Wyd>So=8R|$IXKpm(osz1b#X2oOe!PCiH zDxl|~YBT;@F`}8^RdjI}`x6D{wkW$pLGF7*$B0E9!gY}|a9_`PFqt<9h2orn;RGv~ z*@T2H2~iyS=Y>O7fd)!@QIzZh$x>i93477PWgLh$)V~ElL=M6ukh21GFt>AP3ca?8 zd;-vE8G?ExU0qf{CB*2HwS|a~basNe?_!xB!~pU5P{*p!k#Xr@vGO(&6P z;ZAm9`w;FFIPvO@dra()6I4WGBXw^8o~S<(wNtDaX=xlB*P7hQbD*%%HB zj(<&nYJ`YK2=oP%3OM`)`@Mi5ED(N@a1LpAH&leAu($5XMwnj_siy4+mJ(oiiMZ5z zlR!Gga3K6G!pVacoDpdj@U^!CVdUQT(3Hm`>)`-TqTw(HR-Gvo5PkB2*d7&ZX;b7I z0KPQimDbA5jUI$EhLZA)^(@Mgp@hqv!9HFjkqF_a@oVou%z{|{jLZ|^vX5WrZlc0L zj7vmEL~a885g}tY={b5L{sgySZcotJvZ559bz2)XAe>yx^u>L{Pka0}RG=55-$Vn2qr6ZajHT*RG zgHl6E0J>QWs2V|_5ck|d2x&*)#6Q;nI7cQF2o@9|BQ8}Y!VH@?dleLTFeR-QA{iMm z@km4yz6MkrWGoS`E7MlU4!P&Da4ej?)E~Xu_iq&LiZ+Z|jG+%cj@X*8BD_7=y*u^O(Qv zKKnzQB#{GWkQA7md(OHYpjklmzs>QW{^Mxq1pKF}cx zQUMPKt7EcHm?0o(2*3uSEfSx>d?Vts281pw^;IMldzCc)Jf3ojbWI9YU8fY%sUs!c<{V|20CM|s}djG(P6`@FK8a6N)Dj>3{m$XYg`*5I*C&tG~`yF`4Fn_Ea;AcyN zJq+Cl%ExNe1Z)BGAOPZM8V1Kovb51MOpdoJ?44Dq~nV$U13ZG8v00e1_r**P0LEaDu1fg=s!yW2Abd4VIwi%57Z zxH3IrWN?zo|NV^z(gjsimji}?5)d*mZ6Lya&(<@5K4@dk{k< zAv#2Ls|X@Lm@Qww5Mv2RNg>Eb5C|wBYo7t&@pI%R+_K4P80I?wt3M+ir2yPF#~hw_ zcNBbj5wIYkoW{fU^7b}H@Jh!nvl+65t z)E=}BNQ?(h&&<4+4ITYQ3jhSqu(Y9ZX8uE2*&P@I7;ajG>hEww$#Vn?Sp*~mN9g^B z4+O1&&_StWqthNA*Ts7k_u;pZQxRJL$!;R(G(3Vaq!0(AWt<#G@}_Wy-Jp9G>U(z) zih!N#qx)Ud0;rx`yMyL5Obk-xy;8SicKpqp-SUajYL~|YUxxJYOR;odJoC)Lws0Om zS2#AXD&?qBK&!(yH364=;dco_zAzE9XdoO85Uj9Vl^%}7t2F*&t^WpnHc7udzgW)o z-V0RBkC|b9=A3&gQ|?h!fZkL6x`CCh>uW+yeN|c5x|64euAtq5veuV+Mvnp+r4oj+ z;|q0id!??K1OmCxbNh7y1o?X;NXcjFraf3LMaIT%MHx#q{D?2G3B>+4*{hkb$CN`K zg|PU`$Zgp5?S9BXRi;7Nu0=yjGi!UgvUlUXmmivpJssYBJE{PL{^R0C4I0lL{|v7?H0PI;QjKj;3TU;=p+Fiw8nPnvW_Cu!K?NT9`U35 zdT}W6Mhefurp(@fX3hx^gF+-03L}Xh;eF}LfZpT3g$Hnwkc9#aCYkrMXU}Q^dX|CY zB_utCoTrPyF+K3Tb2qAY^Q6s(yP?BayU# zZsj788}MXo2RvD`;3Y1Alji_;F{!&U4+#(q zPlgEF#GZVd6AfGpS(}d-hV}q4<9IQP^}tvM>)WqiL&e)1uLBfGhpUP2CdWWTspu^$ z*YBx=yfLy-H->D%OEZ+&AG^&j*y~-X>plB%0KA5(kr5~9xP_bo;`p}%5`l+E%MOI$ z=nnEiY>CNTvxlZ{>uraH;)-Jjv#-nq)*NGR9?5ZdVe?I)>$xtj)Jw)w-$6mdw6)1?P8;Q#ORcJPqmu=*Z$9?oPj~4Cl9PN=}HnGufn|sR1FRkBB>Hw1Zp+u5IINh zzJ2AQaa+P-Nu8R<7IeXge%sf6L`24D?I!*$`d3{-)4gOYFSh4b0=~k`ptXR-#HIX9 z(&2<*-f5(nBwK6WoKstN4NUn~o0h=1%5gmwK zVl!B~1Agb+pEXu$*vz(%y?X3%9&eP~O-9-*?J$hW=24Ixj~Zqa!u?bfqZ-qFGBgMk`veHIo zBl~+r}*~hi+s%WpNREW^6xF_0!pU?=zQ4 zk*TTuQZuR z{gFnFW0HH5aopfS4E?SF?Ld(dKbt{ax|?_J$L>q(qG-sx@|vs8r|y@WD!J_5=EzF# zpuFV^f7$zysWii(-F*U$ADh;en72LVRdka*oi@4Bx5@3@pzOi4bXA_hsM&6*%V*q+ zdwIlH{W!J4L7et$inP*GetjW}<2>(Yzdk%(mdiRfuMubb{MZu{c2y<~9^sVsOM%4# z)o)uXJ^2N+x&7~pSc+5(GWAvM>>MlQRt8|Ov=`yZH>7-E9{U^*j>-YPxs?*8CjZD z_w){p?bcm>^}gO(5ZB$!`&VAa|L% zVeI4iH;et$dfyn8R4)k^j5L;goPR&YqwV?S%a@((Itsk=PP?Op4n~f*b!?&xwD4c; zi!x0!{CH@cm7BNbj&Z6_vKO}Ld*paCCZ&e?=SiR6Q6p48ZB;pRJT-7~VdCArO%`9r z)Nc8l*>7I5Cs#Cj*Vf9Jz9XK)BP}e?QG7SUCUJ+0J7gS>EmmyCLY~&q z8M|`IZDL4uQG2W9sicJL0otPlC(i`TkJflobn~uw&!|mmK1)v}*JNgPIOfwgz%Lm; zlc?!g9c&ige4lfQ&7P*Y7v{JLd+}mnJ59&yvt|t*1N6VAQY`qFQxP*}d3%q4OM8$B zLs=Qg-w@))zdy=fn`W#`}6)^-TP1P4eJJIo~tO3_xQFP zuAn9Va?qHeEMHSAPyW19-(J5ee#!=(v;1d0_zuicUO3HYQp#!TRO=H9?JWGe7Hoef z%!K7=v(NFId}Xekbwhrv{`vTPMfJ4X$U6Qg>5m*q8kcyswZ)ZAX>5wqkR5B_G)gvb z7yte}r81T6yH!QWVpWsWkD89cLKAed$Wn(tS|vX6DQ!yE`_o`jXu<5N63LfB7cM*y~E;MfW-3t4=iS?Xue3i`%EBxK1lvN=Hk=E_5)ktClsh z-%z&HuNSeF>rC%`Xof16LMNlGm2?%RUEN)j7wM{t%4U`$qpz=MfA(u^GzOYB_ObG_ z_iw9at%T#u7j^Tft~xAG(X&nzXTDr)S#;y>l09_Cp>}-ED$PwtI>-J;Xw`HuyJ&QG zv&K#v@j?EO^L56b3afYNWd1%_(6;pSC8b*zN6PZhu7Yc_iI4LA$AuS}X|JYxe+;UR z@^}uezTzQn_>E?@_U6igJM!QY55zk!RD?GLoT>8;Z`PeJ5E_2%dHeeH-hOT7!I4A6 zO#rOqJ@e8F{1FSAV)`lMNZ}Cajg8T}?;!;om*mL(jn5=*ELwMQF=UIttCtcEgStL5 zQ!{Sf%Z|o|aa`Z&VOYbzLo&SJBDcZFKxx~eOX1`|e(M*Dx~jO=H;>u_BLWZPzgJu) z@zR*3ZcDy-P2ZHPg?~hkfaZ74@KpW0_w^j^S99)*>tATFiuPUkVmia0-H~my(AZn0 zZ;a>LimM)XWGbTC8oP8aU0OZ1%E9JRuE)c%OWzN-v1JD4w(+yJOD@;m;qaOa{xSsCX^K zhJ{McCQ{ma zs?z3NMpHL?MO{DmW}%8*9#dawY;0yzr)9MnSahT(ts3v~SHEk2WpK{waPOP79APu# zGZ_hq2}k2CcmPs_tm&7$JpFFSdl~UwCeeXK6Zh(y!$pVu0)m4t*z$LEUv>=L{LW8S zXew{AqR|q#b?x(WPTpP3jT5%trKfMtT&}zNzM<(%#>I(M-a{j8hYc43W&b|9$@%k> zI>0<_Y;9+ziUZ@Jwa`N5Lplm9MPEYq;91n8scxDxG)0Oi(@&J$tCw`LkxyEuDD+(4 z;bKv{AbN9TZvPp@(4y*Pra5g%jMo~oBV#z0r+yA{QJt?(LRjY5Q>Hc)J@2(J|JUa> zmWn&nE~j7r)ZwQ4?i-T1w(^=wM~HwtU(T<-FAsOgaV#*I`LT7vrBuYo_&s~-p5WfR zZUgVhF3<2aGc(;CJr$ubp=sUs>*g5T0`>DFtEGSF6_yo+G>cnbS~%Y1G3T~=hU$gs zyLwjtrrD((g6XT=MvitcWZVhuRFeu({e^=(>JhI3go4~3+RkhCm0a-tooKZn)IHSR zkUhowNUg7^s4@Cx`-s_0#0PTzCD&XzmeF{;Mfb;{tnktr#`|M@J@2mSiAvf`Q{~a{ z8EJfRnq-%#=8@&hcJKCJaWQ#Wu(OOakg%q{K!YdVuMybHrbsuA|BkjVF5CwyHSb~! z*IgFyGt5Tpi&g2$;g&aL5!TyhoN8n!d+MH)Jl$S0R>R^}`D2R%U1!w~or*h-F?>Av zv?!^$#U1r#{rFcw^~9II-W{N!zxm}jF$;AQ{gjTuAph5i6@nuQtiP*P1_o)Jl5>%I z+UI_bJ^r)}{qJ0ku}`_MvUCxGA-k<_3ne;iqUg%X42rt)uJfyMojWQ2KH`f>;~ll- zU!H3WmY+B+I>l2JB~je4LSC+>fip1M`KCCMG_4ajes%$`l zyu3+^S6@qRcxi?8tBJbYm%S9d$3xXS8LS4Itd~)q^G`AlFPWgdfU;?0;aF&PGQ^YG zKxFB_2Z_=+f7O?v^ zy8ptD@4HU+N$R%93O^8EXgYC(P422$o9p!N`(m5ccBP)((W%6`;_mP8X1%=WoqqiM zO#U0?%NoW_&TN_@1^8&s()sWQbPWoquP!TXE|1Um;GGk`vCfL;9PbO6%dlAuE*&zf z#2J&1ipl5S9A2Cn^Ov{o_-177&MH3njSt&P!_)O*Fb_JeaCqf8dt269A4_*+pU5<; zF0t(>jS2oWC6u~jTEo8$KpBQC#bP1qOR6P z^&en0%+Rx!>yU;3i&Xgo!E|(LT|9WRB9}i0D3lbG!%$Tz7M%h@psTX@8|kH24;9*B z^Z~9w5vuIRk_F%|&|xq`+-Hn?C)Fd!Q`1llhSm%jNn*qKZnNlHK!b$z^rf>`fVHlD zuABWHKlS+&&5+U+^6`Y_c0`2{^dP((%@E_y%`A!!6#(@FDNaOP;KRn>GM{IT%1&gy z=uaxEVOulKePjZ%?7Nf`pH>HTx%64!cz`U|a~VW>hOY5Dnf zh)6EMlnIcrXy5@0=Em0m_oOla-9%)gDDt)e1q)hE0gNp2dEXgJ2$?oREJ$?JHk}7h z+UpBt23Wc6sApI64hMP;;){qMz#oO3>qgs^(v1Ba4@vu#9xCu5@sPbMl?XNiw#Qfx z)Wj9KUI6h#>N`OJ1HD#5D*&{fFanZF`iuL2slto^sPeX2ag#%{j1(L~9=r{LXs~RA z`ok?j9scQHs*Z6jXxZffJyYOnyKn8!l!QAvuBNL-mOy zVadRuBIoUZ)RELTgWLW5rjp)1bqUPL_6G@0?!X3^TnEU<1aPIdeB@0lHj^e1Ktj>w zD*{9riuKh{_CY}$ONt)AMq$qH{%9EsY3JKFZj1s75=5y7l(jcdNpPzP&F_N(1K)#2 zBeF#hYCx9JP2(_dLzF~^N$P{93X^dX#!5})4M(Hw2i$TMHY4d?PJzP!z!d>)GI(67 zeJB!fLWqfW3~O+5`-ZAxLxL9MM{g!I;9bb)oEmBS88dk68QmMOgIqAVc#DNgq zVg4{ej!tM@nd;wUWqm@{FkuPmos^B7KmUzdM4oRde-lPa+Vh~NaV%qvz z_WcX}5xyX2X*-~Iq-T@PK~h&!HV`%ag`nSvFah$wE||gY6rG43v;*sEy-@XsoK@ zZ%uOk_HF)B2=}MLJ@ob zr7U0t-4o^j3xm-nY}Jxz2_2+xsFTqp$0@P9z14cM9hx{wZRLZC)sWzjp&O^=ATsdU zOr#pHcdWzsQMInnDBxTo1-%OXN%cNI=;0r3^;ONFL?QYm<{Ded3?vRAyB8Ox3i^yu z6W@1O9nu(%LCY+gPAVuKpgSjB(I&=lX=hRnIclh@tG`hWBXfc%-oC2oOWFs{?4Ykp zse0Kv@MA=GLa<8S4^WX_kBp3*`GpXT$a-<~z}8>Ku0;EbBHZ(P)I}2;1a*@zlr?13 z3d9^Wputf%zf8DgxUGH;i%e8_P~?yv(E4t_HbQ7cgcwlEke&e)m@TLFA@-MR>;UDU z3s?_^1(I5w4SZ*U!KcA!ctXle+F{}Q`%Zh{t_Y2UL$eli03n}HN$7;F1)92aj%87R z%$UM$hWam^U?Mlc$)L@_$Q0~+M<*xd(9X8+uU$}NfPUmlWB~YszQdv@MhG12Or&cM4zyVRFS4(qH==?#p4KuWhQNkl~8#%*TL%K#Lu7T zzakB`fpnx*p}~GABgI6--WouJvP^K~ePjrfx5fAZ;g&Yw=xFtT!&(d>F}exGgVq>a zeS+Dle|WeGa5YpacV~p!h>#jx1qo4xlZi$(N$|ID&vHNCz>~F#+X1^F)gw5JA0Q^e zdPaIjf)i!(XNTDqah|iN(^9hAjclUH(FG4t&&=_U@^i5tvVz#kmRQZ?hI}?(>O> zef<(wA|$1+2)d_a0~9!3T17A+pE%eK8Lq@J9>J&&Qk1Q*cMj zY}=w-+8LLYxrhJs8M?XOALys?C;bw{o!i>j@&X-{Q$hTpqQ(y73neAx=RU7fs6nS$ z5O+CFGW^@Td^D3z9;_!=_Li4#Q{-aIJz4Oih9PWxvFTH;EpxoFSLycg)X9f6443j; zG73N$#yIb{f&C}Nsu1Fm-U;4zXc!L#@GYEoBABrq#)d?r-Bl2IZxK~p%;llEyXy|e2tahbax645_A32g9`(tm@!nHmr%7%Uh6t*WuyTB73e*LNq+D6IC z-!?!fvy9XZ6G*+VPyvnDAxndp#S_+sRC16u6%V4KIKqCxHN$3aq?gf09TJ$c`BKX{ zHne<)yRJdUYts41wH3A}3$az`ChC#Ol`mhvuCBEOV^?^}Sr1Qy9C+;k7;EPz-9$(R zUqp+tOGsmiVa)idP!qi3>f^_c?d!qNuw8`^7p0>Xw)_wxKoO*ckyR)YEQL29<_#n= zs#LVZ>#-tu!-;-6Qm}R{W9?RSQj!83UR|6R1QlbXAINn1Jm7CqP6WAB@5vO z5sRQ~12JSq7DAJ|Aby{mJgk#$1ep%h4mBBO8kpg7;UX~u_{A~^ogf>{+}usN3u26w zJeEA^PcSY7aR9!Ogi5fi8kf{+4;?^jwKJ)yX z=QHgrZ)BrdnS||c;m@!OVPV4>QFmhcnqdCHlLF8*$f0&(?g-TJ^A@|l73g)6St`VH zmvh+=JR@Sj)XhdVv$r{8l1oY{zYOV z`6EC)h-Za4;w_683Q13Rr`=L^!&cqex@CKteqLCA`>Z1ZequCnYcXL_fw6M_7M%Na zFK((&gVl`+v6B}%@FY#T%fMaT8ygb?5|kdjOYj74nYRd{tD%O1Yrp~Zh#NnpIswH4 z8Co`?> z7$5Qt0Hz*{SM>;iYiqOMB?{^KLTw=qYzAc98rjj2PtAcFr@Y;!tQO}gptnW#`cNng z2j&INV?B~4A`zFvXcPFQVP)LVtg}U10q~ZMKoeTGI#L(FOQYAaIxck+uum!&$g*f( zLsDrVtx!KL0pttAk8$=Q0RRUQAu)0b;Ww`V?+kDS0S#e+tkVT|V6$%HSl`PIlS6uk zl_#YPjLli+p?R!lZeBx10xd0@HenK^C(I3{U;;45`)oJ2y#S0+4fP9P zP40(=Uf#pG9DhO3q+utS#DeVyqe~|N-Xh~9vza&uME8=`P#O1+t9v2Cu2&b3fA;do z#DaTVhKZ#SP8ZAW8`9}N{wZ1}7V^e7{)+y^GGC#PH=kGC_T^CYHRRz_47zuRKhQvt zE7NY_@5s+-DSC2st|m>ilNbRbjFT6WGe|K~1gKuV2T2=EsACc!JbW z-~wpdl#SXjnppGglhkSs_lnjy<0v}(NoJd|iAk7wX>kz8TM+ITkZ&2ns_qgM4MOcI z#j)s3O7vS8LhK+xdBa0Cl}fsjc{zY*0TxrO*%T@juUe0p3ow~1FNT%IQ2GxTH=HsN zEGn>yN|^QQc!Zbe4-z+bC17Wi~(|fHJV3 zjHZa${i+7&>4cI}98Mycltd<-pOqSPXr{WRro;ZijvcYrrwp5EN{FOdON-dK7G^wf zNx7o;(u1Ez9V3T0&{vL_VL;Y^@lMRlDt2{aN-`;l7m_LjUq4_?Soaj-zoTgK^bAI^f64K7BfKG%XS*3i&G$IqWx5V0k{zVU4!guH|WWmLHdsdui3J z0Mk&1o2LbYBtiIMfCQ0lQC4_R(__$6stz3nUME*GJ{yBN=L(WD$iWfcs!Gd9R=T1K zqcfP?cF%pKbH>uzTKeL}NV3i?iHV8FQQErk`zH%B$TJpup?Qa7gmAb44rW5TaiyS; zP&AAp!T;BINuY4otI2alm8wqW7SA8*Qg@WDYEnBQHO^hE9@cL^IE20!e#?CPWK`=#?Ap(k6jb+ztY~oQAB@2Xy zK}v1pjCmxSHzi0m;p#}kVgSNkAfj(eI1DKeRnxB8PF{8Bp5D@B*MuKH&Uz9Wl#s2d zxW8ZR&g;TL?~%u(D1<`J$k-Ssszk`5DGL{A`B9Vz2?6nrUUgZz5$e`AP8M zhy#h;M$~5ZQ$$ph&>8x&h{zqyoHj@OBWlzdANL|Vn@r9C`nxt*!Q<}TH3)F06kob_1Jb1J+gteIHA?&OA;D@DLY0G88_3dmLCQ3@$0wthx zQbnbbfA{*exa*u#_w+99vHRV3jQ)Gq%rkvW^W!Hpxi z^M)87s+AUD{BtLVt>so9%^^Xx8p1Q3kFnPf5=)xHD>jUvDmlXK(>upi)E*~7VT;h^V2|AWlIKTICp zh1$WjhBJCGJ0hJkON*b5l;3Rr;y?ZQY~`6}N|EWOZsZQVu^F=XXf|n|;O&OBbNuvI z?q^l+7aiLC0qZ+i8HuM;w085ZqzPOv)ZHwpX<-=zi1n4(qMEuh$7w z!Rmk;GqrkAy73P++09a^ln}x#QeZ%3D%ZlXx&63Q+j_z(FSd zvw}6z<*>4Gm{w$M1v2;1!0SwhL#B#WR>I4cEdwd1>0xhTlQJc}-E!nz>rDQhKN2~c z+LGks7GB6Ib^gk;kO%R@12Y*V2< z#fuk2NmZ40z9d}S=@F8A7ty0k#G3_X#BC0x{#1Q^*Zv-Ar=rKz&w~|5j&xR@>DV4} zQ)0R7iJm#GA~Uukv!=8s63*`Shkr+>q?FV2ruS`OZP5QQbtP0Hym;KWzNs-*`&|O< zZ~94_`cz!|r`@@)7OhoO|~}_Os6D9P!Z`SGHb#u(u{7 z@tb;5>%HuPm8a{Tn=S(X^CZTI{wq}#7OwuwI| zdQRCs@?uf*^y`DRspoIy4W5c{Z!6C?FfkQ1DT_+7R-F{fcl;w^n>rh;n5Z0MaY^29 zSIuT|>%cO>Zr^sh4_p$qX*!jDyZTLxSVs?)BjJogv+`&A3EDEa=j~TEma8^q|naV za!2laY4GKiru5&of~#ZA?TTzRMW>XmERWom86&G^c4z%%ujju~qJ7gX@A6Mi9z16^ zYqgjE{>ylp#$+vf=ZLnu%V*X#7W{hd^zdwM*@I6tRU+06y)4cb%av@oX6@OwJX}~{ zq!M@L;Me|DOTFND_Mu2@N)VUbKcQ-y z%WS}BxfG*rOIH1dNmG-Vbz9%=&`EZ&oyt8?-aaTK+>q#)IMDZ0UFqVrP$_B=@!=#o z_S?Yl6Hz$7V5ibVE`%+B*b}xw^Wz&y&=(>vBgYCqX4Hh2l34>9Kv%NJ(rxQ_^*Of?XmwBDS3B zL;m=~>e*cGhjlkze(y6;&?*>MGd<|8!nWXecxw3T8H?8WE`A)isyau1)$}tLD6`JJ z)zxlCzhquxwrz7%i#q*{wykj!m8brd`;m(6-2K8gR=~7549~f!N!?`Q{{1s9qOErz z;ufW}O6#TVCj%*q4)qST>m_QlzSc2yXSpXW8FI7V41ZiRS;tZ^Y?F}~WpSzE{vYla zJQtWL+EN(=R!!f>VC~o@TO@zq{kXu>t)j#_T(%udXm& z@1poFH|_6Gt(ECuyMLg@pBrp(JX zYG3tUMs z>8WJvsoXWiqiP2zebe^A8*2alspy$z+d}?wNjbOb)2NMgF&70k2L;7FO&hj5r0(>! zkK?4hv)#7+Ax5m`S8a1#yDGL`zkK2D(Z{QtuW^i2&m5j!_0{-C)Gnm-lGKASxV%#W z#x$d&yr-imC@K+Qh!%g&%pdqCgf<^Yxb!evG2rIOv~wJ#%8-AgG`P|gl0SVmg_)*!$y!cKEVx7n{#nQ}Gg3L8V>&M9=}Z4_BAIwiJv9-%iICc{RwrSPh7chB)~FUTQ%XPENr-3L5O)$TTJk5 zJEht?Sh7PhV<wpmuZO*}FYP{w+O?Y; z_BHp6Z?bfEmy6+{sk>NGZ&q=}uS=EY>Vezumi2u|5I%Q)4;>Eg`^HzN8L7*4Y_#%b z_6j>ZsE!BkBy$1AjWiLk!*r&VH6q*@<=EkNQVNA8hB|OCjVnUoASC*7=Tm61};YhxP?zx=J z%(CN+x%T!pb;T~qE9;|Ae!9s#pZPYtp{ZA}w^tvf-Npo2>iwyGvW<5_Sj$iTDcpCU z_OOVP)D|V*b!Q9uMZ;&W6leF>9l61nD!zwP_)g))Pi5(v7ry$h#7!yMP&+;0GTbpJ zTh1saH+EoO;2aqNMi^{~PBVPMA=EbfK*P|8hWL_;V1B>g^Ik5k=a)i(t9)N-l&Jrw z!taLs%XoB6!cD1v{D|ABr*owI!{>v3BGx*3Qb#O>gw6)-Q7jtfZXBL_kA*3zW%99X z%@3h+d8S+JZo*=^J9cVlyeK#@?a5?<<>4Cp5V(Z7ZwI(!_NVXBT;WQyU-nCGZa`UK zT%18nWVx>1{J;iQb8U6Wix(e0Xn!>_lIziC+`a5_TmJHt{SiH#=?kwN_ult6m&o5d zeQ=(-He-X{)sV>T?A(4DZoPlT(q>)C%ehB|cWtW@_6QsL>i?y7=*9@oBm1KHDsLMd z{)>b05k)Vq)-+tv`SLh%qtNTE>kPgI72Irp$thbiR$zB3Wud@jTl=fNPw9`!3On;Q z4!1q0Ytg%MMG5p4Ur>reJuHyPP=W(W%_a5q8y~!yk5(r});@BAwpxxkzG`elXZ4A@ zYX;2i(iY19bpB*9or^}irSd{vW&YH*<_g-8=47skB$HAbmx}`Njo07LF*=!QTUsr2 z448Y=2z<2W5)V*ln-e|?p9Zu*Y)WDs*YE?@a@5$rr`t8J`szpbB_&@&rMhO!a(dc^ zp8me*Zvh0R9_G%SPli)&*}Bc&5j8U?ZI@mKYgPU$Lv!4>#mVTlzrQy-Gh11)q-H^J zh39Jq`eRlvrKV0biPE@nL*qtF-uvdeWH-^bFr;Y~J$hi5&gmRmI%1Zcw>!dBSj0 zglv*9g{}SXs$&nuZ}zo5o|ss74e`m|$m1C|zFxbkGs^w5R^WW|-?G|npQK4G#pI9X zq20|+>>306;-bP19S-?h(k&8^@9SLVSG7s&fFliMa!9=H+BvLg43|QEz2#~&O=F-P9Fc$Z!$!&TBqN zT;7|7ZCwe4d}Hyi?!7M-pS=93!rRTC@72%u z4&E+rzX0d%p*2cBpkZv6I=ioNsA={YSFmYw*N0E*zUEnm+qX^`+8DI_9#B62Q*pmQ zs9WOW-IUy1hnww=lt0``bC>VXv-DQgxyPHG6?R^sHh2Au#Q{N40~mpiX2GL++YROm zi>zkdUO(J>FkfG!-1hGp$(jU#&HJr6if&GwWNr(m)3Ug>urlBG#HEI6s-M> zdvg5QB5Yym=(mu4IP-b6JNx?6v8HRPz2_HBzeKKO{br%val^)|tudT_1Ht-DWsM{v zyIaulLRU+9rQeqmz0O zUARtm)NU%dD=umKJw9ZEEP&vo|QObgj!QeNk0(5RZ{UlLGTV-TRTmy?4DF zPS~f!P_Sv8s>M#f+I`{>BPVO{`pee;oMTf`l)t?5Ar<@S_*)(hdyb9TCdV*j&CYN% zJOr^AN|8i69fCEe%Dk+HA0nDL=xBmg%k=Q6=er^$;KRpUi{gwgt>-x#$3yMISo89? zH{vU!sEGZoT_GZ0@jeq7J(r zZ60iVs&k07$+2kp{G5T4T9~Rwrm7?p=i$DAf$u+mR%5V}@M!}2(c-`J<+00`dC(4w zL7^30woL#DVOrN01L)bYYZgQBxXnDmsd8OLEMm{SWYe#l;&lIz9n^HEBVHzb>hbP< zySo#fN&N^yla_!V0T(Pm<2!5Xaf90^)#{s=fQxGb;g*9^+agc_9bxqSOb-^(Q;~X$_Btov*)D!!Ge*}8VjdkGSwpnAVMktm27?7* zCPOrK#$q={X9=`u3HYm?tbP{=QG?5cr^p)oShv)Cn0D_U7A3K~@OI?=|E} zn7PSYx$j4fmOPsjB`P-d9a_b!*RIt^?+(u-Y=WzI#9_hT8!^dOSK>Abi*gvK@i0d zT_9A5A&^kSZ=NUVwtOD!pM(w(gdD2aK|Ee?d^0)wGPU=Ywol#=8-~{8chISV@)MtQ z9sMv7P=?{qSD{0hF(d}V*MpujST`S_BPawq3A9Eth8)P-RfzxhP)TU+jH8!4I`T~# z7!|QLcDbN1g{B4&aW5wwdU|@uL*cIYfoicHJdNmUzMOot_;C*ZeVhp& zh%?cQC7ge7%BF|?Z@dGzizu9daRZRo-p@9k>$cnhpby-?Fdq*=8uH_F)?#b)8w>|I zS{x0ybONAlBD5s9UyZhGqY~H}U4Ul@1{v0%XBxI5EdY#FpyG=H+XciH)EolHMagkz zq#_^3A@c~W$$o%~z&8Y@vk){5Ezz$uFQ5If^^lc&q|4thxU6xo{hJ7QqHRfzE;^gg z0*M4(ik^unKI_Qh+ifm*lK=VLy~7BgA|E_tl=`vp)v$Q!lBTodMLqy-Lzs5ZO-h2+ zN47pOpqyI|0ZGY~9LfIu524q>&oh9@ z@SEvFw2YR}TQuLmBu`X!Ah(wpI?-r3J)(~IL#v-W%XrNub4y&vP3D1 zUp|MF8rpb*f`T7xMacrpEBmi!AboChpVC}iDQKLOqisbrfduZlht84@mprrd zB?N}h4`Ys=tUeAMi`b$HC_KVZg$MhZR^e~f_h8Ub{oH(BbEH>{8~+y@TK zanjENb~<|pIW)`v^Acl(ffR=x4S3+QS4m61qm66@4I$w)gX1C$@GxCfWpR-NE=an= zxW^EpV1(!do_^{~frAa<%|&yR9FyL<$Y$G}_=4{bxc^yxFXgYmp&zpQ#fGctm=^s6 zH?bVNZLm5Z^67xm&K^eD_jls#AII>3-E0IuJB?`%B8a7y=%20c^a2RIZ@+$>!d?+F zP=H)va=$dmN0z>oqFnZqv&^z4bpObS40~@+f=^!|70t5u!lV9o4D}W_L zu%U!5AIPSIz}6WV8MXKa8{!m|BLpOC0b{9;)mr+Rm;ZguU9_gz2;U6)Lp!9hYZ4Nw0TI-5BEke0g3UvTQ` zCNO3I7KlND4dC%cKamB z822b{2DZ%`<|;hWd3FsVyfA&@{VR_B?|Vcn+K&*<4@Hg;1zv%09)UfYxNd;| z2yzbnoxXl2?SL2nsLAC;n`e;|%&cj8j;9R!Yy@zF;+nneW#FU?PnVqoVsVC45 zp!NJL93a8W0-yz*FG6#Sc>)7?q$5DRmya8~!dwZ_2E%b9l)z}4a)QK{kia_zRDX?w zatJ6LUxKL|!BXPTLF}4L#FU9&4Y7Le)9w)KUIDZSWKL6>*noHJ#)StX?On{JAOczm zoi7ChJ!H@W>|ok8Q_%o7zQ^mrQis%{ymz=kN1no4`;yG__#i-4nfW!=r%cA>)6=X2z7)Wpp4jVWGSS9$#jFFEx}=bWU@%T~)2y$r zuOODZxR|1-s90THEhr)q3)~VyYCS_kA}ti(9Lb!d&WD>)gY_gF)kMZ_5mv!PEQQ0^ z^&R-<>cCY3SSj=#J7Ljr*Rs0|4j(;w6u~PneTJw*a+~^RwA=UQoSXd_&?ii zm6g9>uo(7^-UXt_2<}G-5kGPsMKPI=xC>6D2H17y(1fym>U3M-w(5v1eX0GSGKw`_e^Rkd3)bsK+49-$OQ z`qKax1wvF7vFlHlzTtm&)(hZLA>3J#c>#f#x%Idge={Oif=mVgN)hQa;6Trp{+d7S zzYwQlM^A0IqLkEf&>?-aW#o8SgS>~(*k5#>;Q|}m(vyiTr4AKTfQFS4jz|Q`i!OYJ zOg18S67&NMC=7u?M%ntsP=r%6@ISbiARY^Ay;=O`Yx43#xG0~&evG4Igo5nSSMdG! z72Qa}KW7+kAnst%_w11IzYOyNE29s7$0vvFuw7U#uk+GNSjU{Rfbbwhh=onUq%o3H z_X-t;T}n|XFB0iPghyuBCY*!a^8b=h&P~>qmT&6n0${TU>gUj@cvY}$YXcjNA;gye zK42SiZY}MeRT5@WAs~VQHm4U^SxvTV9GNjVh(t|=q*`!|2;K4dqxDOvyUd<$@XZ1q z2P@cwWz9?mJ09lm1aYVF9oQV=Po6HN#GIQH`k6jsYjMqAdJe~D(e++D=dr!me2^%6ZrA}z2<2BQVJ@wXGg(u5BYOw6%Y$Z zXqOjr_r;I=@8npqAbmur%}0Lq9blk@KmXYa0sKh^i_h`~Ru+e_0k1BujZ6q}6aCAEY_bI_mmJ&=PNYL*>I!cnhqs)* z%)-hljGTeMUr~&RT6Di9$7TQ9WEt26k`Ljg%f+jb1!vDf0Px|d%8p%1p;3zsw*T|% zM73Cj(F1l~0iRyM$Vqzr%2v(F1^Y5zv8QEaWM^_y-LxTLQ9c z5GqFVc>g;O?9~4~I$>Mw0#F9hpz7qqHAw!1qcn|p;9o;wny1g?uxo`QVX4Dt9kd#C zQB=SVfgIKg7~fY;98 z{~|Z{YSqix&Vr&6C{Kq$Aw(pxNTLn}^|iJ4vUIDdAT=RTG<>q*x2+`Uk({BdU%%e_ z_U(6`q8wwZCE;jP@DRZuU*~hxGc!x!@s0uitLic`514A;I6V@L{?@ZV6U2=)QGjt9 zCa1!z@?tJBOgs@_^f0krr+LJCG*OtZ+Si&EybZ&cT(RuC$7VQG<8C^8DL=mHoPR-8 z^}cGdIJr*GO55TT8QD1Xy z6jc~ywO(Bf*8^)Y#oNQf1C#DT)Tb%&2;W#{pVujE0ry`e1cHLlcFWIbdzJ~FJDY?F z9RfD2S4vYHbh3{$4hoR9osa0aLKF(Tu>n?~4BlWmr8Ym08xz?3Gf4L>r(#Y{&M0u0 zq&|QQAam;~A|n7qJTxcW#MH3uq<#k^e}(!voO~nXR$n%%;rPlwgq$HH8BBi9T3K2m zS6B%oEQzac+<0B~7X^-_)*FrRI?F&R(1U=Nbq)#jVUI$}8#b)56 ziUr7D-(k47(fs{@AM8jb8Q&rC6Q(WI=TLXX{!^nQPf~1{K4HC*)sxOR(cYW()}k;3 zfV_G&|(TZ{jnE{tg$>a5JOlARl4Bm=^LOoK84fTR2~)F{}G4iu>q5;sTVHJbllp+7d#MNUO33Yl^6L++;-*4mB>B)5XpW6 zozkLZazEpgtGt4<3S})n#QzjQDEtv3fM4mDtR#Wv)vKpa=p{OXE(?ENIV=E!VS<72 zy*l})EuinP8C6yS)nzqDacuSB!-p}$qk5qnCO_N+KORuR2uUnF%YYOSg(h^9h>{Y4 zZw67tdWgDWpW*^_p;B}6&t&sJq`tm>7?>M?j0bpKHCl=qiVUYP!I8`lY%2*dF^Z#$ z5gTyXV{)ke;4W}|2JSen3c4FCnayFt}k{#iYz%Ji`R#*j%Jr1j9IN5Uv zJ*I>ig4#GErze4 zz{5RBb_!M~1wIvmY_zDtDo%+DHT~UkNh`Mn({eZUG9z9bj=}vt*8h(qFjO zLOiw@`zicjw@BTQG{dpGvE+;@#>W^ z!ZS#CpW0ClQ8ds25fT(4H3g?ml%@PUeJ0(TgY}(^Fln%3(C`#K z1BxvNaiPk}%I@IH#xZb_IF&+dnNPvZ8DtodqCRd!^kF1UHBKi`Vi=7yLUnj}e6j`sG^s!1Q;`Kmd5Mz5wBYy~Mk=Xp^9od#E zsrzC3$p+n}=W^9NI-%-7Z!|GChRTN_CYy2B7}(kCaDXrf@&>Z4x{y(QER;c*3ve)0 z%{S1{pnjuv;YM{|29Y{I2J;ocrOdWE@g5nkYmoyr?sH$K*eH zR)906fE=-@w?O=m9sFrbT3Q;U2ei5z@W@RSp$)%r8u$5|eTZ^H5TQF+hf<#!1$x>d z1Lkni15;c_j+}=dOw!6oAivO!Bz2$*@>9=4Aq=sclT%Y+W}Yr`|IEC62dFWawl}?9 z#p~k@>R^oma~L<8d#2HW;OwRTaH9hS-8h}j0fb~UQS;MX0eHgMze2*os+e?50%3r- z4yH?QJ1koVA65z*9(KSY0uwkMv~7w;t>~3SFXzFW^ME+7@kTLg@cJ%4m+?zJgbf!4$%HS|8`ERkbjcgM3)Q`j>)eE z(cVI)gvJNq69EB76p{&udD6RL!pUl8%n<6Ou9`0URvpBB8!t!QFj88p!SJ&~Qv(pv-6(O&EZ#pUY>BRpD z3oGU3;^A5Ciq>pg%R}MoWJrU8+)b6NR9PRL_!6_8``e-k5i`1QCd9Q)!n^s~xe~(C=f<-cBxlvyrgZGFh z7}(g9(3eIu(8zP`ZA!l(3PzD`c0El+Ca?&QDveBX7RTQmV$b#JG%uAxss`Mso3eU;uXHJ5HK0@{Ivaj2B8Kd&uZS9vDGmZ8Hmt38N2mACz(9e*OA| z88w(=Ww_d9L^BEz29$U%XKEqKAYDovu6`86u^jJeY6NQLD!lg#&pL&18n(Y~j=#xm zJT+xhY4>}uee&5T#u8oC3nWDl(O`+GSzLPT8|NaXz_YG`3M6i}G>SXKeIX*kWV2{f zx=GEJ@NG^`PTDvSDE>b9W=+Zg`EuaMIUYYBynyvd`W;W3Lxhq^1qE#-jvkUM{5 zNW|FEBaCE|QFye>G7P*%=AhC6nMm~56$xDq4lW!S0Xq;9AyO%Lof_o$9BBp)VAG91 zaOp%u4kgCqj-qDn)yf;$*q(4rp#^QtB`)XL0{9%zgEHc&}HF2UtfSNbyyOTH+@B$>aB2NwT0PRK5dO$ATO%fT)nI zz>^VufJBm8brfDGNX6JN8GgL6k6leJc<;q+2q}pgtCZBOvO!mV*)7>kZXiI4aT?D5lf^!-i| zrIs9PZX8m@ESf9eM!F2feu5~0*JqV>*WhEqZ;*0`33F)Y!oq0{91_tfW7#e6)=EUO za`N66MntJA8X~N^oKU@`XJdO72$|>YHBvwF2!)ZpewTX3IFa6iz_Q6-AwhvK`}Rt& zDW@L$j&bzUq*33)aZ3d5VqgeseLnw&(7T+*MyNyR6y6N`RU{2x%bAAaTZIa8F$0`F z2uldTpONY(x<`c64z@1FW#KOb#f~{4*gJ}<;Bghk$0)W9bd@NwL(#B;; zfAQiHSbgDK8JL(#v9li;f{hc(W8B=9q0`)n+!MJ6JJ8)AqkDsFCJeF@SmAhBcet!O zXn;$ne=H(Gbe>_8z?fD7Em=bEgx^LL zEVMrx331tL%PulXOt>?!UdAw3n6O-h+WF+XhXoR<%Aez(@^BrL!g4aX+yBXW|^rx-Y>EIr6^*wg`kQvhF&=jT-Sg;DPn=($@F{% z)(x1RbE;Vn8_|;s@^yYz(d#K6hfo}fVvf2L z4SqP5B!@X*?fwQ79NrMnKhQ<_8*cFOJ>Qm%C){_IFz?*?`}2t*^`GZd)K${Xs5gE3 zq?)X4-lUV3{8?d0*4=%(!Hx<(gLDJgPw%ZdB*u(3zn9^x+SxT{zpZY^RD@%##jo%u zhsLXp_R>&R01HJQ0DZg=isXk+pO|;jpGg0MAnjIctbTgkH30zuT|B37q~=&qqCcW# zU^HbZ+tC$Oc=fP*ya9`VR_!Z7b8$>oR#y8{Q^avQOTsP#I3_*`8i2UW^(;Hy zyxOKc`!Tm=mxu@hY|D*_zKsYLhzb&#WQEpFP%M=}CBLDuu^uwZ$nt~E?m=4&)Btx1 z3Y?OZ&7YluNQo=z2wzt{&-eecipl=r-`hbO<^t|Rmzo{Mw&?A>9t1yEWm3`U);-}qs!>f=X7Jc!2i z%}i$yumUGiTv{5hIsgDhuy{ebDCW+s7(B4pY%eo1P>NL1r+1UoUEcBOQ}+2rrQO=c zQOxl+-#ysNMIjheOe*kd`gR-Y=`Dv8yh4idIS^_ChLfdqqPP2G@S%2xW+7#*0LSr3 zgiFB(hrU)+td^6LGc-Ndeuz)ov9yCt_n;Df;ipNZOq#WgO({+anX{Mml#-EAPd%8s6W^R1otc$|y^QV? z^a=w0Zzw7;aTS<5mwqXf_TMkTzvGwQnE1+QNb)nsBJYMVy&ogZfloN2v=TI08=FpJ zSdNE>N7BeB0slc&0Y=>uQ47N_8JP(kUEzCpji+~n1qCUTo!s2J57KjXm)PLt`kYD( zw&U-stTWWrT}JWp^V7JtQ>OCnAU?Hz&g=cZ^{K?Tl(U!l052m;S%^ zWhM`YiG=cK&C{A2L{@({4Sipp-5FO=={%Xlui_%I+UJ?3tBNG$+%;l}< zRxzP(1iZFGPP=thw-`g<%_qB*7WV}$8H5t+KNqqF6QlN+fUQ9;Z z{5@0YGU|Ck|4f6T@0J7W0rop~=1j=q$yHhz;1ELRU5?T%T>K@}X)s8z7C4)DLyT{F z0sOb=uq_HG!1kwz&WHlUjiMRBZ*GPMLkJ1|Uk0+SEG#U-lOgGqf%4-P-N{w!2q(_3>GH<`s@UT3Jk=I|Py`)*(bqPQX^ zW=grZlOD0sqD-bv{*BR3pSy)bMconhRp+`{0QwdxL`7IjDdY{%^$y;4uBkc9i!tDz z8_|A?K!8lQA`FWr=H))RAb{5sS|M3a&Srr6i1aMvh1rpZyhG!NZp#)QzyNj8JZ`W; zI`|faJMXRR8#hAB`C|VC1GF3u+K=r-X~q>k*>~GfR>ENeObFJR=V&eX_h4HvJRnYJ zy8#Pl#zv7=41p;KJPZx>LbBQkYX$OZHdu%k=zIeT7XPui8vq}&gy@}5qs~kc{$m;% zOt_&zS{Z52Ml01mx_~EjBm9#m%+>t-JRJ***0U=rQp1x;Z)Mfo=Oza|VUWA9;V|-n zzd(LQBf}YZ{gEZSJcRrSM=Lx@FdUVnc{X*NF)ls%2wx;|voY7HHl5{JMRn*mO zLhjCuGLNE*?X_!kDC$Boa<$0?983nhy5QBX2g7nPT(PIo2T_49c8tHgTpPfX-7bINQ0xqx38TO z7+Ge@YFsUPy!sAgq{o3wdwfYxkCQ@&vNm5_VxnTz!^xvBjcQnzxzcUk%yU#i!#=#j zZg_6m!!(F0Ut}??Vuq-kAaG>_g}J!Uh&tfUd{_&vc7lGib9i_I>g4_V5kyi^-L{J` zO^%QEKuJq_C;c)CHi+=Jp|Z<{d>2ctMOX|RM zF6hM}Lb*Hr55VgyR;-AEZ@7&9%k{iRk61CEMSE9Ns-jWrUv#z|fF(fz6;D`}y-q|5gv#G@>_8E*NS@ zE72K751$6~tZo3Fpa!`CPBTZS=sEU(RvDKA&22iLQi5F}>AFddol}51IkJzWpY>7T zjJ=-zEYTApO-u|4G9lBuO(F+rPF22re>gaC&`5a(M9mbi+yV)=os)Ap3d;9R#08Uq z4bgr346~aM{!{KnMC=KdY6Z4Que=t=x5wDpdR*_1s){Nf+;hoqDCJQo2+sCqr(J$z z4g!g=u3*A|O>u=E_=Z$JFljG%+EFHw#pr^8KcP_tyg^Upjz9AAbI}IA0l!Bi^m47O5~F2`jIiSpu>)Z zr6pqxR@T;wRE_MKz1xo$W<9=I<`4KGpH zx+lLAy^%YpHX$DVfHEXS61UHv?a*>`NW#Phn^HXCaR?-IGtsWhzu|^!gqs|XKHOW> zdMFgE>hh6a|Gn-~4p4>UQy&iEEJE!G4vLIAnN7JD5^`ooq8J()V2-8&mNFbsb#e43Mkhs`$(bjf9DqcieU4AEhSQi;m4U}I2_W_ndW}K3TC*+75M0P`1 z3gH-NkPo0cM&ZeS;0TnAaO3?B^@);RtWg2T^&NQ7Ydw|IzQZDU0D zt83S-6Btd(v1lxGa=dI}@&@dRMss*HGEPVG%da+t&ruRQ4Msh!O)tEV)xp3RCiE2g zkQ78(nhJA}y6H}M1v2jlJbav8GZQ}i7~;@tcyV+snS8*}2`KeEco z8_=MGi&K%G!;n@vq8Y_`V>7e0sJWm!@f~Ot-27rhlSKO(6$Xx@M=v<^Mk2A;Nxz2J zDp(N|qqoAwc@!2p=jcPPVH-hfpdbNcAj1KR_7YV21tA>aRjP-dWJsmu9J{@egw8Oh zLS*~^tj%!jXQ4;};F3FTCSuj!;e$nX!`n!Rh!Rlc5%n$`1f-N7%1oye6{#rw@1%7= zX(!SD_dxSnB9Rgm9qk5jX~H~B4gwIxmS7bITxc8KmGmNU35Q=5`oqN#QH=%{xbR~z zVr1+IxEKnWV^S#hR?TmNO7`+ajzCo5HJromKJDqGk|FlbGzk$-EI|VTo)}grPp|R? zB-=_oZcI)6$Y`ym%-i!>;G9d)IuQbjhT@8;GC(?3o12?M%YF@7hBCN{L>ssm4NE-J zJ0v{BH-RYY=)omjaHI!oH?F8%iBs{$1TX3Z|kTD2FiFqctw}v4U zfrIQ-f93H!YS$L!dbJa6Hdkaoh-@GysNPR?!u9Qn4$LsLO4R7Hx(qXz(n>%;E#CT9xq~E_P7}-qJt2|*xpFh7E z*0BS195)}IwM6#rYo6T(GzOoIkt=mu{%P?*gzT%c?9gg$SRT`IVkiBZ#zqDJK~=Bw z#7X|Ek~xMkuQOUWCi~fiuJ~^(=j_yQ0O+}qpKs+1lqY$;2P$2tZnMMA2Y=Ct_?yTl z(E-2@wBuKch=`DAVm2=l1h;rm;KbysV1zgkfr$Odi9t3AY{qhAO6USgA;EuZQwZi{ zV`Am++*yf^m%xLdlp3TLfVU$>9|2$+7Agv(99F~X&z_FuAID%;6aqt$)Yr*r<8xkJ zo05;MA6qi~KcTTKi`}!FoiM>%AFmi2XZ$5IGaodX)arFB3=Is_zqb{mWF-i2)fc0s zI+qzhOQg>AEOWNqfMD*O=FVvS@#dtzi@$p=y1%wtC1OgRl*)orh-vaJ-S@9}C4}xW zN!PJ&#}G@AT%Dvj!1Ku|D0n03LKW%yQjNe0^Fn0g+xF+@#R#2rmMJ_}+TgKbl9wt) z0>%~=0m|YtD}Wc4v$A@L4@{m!!#_GH={>0>@pApkl!A)k4Wgbu_eW`lSBx=6TUl0? z8qD8>o%R3wR^DY;X)<3+pxUHnc4Yw&Mn*a&l_5i8|e;J{H)P;yPb^kzA+G6=D_ z#P{9z@+NRW>?qS#h$pDTqa089=N+u#m0D6!V z5Q*}S-d-vS$-2Nb++Qc7E{0T<(wPf47I!_AHqK5M0K})*$~Fr_Ll3aCYv`lia}GI0 zGV87FmtYK0tD%MuQ&5^EXf4=hb;;{8S<2XiD0E8~Pr%k)RQ=&3vj-}b>zA6QX0HS2 zvNz$)DoNG_NZU84O3UFLl8W?Y2d(MrTcNhug&p`;I8;S-U>I7W+khe9`t?A5Cqj+u z`@pWAMNK}#_|hc`4z-sI=hNimN&)Ub`D?)Vn)>=awav)PG$6H|p!E;jCTfueu1-Ju z+wCKK9a2X*7-%L&rj1Q2u`GNK5iFD8D3q>?KYxDxF=-lIWBC&&lFItLO!tyAEYCuo z;|g--(oZ|+eM&=74!DJz&~od~DwQUJZ~WC!6o9&&&SuKisXRp$PGI9mpcgwM(KIiP*csh zx?P1uOIsT%GRJ<2Js}~+#s_Zd94I>kwcAI<#M}Zdi!?51{Jp(QseZG|XqW-Y4RI4m zyrd!hEv!6&{j4~4`Vw~MHe?oH`8NU>Gnw$hjj|n{W^# zB%-@3(~xJ0F*VeGk~^^^;%LoNI@NVK5wR<+RQ14}h(bi&kmm^6*XT~f_y3Qtvkt4O z>Hhvfl#&z$lm-Pvq!9rDMZyB4y9Jc)?iL9_S`Y-029fRtY3c6n?t0hJ`+lC^8`t}X z*To65=e7=R67f)pQ!Ba=BFhEwuhCNre?+3GV)~=C0$j`z!he!v+iQ&85 zeiQ27EM_54s~{uW4z&)RFjtenSpV+onktZ&25*LarA&^#G9`@oB=`XJBsd01<%JDW z{kP|z%pfO9Cg?&2Cl=EZ>u4CmX-@DG%x+$Vn9U=X)8dw4h#gJFcnKH8{@G(tF-97^ zM1xDpE69AKLL#ek^3Wbv>i$~`a19hy!lL_?;tngNT~-CdNa1cv;NdF4}TvHbw_L-kttZ&n8?P7OLGnFmf;C}nY|<~4v7LGIn70{ zF@Ic?I(;WBM|yn`-zRdD@oFl+U02_v8nWK~8c-R>rrt%Zxs9QaUr+=AIwtVv%Cwv|B- z{!_I4)DF6|7_0T)7>%OCFp0vV#8(Q<)@QyivZi7U^6@;)$p+&?b*I0!@e;r4!*i-c zTOG*dfC*tV^*y^TwMN_AE@Oouyy1n1|Mf+UFcm??J-SQpAvBz^{H_*;Uti##AP%f{ zJh0V8R(L}Lz%so18zG0ld|iG0W!RQDO|!rnfb(|^06`j9#9&MQOk7+J97bW`aF737 zCm;Z8pimr27a`*SA9()fgFlzlt^1RhK;re0i&MzYzex-~zh}7C*4E&m|8YrQ{QJKW zr6Jq%C#d*8-hyDn4ES9C7?r<1EVLO5s-()Dj-0&O)ci4C|KxG}{Qmno zP1)N^pz-^J^T(KoWSBsUTx3EtNlcKiDbb_4 zOTI5ByaATvd3FZ3_GY`!j$MqRz3Y{|igUzHQ1Ap#3SZX#;J_UH2<>Swp@6-$3`_ZyY^<*94^ zFg6NEo8i?w^1+OsUBge|nnlz+gm5+S(hk|LCfQi{-7zv&4|F*!L3T znXJYh9HS20sED@1Rs0S$EOf-Hvt2}KF=wr0hVLard5-YZWG;T}4z8HI4Rh&3l_bZ7 z#wdZBo9XE}+pXvBxrzsQ+$L9=zUGD#_>>m7J-f}g?=UW}p^JQa!d{oBe?^G@4MTe&Lc zEiOZMwjKV-#bFg?x(DoJd{t*y1U2rzI$rrEt62uba#6;iVTVQxm5!ngDcvMF8PX;@ zwx)HGBx@LV!$4H88CDisER%RRl~Xr&csM%}=Uc4@xFpdFiOAikV)_#NSi=-iEvez; zKQF-?mFpy)oG&i?c+m55=7bB`SAyJM?X8~1NoZ00g)F5Hxjsd_4|4Y3;|rRRuw(l9 z(^6AM*C$wX|M5mgRq!woz9%{#KPSY;>0UJHx7o5Z*2bTt#pXm|HG&HL|>c{Z-s`-FS4y6Pzm z4n<9NoYDK)hhg!N>I^CpyF66e4HSweQ2CmZ@;;Y#xOVV0=>oI2PJb<}U(oZK(m6^5 z!>ev*RtI!Wu0B>L&3mbS!iuHGOxU|zy6P{LmNVWm3GcQ*zbW$ckuTqhSsXN_5@uu^ zESutcyRy_8_R=8~5sWvsNplC2-zV6oFq`5Mv!hmDKHK5jW=8BUyf2j}{F^C-in6JX zi-W})TZxMujvrP=KOoBBmrRJ(K_9KK$7!Vx^!oCZ?Kt=|wl$9UXt5cZp`K)I@s$r# z=SvF+EK2Rrn8WM$Qi^HkT^M5Wtr(2zI?-R!WBbC*N^vMK`JX<$eB>}wlymNx60FT? zzlFv5*2vR$>R^*;ouDo2b2aeT5VFO_G>oT0ypBUqTxX!8IDz}D^H*a@2KAv;pTw#% z2dVKYDfQ+$YTKy;>GJZ*WE~3m*ED9IgTOMy9}ai|Hz)8YgPBihg%qtWeW6R9Qhxn# z*DC+_*8-dlRa}pYHHO&I>ljtlW>(fe#ZY@ni&tIodLoT#mf~I}Y+ne{$?qPLQinus z^~o<97}jPvs*o(cn}KUo9?h{%f4Qww6XxD+BdzhKx^)A$ZRKX7Yf)h_`96kN=Ls!8 z2dSVF&vOS80i{avTgwKEeQ9Rpc}G?H2^mSrZudmw-Zxjr1Uuh2VWeT?qeC@JqwiFo z{>*Zs)9j&+aAT_*sW#rlwL1}|-fa;Y;c;@-7h7uy+*0blf?$g6AvZ3)pxWVWk=78=p-xmN z%sIVguzYLK?v&;gDbe{k-e)%S9nc{T8_Bb|@}+^l_PDCECOeQ8p}DG0p(tQ~-6luL zouW?cshA|?h=L1dZYf&jh%1?%d>{&GHLuclp*h3st)4em6{UA+49Ufr3+^lIJ;-5*?pC+yj{LB`u;1Mn zMw6fSp85E#H}cR*$n}*iOSz-npzTBawGriKgzgB~s`@Vx#!-jN&ir~m zVlkq|T1I->*%?kZfp|>Q?CD-fr(WKMhEOd#!ms&?8O2}liIV^WQ5YD=rWK}BvXb!( ztjY_;N%2R;w&*5=0-7!2DM8%@u zK^6aj-8|@|^IG(q(U0%qCw*O+e~nf|!S#Z^z%i&fmU;D?#)K6c>tYSGt4}UDl;(%I zl%OG2e7 zipKN`zdw^`uJ_Sksevi2O_5AP!;8tHuJ>}Jk}?DX(am?LK9Hz%W-*`;qUg66UYuHs zl5M>s->|PVT%r7uUV!(S2Ne|`2kOfFtMI!DVNQl$DMnVeiXKsB9IQf0bEAxzD2 zmi#JoLp)d~@!sfzu9NkyncE0AW7<6JlSXW&(=lF49t_0f?gdMJj-iPTF2XMABg2@U zK`J`eRt8%@sqz%b{2t2!;+vs11Y}!{a(->~~kqvJ#67F zc9(C}S}*$k$T##MzUz%@nsnTNJ}#pEjp*EbYLAcL!|V$StQq^PoYyie<3;2{#~k0S zjNH!zU4zR z%2~(C)#0k_Zi<}c^f#*+e$ObzZytVmXC!B5SZqwFvpU8us;A-Nf};L?At6YXs#Z+fidlw_cR$RC01m5125o=58{dd4Q5P3h(vBW6L($;>yv{ua;-NJ`qHhE(9AMFf9_5;@)DG=~(&Y zf^p?a>cDR-1inKZ-kRh7r@NMd)m1Q3HE2ZRemLKIjFo-K}vwYJ>T{m zo%!PyIP4|!++L+Vwl$>9+bA9ozkVHq_2MEw&+Rw*;`oz&iK$6PHt6=fz;uDZqG!%u zf+%g{h1HzNV+ZF^jI$JUe2u>>w5rA1SJVzz?buv*unr^Ns23DZQ~V)2lbUveP8_PQ zT4J#im+LBD2p}E6Yc8P``bL?7TH4FeNBpW_ zP27q?SFwnpUtcf5?1~!G7rp_fuZijYg<}pafjn#iL_$wcc=su9f5Ld>Ko-1Mwmrao z<@HqIdyKd5^igoExkfP}&RzvO>|WO}-mP!v>-u4;jmwqGc+0{?Lh5>LxlxP|e(s~AD-W&j{;1uOZkK-j z8iDAx9!k~I(zrv3bZ;;}xo_MAdzY!c>_g0w$L7+nfr0T}3>+Dw!@X;jLuM$>s%zhR zjIx-pE@}(+f(w?9+!$Uumn|;#9u4if(_rLM6ig{PvskEMHo1WFwx{97eozN1>w}uf zv^;!E!RuAbN^ZYK+&6iDQciTnq1}*ULkswP2^<$Tvu;jdU!TdMI`ZHjc4G1;Vw&E8U0l8ob7TUmv~g}5Is*4AXD zm0Uqgu4=ea(cJz~YfMO>`6l55=F?J|NAqm*XBH%|p?bJ(P57aI%{x6EOB81jgYYST zz|Fxc2T9Xg?KxujN1K@1)7Q3E;`PnfUtpl^zeR_&$ka5`pUdE_KX1&fhbx3$FS*yJ zrG%~p#T_(xc?&*G0qFS8JAUu)oeX}h%taA({f$Y$7MyF;gEIQTzv(R1j!E$hy_@4_ z`eyh?8VnDF8?0#l&f!jAPEv2j(w$NqzjZ=?qc=DGVUFHx@+~XkF_U~v{#M7K&2!f! zsuW3EzT$^UQ$=bn#l+{>#KGT5|%NF4HQ-yF6w7wIXKcvxXeKTT7&wWe_) zn+iJ_PJJO)41^8&eq&9K3j&L{S|iVlNtx@us*C3M8}3=+>yw?XPG*z6a!a{MXjb)! z!M=<%tJ?VmoIfni8Jx~$P@GDC{t)~B{u@mfm?|5g(^ok6~fDLr2`V*Q1z*n^M2xZVZ#j{fu1V!6+vW-o?gL9vblVE_w2P?%J1fx9lJvD_#?e7j&0Q zo9Z&ZP{eR^h0|RLm0=G#enj}7Kj%D=h&0}3^M=aBd$oNzSQY82im(9swMMB+UpbE-ht%HN{Y?*Bf~?27JuWv z(S4I3l{e4&fP*Dh)kcp_d)+rND2Vx%BDarX*5Pnc=O;n}=L6i{-oCC>rK^jJk)sP? zv*4~IZj6hneRBBm!;7*diA-|bc^l|X=>t3-yD42K* z*w?V6i>o}jmD1YI6S+B|!On7Xd@Fow~DV;NW4@*zt3ysV93+XYyCS@ z?L@kDku*91Hd?wtTh^u4I8(n%9N&*XWfHC)&dLo}8V8YKxQ9o9T~{{lS7OCZvga9cH@7%mLPnUx!|?Bi@@y? zGFSYrGAO=4b7Pc5`x#G@?!3LdbA$X+7hhWco5{mySFO{ws`R|)G}6~d=R2!?j{Ji> z{0x#BRt~hbj-8PqV1qQEcOV`^iZnf+JVlE_PFWD1YKVb2nu|aRxV9<^`k#blF?}Qk zvlj7LjT%$luFqz|BS?+?H&$<^kZ;_vg)U{xvGG|Ag8K+K!iygAOm?ohT3kast^4?W z>h_g!XSXr2a`^R9C31%BztaM*_+Y=4iW9#U68gh4$5xwyQ2d_=s%m(G_~uJo^^;IfN(o9;tq$WyMAar{XZk_v=1IcAfF~OAA>vu z05b3d&>+C!{a@c?X2u1k;om@b_kYI0rT4-fP&f(sZwB7$FxFoD8*`V!gxh|MjP!+a zEyn-zBPb_@1XS&xF_re8Yk5G~FC5Aff-DiFFCead>pyXId?KR8yO2acCK!?VavB;& zls_@)fPY^*cv4~fiSkSTtwi_pTf_k#7-S7ZAeH;~2mmsZhAg9ik09v?rvpUc#NqWn zM+lG~F^-*Ad2UXi7F{P7O)Fe4@sU~BxW|i~$X@2Me%(ou0*;?xS?=wckCxj>bZPZs z{qGU{jn(Ob*Rgw;*??DmVX@MGAIWc%yp}zM8&`rB|M>vA45DJS<~cEE>oz-?Px z4kU5Pn|kzxu2R;DfvdwugL$efP~9K*Y5F(Ay*cibT*G+RKtJzA@dMK_w}tt=XD?rb zne3d}qn<043Asy!XMc;j*Aq`#RW~v6K;ikAzwC{Lwd@NnBbB2&(uDDH%5lPd#u*Ft zr^P34yo0Zn5_@Kybkgz!#+S*uzylD53IZ5p6%CTEv9(>6;oRA-11>YcRV-R3?pMQ! znm!N46G-W5vu~0gsXDqPXLL?)$=EbIV5Ztdd9cCnS%2MSUHS+%;c(Op&NINcNY~>k zm`F`CCy-jZEpLTr$xL|c=I{Ru@+u+M7J0M-5}&?);X#E!2-EEM33D}1s!Q}akac~A zVy}>h2sC;~?mZ)F!WI689fi&G#72Tb{$kO)0{s&H!9$#gdF!sc44nQPG7Q*d{lBvY z>)wKOpyGvlQLXqbWg#Ki+1ajbEQar{I!Z71bWJcokbgdF9HbeUlJzn#>_90{YIex`reU zYN8CyV}0L*ui5@Af-BzM8u8WPqFav}#Efm=HyOol7jR?VihI4$WR+iGTL? ztsk=SF6tcDm17LL+23{MZNh*sA7Z)nyg7QYw+5?+?qOY)Ek|_e|0naoXwg~6S`=ZC za|02(hgH|Iw?JUUM-hRuqTROn;BY%n9ildj{Egk?+_HQ|YU}Q+nb-kmMr^oer397q zLE5H|FF#<8y7Gjf+9R;4HB4}KB)E8KkEL?#0uLD;TxG*ExOMB$Z2!{gsR=NK%W>s$ zN$$t-=`?N^{LYj?|9|8za0;0Z9?OgJ&O2*7l(WfedNeyXU+cUsM0UI;c3FMfBifp9 z_a+x^_EWK`#eEI)rNJk+@6;zfzlV-=IdN`rg}i5(X|P;9a=9U7s@m`&?9Ct_Z&L8d z<%fHVGTA#cEP9Ay=XPQR3G$~OgSl@HDg7|r@QCP$>oVOHPEAd}gt*Pa^Kf@X)Q*zh z?OPKj>$~6hS$x(Fy>~JalCx-w#_pwD)NIA6O#Uw6Yi!ey&5oiRE%S_Unv7KXW59Q(Nj_($-GL8T51QMbu_VrrP`~eJ-mYVdVb%fiTAf*>X zKYPUXYLCgA2&@iJpa^fO_a_@sCmGs(@jdOPtivO!Qx7ax)2i{=_coAb*HDeE-R;#>GW_g&-a&KB*WqI(iJ*S9C0KLdO z{GiB3CWDS-sPiUlHk0>BweYU}a5*ve60N6!fLmeUEzSMG%Ejs1CvjBEU4*L-?Fa@u zX_0}^JWm{Cn_w6F>l1BrU63cS*|atSBK$RN-EBh^CoWwEe36E+{e6 zANTfK#jnStj`y%P76P&*_PdV-HBK}*3rr9x^5UBdf&#@9WSF$5 zx*UC*tFN`4pJMFWjn{fft^F3UPOEfXF*rfbxvi`HxKngV@ zR-MGF=M^Mq$e=z!oY$J5i(=A;Z}tni{Qwq2xq6d?uXcU4 z{NmUc5oF3VhlX9vxliAP*DESB$@0~Y;)xhc%Vfq*2SuHHecRm)E`~ilhR~BIx!cGc zvEz>Nlz<%7@j7|=Gx5d(SII>6r}wPLImxD;6q%h*#{aTAy7a;I1QqAj$fVXG|Dz9% zmn~Q5SPkFcAc`p`gCpPF?JiB)7=N<;$?me+mQUdq0cGn|taw7#nGP-uWPtjKz_{6J z7y9>kqRoS2;+IW0bvE%-ezI8T_0J`*O>TW9RsVMTYtv@e@BY50)w(R1F%Qrv?<;xN z$moY3mH+y1R2YvJAwhy`tj2Fe`skX2j}ON7UZbhT8M-8810*17x$uja)G%^rPr7uX z_-+-w^`^(uf!!%3G6J!J8-G7n9N8G(Bg!P}lxz|wUIUS0kL87KQNq7`hmTQ1nmk+)SB`N^ML zU)RLrm#^ffnU$5XR@OVk8qI%W8 zI>+^k$Zk@KVo|D@sw?g7YaErsSZc=N#nz@*QTdHkuVOWESCNU1etN?c#s1o)Y6@aW zd6CARmDvw`_SPV}Z4_H}%R^FOtv;J8udA_=07129&-30%&Z6cxB>;&nr zw$NuY49!JV6pe!FMXaeQv@u){#6~~eCgZgvhv>xC#9n+$;L5vhONanmm5xu*c`Zo~ zdAjKrk4Xg7atW`d;4~gmdr!z^ql-+MydkrpsPx0lGGhH%b@lX8-PX+dC%dqZesh*$ za#MZOi7|f7)!ehswtpOGA_Nw7g4G1eUKVk`d*fxVLGjd1mWIfsn;fECn>(%;!S?)J z-*r*JNr2Cwz@J1xqWy{Iu>-=5MZj9*LTY5I9dBVupbf=F{1pNr#nybN)ko#@AcGX4 z**ER7$bV|PdfG#o_ua}X>1mxs136K&p9}xzTFVnV&OEOB2P;ALVg4ePj+%SjC{UmT zGyZqw0i9|wnbdkIf{N!3VQbam2VITw(hm(K8X{4>PJ`)f5KfmZ5)~ydYP=8N#{kmq&M|F%!3rqy7(J|Te z*@YPwH$l<1nBd0txrvaEF^AYQ7kQVH=1c?_;COy!+fIF|C7`8;fjCoK`r?5ldXnAC9&K zeZQtNHuk?n>Au_ttzoNBAw>q+J`v|U&c2OKd?A2Hp>R&N<-hdUIYV@?P@8R5Tdyd0af%J5K&~ zeh}qx^FV3lLKahw+PM8aLsDdplgk>%=#dM-cbuyyDOw!!nqO3k*yHgMhj0Hk&xh5- z=;6k;<62)Z8QR47$+s*q;ql|mGvBCVElEzpwoWxhKByH39KRYIzP@=Uxw77;fv_vZ za;n&6K1v|!XzxMn7G~`swOUtc^bH9OMzWmv;YyOyVegr}G+M`6<6+xkt21VzS2q>4 ztE8wf6pQvqdVA{!OZ14CF5>+@KdQa)RNon|^OFKpdgVt(M{yOh=moCb3p8H!h&SV6 zePyy5Itp-U+{Z=x``u+{EBU0^pkt{Nei4tksYr9UJ0X|M z%l1sNchO6o7H4PLDBHEb15BHy_UuEG>h3C&KVuOOE?JbT-`w&RTE2eJzo^Ls3sh;Ii|?y-5Gf zH?8gwA97djR25jz-g`|rRaZn^me1yqNOG9l6M2A;k=R^B(6FaYSY?Pcq<8^S*Z=9b)<>K_LGnuBH;YAV344t49e9+q# z;9@G-eWB4qa#O0p9S;*vr^N9@&%IFhjnS)|fr;LWktB!vh-FSg-PeyZ#=h|&xP@2{ zo0~4!We+kNGbH$(b1|!h4gF0go_tIG{=>Dkk)F5mVwr6fXZ~Wp=*i21{NC%CMn=#5 zT31hbY5DU+=Auy@G58!`RH+U;slyZ?4RZ8;fi^N)xvwBV${mW9#kYB$7>^~3akCbu`Sp+y_$^A*KMkelmKD*7WNs$0Ky!rGj$iFJh!uP;N7g{07M39~wUxXB1?KJlU@pO4wB`S zaY-ECgCAME(KSSWEbZhZkulp+(||*`R^{wqa8aZtM$GLw_w#uE?-AV{e`53 z9$AW38T>Gk|GK=qKc0Ah8VI`%pDiyY*DqXJT?by!CcHe|-%XM^eXVtcz(y zYv+#Mrp>^Eu!GDOYv1p`8~jeYr(D#dk6vuC&Z+Tg@Iz>byN9;f>XG$|Ym;1anP{Os z3og46%J!n8+Q*%Uc%9DYkPbbq+L;b{+S-LBm!bQau0F52tAlZ*z&lWGzCc7sx+X`? z>sLD*c!jeoiGUn35F<88}`&9?wH(K9R zU1UmA@0@d-x;8O**||JxttNX#G4kc57#*9g6FO6yxKgo|pxVj(uX;Nrqd{#YB28n9 zZJ!Tc+#vhe{!-#ODl5b7(lWh^+U8rSGhK-je5@cQk#;(R>yD!;Wf^ih<*b_4#&P!t{HR#T3HkeFIV9crbX6{Uci*MG>op#aDVleAy5}Q+*1AW* zAGBdZk=*&2@OQMz=7K+Ev_L*f65oS6bd1tfs-vDK`RnVFW$Mm6Kb`M^F=ICSKIj-t zToz<9#Ii`a_jedf=(x(d9Bgg63aapoQN6V;xd_0Nika&>a(N;s=>4k|`u-g#V<;Q1 z3tA?=X!Lj3mS!0NY|$t9*^Ytnb`mN@NeJ_L_1y{xzuzCCKw=Gk`gNTI3e zp!qXbl$6BwaqO+}6x7c|RZT0-eA?f8el7=}1)Nt=F7hnPtT)a&T}1Z@rl<9j5bD%L z#uZ#)DNM^@*++Re_jUwi|d!hAGQvJ0L%b_IS?!f(VO6t@(T;GGjmovAOT` zQ_Pt%xoh^cfyCl2yKnLbh5nMw^yx6;{F&WdY>LJ^I$9<#WUU^3`hMY_mvrN+FTJ+$ zp6Q45n>!aS>Da_VdGWsd=o~70ve~nWH0=J})OqpktEhg$Ui?b&pGDfUobDh-?}3TB zWFhLn`+m!Ov8~-wk>i45w5;`~TZ!|`hitLQE>I9(3N$9;j;{KZQPxe7s~ou;pY$5`5s{w6QpPtw z@24%GrVr}=slwuEKQeIc9~$z+@#35(`}4$31oUu>Gue3M4L1(-gRb79j2erOsezqQ zU!d}h!kHrES=Mpq1GR!Aze~lpzMPewAC_Dc^xIle%|9WFXBqRV$htvFELq!qIQ>ZBg8;3lmYj zaaVc9mDBnoZ~Wv>`PfeN`(u$ma<-|fGWoy8&(CYOdvzHj{7T|K&HxrYvZ1$kFhQf5 ziSIzYac~f0!e;#V9Wjg866O7l?HLwd6uYlBQ&whbY2~O2rlYi6Txx=AtKNL7rP7;C zAFqB+@g?jCiZB{->7t^n&g?AG(U^OROJ3GdK+5*!4a4rTa%;jb8`^0B`i|?Ft_5Oo zWp=)n)VA2)*9I+b28uq7+#kO~lf*!-+9Ee8afs91BfZy?tx;axo*{XWYAj`zGwYt^ zN!WBTWb*N*W}iprqQZdZ2l`Ke&<_qduk7ksStR@ofKWf$eh^p7Y_}m>T6!nhs|^xP!ruX4LU7}TNF?*n z2bRRyQl~TA$Q)Aj0ZIXDO!!S7(E5sR-hAql_`6cc*i&e7t?Zx#KjL7|AuD~ZE82j^ zYFoO{*=17+U#!E6UvB>?{nK~j`dF^3yxH;dZ%o_ovFPttQIJNSr7^o<>BjyjD$t(D z#qaLM-jJ)r8<*G-ka=jiHKBp89_TlHEM9lgxi^A~@;b|6D1{e|T6l@Jp<0adAkFK; zryD(jV~rgZk@onX>$^TpmGj%9B3RCI`ZOB0qpPToO@>f#DXX5`GXKZ(7oLXB72u}`5*rPvkZsh@A;pc87E$7NM*MolE>1i2o5yBTLDV%eD zzWu$~ouJEv$J-J@IE8FQMCQY`)-eN_G9Sl~H*NPnFpQOJynU=}fE6F?>e++7|AT9h zUay?UFM03N9`Ta&gS?5>`&CS|yS-aYOloFy$}(sbC)Akcc+m&@E2;vPp(+ElidBcC z9#t_-SMLwMu6QyYoF~9+In<&arKI>%e})(LtRnie0?W?9a(MYjEAiEPhN06l?=i}R zVM!U}<=t5!j<-0jrnpAYXtA`5Ss8yWfCwD6X)$+QJ3EodlajG9R%{O;AK8xU>1d8# z&k34c_Re$SvuBEwpwvGZ);O|iV+{xpUu4&O_U{n1>fiKm~|?IV8W?=DHN(R-jLjKGWR6 zn%F~92TzeI6CK#ZSiO}e&3Umd%edg-9UXMPO!CLdZ=URl@>ZWJTn@8S1x2OBXRlsN z8Q9J59Vf+;2HEsuuXcrgP%z!<;xuI+Fa_?v~F`u3biLH@ZI33QtpOZ~zu~j6^K{&Kgd+wA=OIR9A!|fiF2P*^Q zTQ6iPqlW4<1=m#UrTfQ-E~s1gg0~Cbse79)kO&m+b2e6V#hi&{=UTLzBllEFwkl}* z9b|Vc_lfzij0GhP-fHSxyC!3AJ1puk5P!M4zTQvET>x6ZlXvA~c+ee<{rNt&8Y{|8 zS_JMsk<-4pEz|v1H%}XLdJWhjdq?7DOXFk5@3}p3zV$IIw()n7$Mcu1PW2oKV&oCd zJw?CR=OaYIO!$2R=WBE8R)>GYsFBLgEY4?Bc3JOvbY3m5AXd(=DN}!Xerk_770Jv# z6FGs!$4-87IW#t`x?4$RDDUo$cF*9TmxMF=wTI`PM&z#qpU)QwoW-!v1Wujd!5w9YS8G^ zcvvt!(yVi&H{#+cZ)cB5b|S!lWmt9M-+nLhlIF&A%OOi82T2V#{SlApe1!awz+TNnoJf=jqB=6OCaAVtKqld z5Mj*k3|Md^k`^%QYYd5Qr3HqMUl`n?bRl@<;4p9PoI+$B&GulYbi7-&X}8F6XKJCQ zoW~gxQP@vZ&R5Pi&nrXcx_^(QV(V@bcU$w8rOs}32SatRq0P_bPVlYt(Mk1!H%wVW zH$MH~9AI-iMbLF#cz@lv|66h(ItJEQVrMqi6x9k|F1ous-kmO^zaqw>4;TU6T|V0E z%Z+Z$R{UirZRI`q;xl^8v@@<}#1GQdyqlBMUD8j5{JNhfJrW6S1z`t7rgFjZ7GL*U zjAMb1W z#J;}f&18b)PgI?-3Np&1Dl^pzbrU!4n4MZ(UeA8eyJ^!!DYjd2S)1baZQ~{j=r^=fZ#MZBzo!T9&+{w4g z)jz+KF&;mr`vuz z@5^=ol$=|R`U)+M_A8yOby1G>4dYD}+VdEz&F|lhSH6J?HfDJ*bT{LzMjkR zl<^W|ex|M6=4aIYqP-Q0Ol{U^!RY>>IT8a~N5`iOOh_0pXvM%;$^ekAnJa$fzTt|s zd1PxaGwR_>gf>0>so3m}2jiStZc)3-nbn2XC1m%vXW((3Fxmc{{55wPKw1=i@1r%SkrjpwN8JNhl9F;KCu=J^mLX} z7MC%fH`~csG_^Nxswc};;iV}RT)KL0N|I~-J{G6K{X zQ%B0J3#~KP$bS@G*uK6KSHd&CcWYWLN6RHi&|043>eaJ}tBg525qGMZaCxc*j)u_P zJ~J8|rcx zTdSeknwWis-ujghmbUl5d>9Fh5W|q#UFI~7&Y2@q?Y+pdc-I2h-^4e{Wj6QWyftHp zWH)e_@x7n>yEB^E7SuFTZ$vnt@GoDNi8;Od&fW`OB&S3pxQRtpC=6uSF{odB@kGb7 z0GgJz<839s>DgIL&x-aQ{=0Nuc7>o~btT0%SccC7${}kfx;TJ2;dHn|NVkKFpXt1 z6+-~^k@~ku^4;uz1;sJ8RFOb;!LwaLqy})ku*rM4>LZPBx(WY0krMX5VyAEjS@!R} zS7!$_|2!>dnW&xJLtrdu0l656ujzlZ*%)m8d-YK`K28I|PFlbV`umGzsqg<)uZCaj z4PZ3TAFcXfGR6xBYcGp?yKrUJTydJsi)T zhp*sKf3XEzT>07lNWSUB7eJb&lGBqO#a^Q^l{(NixA`H(=i?6B-2z&SEn)uxl82Lf z;n;Q+4-X)iBH!kC!Zlx!WrIa6{EqY8I#yH${OL6eEa-Zm79*j_!%`N#o5{}&#NZ)p z#WECXBY|lj3>Ad@Xpov|ef#(0DLV^+%76q1!14gYKuUOaC&^3|klJt!LQsf;-<1h{ z;V+`)>;kNhV9=XE0$l*%B+oFy!~}h5_gx^v>nrjZ0D_^#Q56HA4L1Ra2lRB*1kex$ zx^4_m6lRu}(=B+ZUqou(LyM^Yeh7=$69K_5&cJ32J5aj*UVm<38NLQ2xEP?fBZWm@ zz4|tyPmMJl$UXizrDto(molCF&S&7_J>rO$qbZx6Z8I^M_+ z4#WsZNI!@6O+?`*Y6AMQ3Fz%e9FNp>D62z3@_2zA^YukIBj~ts081Z2xTzM()GwI- z(S-|p4c!S;@OV&~+yqvLmcD)pXd%7`hYBk=C|?Jx7^FrLDAy(8BT0LBcz8e>-s?G6 zApz8pF7Pbj!Y{R6`>Fmj6%)N~3A#K2G9T~)aDd7KW7ALi&i(s=P{_k5n_AsO>5A+g zVd}0WE2I`^KtQb6wsSt~q3gwgD{z>Y2Ijo&DE?6yOuVdI>BNK7iiN@c73h2hO$3R( zNI`OdDc*kgkOb%|1t5HgB?W@VBtTMC&U(-}2RLjqE34Zeg+WVu*A*i~1HQfqPS7Kjut9RT6yC!ZsqgL%1q316K$B}= zKmbyL64(&T^KXR0oWMMp=TSZSw-x|7wHcm_+b@tq%^T2P0AMH!s$F4>NGcJ?J_5t; zCm`_tG*A)hu`sc*+xClpfLavLTaX&Y(4z2eOeu?hpPd&{oC<&wZ$LN($YiO1I)aYQ z2b!s)rBwp}z2qq7ddq2$L}~&zZh}4mt~J*Zt^;WNQ&1lVUJzh4Oj2W|{X!dN-*Nu{ z@fILts*@;r;U|xR0)1Z)XssmoLq@~rcSbW zLe~k=(b17w>YxCH0t%_~z}-#);s+42{Q#TV1t?0uu|0+{BPVfBxOr2yp&BWP(6fYf z?*o2fSYB@KSCH&7Xo3KfG(A6m957j=E1m2C-joBQ=|wm|zSW(!M4;MhsM47aL_S{g zh2kehUAcT2N!>Y*=)m{p$vSq}Pz zGs?u&v=eLtQZ5WYAF3r5-his3gjGUXn!5L(Hy9JdIN+zy6GOEk5ST#4%fIqY-42q3 z3WOYqoR;-Lg_p`@o|?E zV;FIYAcaClCw+~elm4ig7&#bUzfWWUwa75AoJe8if|={3pcstI0RiR!*cyN!xr(&A zAUp!Z7YKE*DUR3HUIM-;=#GFC0hmB7BO|aD;z1*l5)yRO)HRbE$ln7xEx_d_8|Zq= zIXE<=1vH|-KpZe_Q0E8m84o}if$lNrLQR1_2|$~WFZlRcada>N>-+>HS75!SzjMbM zz^rUQ5PdgA>-8&=d)&?w8oCY3nkC3Lz{>k+#Wn!Hlf5Jzx!!xdZU#UQQb=>K$dpn! zOMdO4qPYwsu&>j2G5i4C zzik*nJ%GqVDwKgmc>-j;zMwn8xuJh0YrU{21e|_>^|xOu7(f!7`8o(eOB5RV6MRT` z9>BFDfe{M}DnMIE$6YU+VK#`vd}rh0y5DaEMr05a08LFVgjXhD*@OuNy$X!=X$aRJ z3uys;3Y4CJI?U(g>x-29(&z*R2lB^+-Sq^Jd`$+>Oo1>Z1^RCw5Sm;tr2{m>AEpxm zaBrkU>vO2*2B{3Bs69OR401^fDL4qz>j#g&2f#+4^J4uj50^(l=`Boaw!-qiP{Fw8( z4rUA(13*&ZLaFkxAmCiuLBs(m(hIL)N4tV@;^0Qsm*k~DC*NPCzkHJPDoe> z?!_U@4dg$+fPngukK9U09Lz{M`Aa`->VL$* zwE+y<2wb~%phC%~`;!8=Mi*ernd%iWAma(TOs|oO%YXvy4f@UqCCs4g0m|GRmL>Jz z2ZGKvtPt4hVD*uz=zxJShYI1uq$Hev(Esf^&{&+G|GoiLQ7$-RqoaV)WNvwSUlx#) zKj9{q!~;nbd@QQq5~7bl1eQye{R*yyJ#%Di zY#UIHag=YuFuHd0W;4(uA3E%&o_Ebb$+xFGT6z&&$ULDc8l~Fg!fm4GQDPPTtz80iwCS{z6Iz42&cARt*S4 zk+5lBhOLPPz<8Qs?zB!M0Al)yN1?BDem6~lZ3lfN6r^q~&~=dtc7kp#Nwhrr zxxickx-cOjq4T}E%#b)s_$!IB{RphUx8!796}pT)@FH)#gQpM1A4w(zu|3eqdh`1R zwO2YA#H01H4M26>0U;IGbVtN!3}SBG{kA;~6?TPR zw03mh#!2`B8Vc+zuy&C~7qDb5ySnd07`+&lIB($qE3g5S2w~jDL7xGLFU5w5+TY9R z-k+uF`X``4ZG!Xm!+b`xAZZo}5wuzxz5zN`fFx(6WfJg&odl?z*MYhQQrB225vlfb z@7}#}ryX&yB;fiK4u^O#SpBmrW1s`R^ny3BdICrc7>G6Z0W=gYN()Md+8~1m%UbFm z1$hQ})AB7e-nbwFTG5Z8p|$V~e!t~3Xu6j8`(KZf1hjQRz$^a&y7Jc?pThip0phOd6H3zkk1m^A#X@nE)!=Rv@W3(Ii7M!teq)&A>CVX9mG_ zr9{#EQBXJoYWnW_n93SMg8|$Q3>=8srNF*dm~90VKf#oP4+LVr^9FM(aq1o}1F4ggv^00VtE>4|r^ zm=ugX4<;xGhBKIOWa+=Tg#~+*AZ(%l=l>Fx5d@Nr2et+zM=@aqfWZTTl)51G25Mbl z;1~+-cg=Yi>41bcaPZ>bI>d$-_CQ|NuJLtF1*#^8=FzWQ2?6#SxD~EIq=dsAXgebXhUg!4UKaf?y8Z$z%QgD~#$OWy zMHEHSBPxxQbQvguNGl?xAfOV!TZR4zJJM-N`U^)d1sc?HwF2 zhO7c1?6dQVAZDWXVIsEy1{dco4<|Y>UOJPN>bPJQCm%{o2%ok~+tn;KkynHaZRNC$ zj4aWAr_UFQ$GeXSRf8G3&(5tGts4MkKYwt6Z(~0|4mhAQGw8SV~BN%hF@!x;QwBb2(NGpf!w-Lm|>U- zw;<3I(ZTr!I?9I&a@F>M%?Sd(^wEAw5A{YV=86+b>i}%U2p&nAQWOjT*8ls= z^@>X1V+%XGMycVI1pNI?q1)JCZb>|B+*>wGUeKwuEq_CjJz~z|BStb~1N1i}X<`t! zoRlkq)9q8VYXyj?6klM-+>~t{Ur_e41eRXJcuq1hcP-!ezgiQne`3ez+tY@-^FIVV zhU3()M0H<)hI-f#5iWs1a6=7AbO3tUE}HO|7D8SB6~!&asp>e632<_9hMc;yKP7+u zdNR%KVN5;5yvtLqvj|_95*WiR9)&s=VYm*k03K#%5MkJK3&f??7mwlP{Y=S1-vPoi zmIL)+Q5#l*PC0)3F^Z~m;6JU}O5AAkCzD_oyyMZ=K#dVnu2Dq7qmW!AVdnVi+S;CF zpDOcY{3~8yPQoNQKH9}AI{FB~)B!SOM}@bgjl=2>+jSDJtb1gn5-z^jpxZGezc+uO z-SNG1`)A8Xcx41g0Uo#3R@gChmloa~L@kcT?SzF29;3S)S@Jb8=o$}EckJ++t)xKM zyMEH&@e+gT*_QG-*-WF4Af!Ff11AGu@s8ny3`%oVB-Cp9fw%GTL;-Qd_*cT~=y}8% zHaEU|!)lKV;6m}9w0(kb&lKC{YDt-5)8miA7N@1SL_{=*9zZ=~hR@K>9w%e438Z_3h&cWeT8MdGR}-&*v@o-HPCUuYgIO;F zBGTYHs0I2-PG0`G%XK}yH{Ijo$x^{fe~bV=0ug!GaYx>CLRax^-N?CUOzjJkRqkE6 ze2cwoA)Hm4h#BW&>R2(7xw3Kk?bz~p@%o2v|3lqMQ!b;*gbj>=C&PJRU~)oB5jMxk zLR=dUti#qaV0`-exA77n^`4;W`?LIs1~=s8$qD%d7>tDUgMIVboXdW+iI)I&T)Q@9 zT<601Ln2m#AtQ$!!unzl1qNEnW7T5p)r@_VEgk9reUx;kF!38EC~G#DQAwDp z%gOCRT>h0^;c2l@MB3UK5)@65w-YoL01 zNvFezQHVnB3h^9D%twpU`>@mvFn%2m;yF@XB3P}>%7u-${(}peFlQHOCZ&bfii(cz zl9pD)8Qy`ypp)9WzJR78DuB6qZ7nTWrt%SK0F2`D5P;eM=ItJXxi>D%KPt*>^7C87 zKZGXOL`Z77KF|QdL3V{d<0gxN8U`15e8O7-V91e}%L;RrFGWSi8hh@Cnf}KB7e{On zh*?^fHsg!ZT*88b)iVnYA@qluX18I?uRE};xNY{Tm8IoZpynH47~4<6Us^8B#cPT* z{R1pK;3x&Qk_0YVv^DR8%qQ3rgiL7R-Ri@9+iCJE8Bva*kgm@()r2L;^gDZKMoXVxdhzEKHl?DFJ449L2 zm`XgFkUlEMcZY@*_4f|t$EcwZVe{}j14BY~+{$dZ{|ix7qtHbHxQt+65Oda0O=*&v zx9?vvFMI&zm$;`_*M7li0-O~R1q)%El=%B6asUZ+m<9wTEOx=BIB+1?#y!Xe#JE*t z^fXNSr}vHL$)sdp*yE-Ue=hMiZ60RD)9~DID`+AcTuEHvM#1cMw8xI7)zqj+h=$JZo#F9J?S`V%ARI+0`3)s`N3D}?kOr)Wex3kkd-&wZHpJmrJZE?w z2ef9HQ?qs;#>-_}^%2k)r-0ugr9;yRL_VeC4DN9LUR*p#CY24ZDRAVRe?K$Xm^vck zIypNJVF3WV6KvxW(?9h;6O0mK%;(hhi-?KE)DL0X)Cx`@&75Fk^TZ?bBUYH4oZPps zUmrlt!ZVs3$uMm_Wi*3tz)FN|&?Xp$d;=`o!zk;b z*5Hu?MogrJ`?;xFvKCF*-Khm|xpaA`@Co|iG9J)erIhH4U0<*eHo<72MnLAw?Ned& zGAw&{?qr+HO!KlsF;FFFj;*%b@X9yf_j@Zukmh~^iSCGdCudMYJ^`3geW!9FPBFR$ z5gjL@ZvYUX;7UCmIV*GiJTF`oLH7}27ML8m5W)rsZ#X^> zn$hyZgnivhhHwT)^(BuoZ>TMsAN3j-;_lRc3t;Ique*w^yAl(CF^Livp((9eGslJv zMWCCT+ua#NL+6#xR}m zekbKke%mcF9`lM|M({BAOhl*Jn+owDAPm#`O<;7s>jTcTV{_R_MR@JY%-WQQzciLEYCarZI-GC8MV)Vvs z*9W>TODFKKQPS{1G7>azufhf*wk^snB77l**TE!7dh6Rt5}9 z8zZ)L!Hpg1DkX1v7lDFNro`h1!5ba2*MzGDv5J65y%0ey;F*9htN`y69$sEBV~c|h z4r-v%39k^;QAVicAmltni4HfR3dBlK^Tm~w939z6Xg*+;!9W<{p1yxXv-@b2Cj1$^3JVKcc%^reSfSfN$zb2TYnRHjqlQYD zq!fW}NpjNCkD-8mf^G9QC7`m{=y?!D>~3j*YFdq#Fx5CBB7zDjy$>BI%R|s+)JSj7 zKG`LbYzKj2n-WTlmw|zScV`yxX{T=gIK7En2ZIS}Bi;2^5x{V+oX?XP>f6W1wlY?neARI_JK<6zRv}W9i0hcXQ`y7!|jbs2l z$yXu@1S<&DlgnelT$)!yPu=n7Lnzp$M8M1Q%7NH!?AYIxnd`8vO(0dE9t8Fb%*Z%p z0HYHx*nGK!qfn58DQMKdTWEz|uvm2og_TWTQdWC~ZIle=eAlYI&;oE$1)R|T$OX`9 z07=pzDmo%dJ-l=u!Fu|YX2XOo1Pe`B}VTjT}?zGr30?M znm5>~aOZipriE~O!O`Y)834fwtTJ<+{8QS69!5GJFDS1!>Zj~GOBM5>v<4#7TdV!A z1<}H)r1av<5v}ee!uTDfWLd3&?49LRVW&kNY_5K=6(V~xl=dTzsGv?nrM%evjQf|* zmRoySEs=Z9o;yddiKvC*@tN6~DxA+vQCA*7ta?-`cLdx$yhJ>oJtIW1=Iswbxe$jV z2;LR`M8r~uS??v_)2LC5(`0A&g54Ehb-3x)9u`bmuZ4R2F*Ox-E*zzlR@S22zFyCt z_acG}+K2)##2h)XD0}?)aSJd`giA?4zyS!C-%3hO=#EH+35be@p$6|6A6LK^%noqk zpF=_pBR1~Nt9|IS4$SNJ(7Djls1)=yVM2%pUcaEP8J4VD%y0hEvt=iN6^=$-5slot zXlL`=|6<~AB5+*rWvm83+C6*QNEsCYfUU%ppoeBL5DJ>(FoSe1bgv>$Xzx=KewSEgZx5h)a3sXj24RUf@@A> zh*yJNZ~hI*8+saOOP8we`H`U;lnD5EDPb_yyEWEy~j+ZEX=?U2kBFKMG_Z5kluh-wy#o zZ;mhr7LK7HH6A!stqajsah6KkC!lqJ9A%N(}x!azHQlsTZfwSZK`ieOi z>t8-6c9o+hAg&)eNG}TbWpy>Rz%zBCfKUijf2DVDAlQzMjlq5!>gFbOb@jZwJc4~d zrZi^of!MzbNuT+0m<2*3Q37H2w)hoW%Y$AduH<}KdwUduR2uwKRaD8hvT9~>irJ37 zf^TVtNuv)OZRqfspY%PQRH$GM#{-+1ub?3bnq>VRYJfE>d=a5h!b}yJ1h=v5fd7J0 zxFvL3F*YaU#A>vdczAktAzG6NQ;(q`S?sR_oqSP*-naN-q?m&fXLcWz-f3iLD8dp$ z`nMP0yQ0+EW0W8rE-PGw6^)p`y~2MN(bDpA8gxRc^Jik}(0$W^$P>@KBNH=ZP!FlT zl{<*5mSOe!hRYSPpZp8u$&W(ll)Qq1-(h?O)lA2}Ya6JJsoB|BtGp#>Tj#Kmwjz zhTk6VL_-iGR?Vklo&zubU2WH3;QmjR1;mBwx=khXt_BoE8Q9`M)#*YJ8jsI5F1?>z zQd(-gx@b8+{-=__!7$eXadrFCb-Y*@&TRsZ_XhR^Ae1OzJHk|s-T=a!7LQRNgb{Z4 zP$rka$Sfjd5nHNJvdl~%H|Hf9Acdw*h*uW`^gBSAK}{E?l>WVP(+y4fKn!U zEUsy4@}n9;qX3MBHxpzsV0GSHgC@rNyeZrDzyU~;8JC1@Zwa`E^05-UQ4F%V~a@HVQVBs9l_yG4%wvk&!nAVw03F1AFAa+C)o2SkbE zJUsglyHMkty>Q`aOKOzt(9jV08n=y=!i^Zh$hft&2XqxUl?2#cp)KGqCZ>lV6Vt5v z?%aA1}05j2_EGn+yVD%~9kV0LRn>OHnOku}s$9rcQU zCJn4QX;#91t~r<4hGI2AqKd4RZ508)?@ zHw+A}7<)Y+p}Hhmh*05y@G1qri!V5dcxEiak$O_=^DFLkto~n)bl z->Q51HRN)BfIo!U8W^cUqNE}OB3L%X;Wms*54<@II*Eqw)gpz(_;8?6-!sh9`@-LsOOa#J5IX#V*7^0Yk?Hh5S zA2Tv~pK%uuJ%%Xff)FpUKUUffZr-S@tMdav=>y0y|gb}*bYeEjdBUw z=n(P_VbiD(C0-6Lpc!3O*d+#l+XV1I#hu8IUHtA!;vtlUeem=oe!s-~o&RSxQZAes*+pl;|RYdRp@B+e7dc^=MZCQ66iX!E}(4@++#q zizwDCz)gFkWGw1siS-h1EM z%TRQny1vEQLp3z8==vA>cmQA$UL;%OqrNA!N+9|D2M<1hl!LcoDi7>q;znU9iPk2pgEM&KH2(p2CGU~X&>Yn(%q<`wpu4tfmvRdn z|J)#O;zT(Dy2!}@h?hvi7cX99?YB8!1n~qh?IUVNs`Iab1l1to!IV1aed+5It?X~u z3RQIGd1sQYe?mv2K1h6GpJP9HBI%&VtQrZ92J|iYedRLHc8lnS)X1~vCH5BHBU+M( z6(bJr6cSA!{QwxPdK3YA3diMkgZ3bznaffiHG$i(=#4`D&ihmx;EdsyXnl8WpuhhD zKI_WG6Vi3p@wwouH}h>!^fH4|56xX%n42>Ic}TzwSa#F4~iZd2WCzYtO;@*l__SJ?STVPM`QeQ!PezV1`JBFb#Hw~^| zO^yiV5*n9KQbR6bag6FX(kV+dM=#C&Wnj&=XMV}PW+f+iU|WRUK%PBCK=~~tc0aPx zqM}Ej8K%&Q3lYGw^xY&()zLVxdNkfq7>{%bH6;mB&{qpE=P;f<5CU-% zTi&_zhzptw@c3@R8JOsIg?(4o#Ds5fFg2J5q8$KbHlfBL@MYVM(L!42CPD;IX>V=x z#)JNVDv71u0HXRp8v&ulV7(dI9QTtCm5-C(n0!aGPcMnPVDTYfE|?W#8>;e{QSZHg z6TVUCb3mIyaMz=V_P;7)h%N^Z5pDxfjy?xDuC`@*UPMy`ScG2Wy^8cFmH_cBnWmRe zLZ z#D%UKxn^?5{138kM@4=V9l=HDk-1BAbZ}xq5rJ!u!ink5gKR2~QTNXvOius2#Em28 z{x@SsQNT3=`$5k?#nV@WzH1q9;anp|CzzS7fVwc6_X|)A0aJ-y4%E>aI+tKA_>k(P z)?rx65_KfD4muPg`YKV#i&6l7YvmJ-8=tZoy z5_Wb`&hr}j=VX(o`@_wr0UTzUjiQN=8`E=7-{}wQo1ENS&A1-4 z zWc29W!3Cqry&$u{yt7}xABMMbXzC(YIG$Y%hV3EhYIx(mu2VkltIVq;ElqB%Z}%HZ z?Y392cp6N430MS}9($|YKcw0aaK8kA!`p}1zvm#Uk{kRu0|2ie51+HK$*A*_=0sU1 zgL2VrqkYDs=^Hw;2%Z3WfM`Z0UelX9-p`hN+ZiQ>Fb@|2RxzIa%_gSh&iK}T3Vf{ zT|&`?Ca?J|xjy38)yS#*=*_W>)Ld3&cdZ!Pm5P`C3Eli;SC7U|-`iqFG4*9zJ{nFr z-Cr0tZJGmv_&wktyULpzH;ee1-YaoYr_PeoZbnfKPr*)2V#^UVjO65qYj_8NpeZyzQHH?IeOkHhsh0%yvD-W@dM+0R|MoPW?S?`wIv zXRxkID5@~Z0|)eN=*#=?R3OYy`iZ|q^-7do#BY%QrGvsuOwD@8hbV-3PyxooTs#^7 z??s4UBXz$G*3m))5{;19JBHmSD|RaddxE|gMIW}6dqD)J?&zM}v0!wrbF zHNf^b{&WuT@|38LN0~EvSI{^;MTY8)U(~BtZdjmQD4kK9Jj5|@g z^oJTn-NcqY&cZ^+AR~ReQl9w$kZd0!DcbPL2ZjABo9Gx$OG`hu6F>FXIlQ4hR@JUe zVmFqKbTPB*Xq%j#3LnGpFXGNOqpo=i>W7fFaQ3HTP}nMZy*>ZY?#5)5TrG#q&X)}B z&1SkDJ&u$(x90V7ibI97$T~PCW>LTNYFk9)(6#t=(*bR}8%J2UbzWKa83%Hz9d=Xo zqYh2Kxub3l$_EuV7 z!$jq`>aWM&#{Nih>2bHcTGx?&?_ru>k0sGDFQ}3eR~zy=-mD z%VSe|6-`<)mE&uyFV^FC^0a#}g0 z6FGHoNuHVZ%6qx5tBf>JN7m~X9$$#iYb;J7h2-0wFkfHf+9@kbH9xsEz|){rh~KnC zg^^qR!8Y>(85incF~c>3KkP0Vh08v*LhsdHPLSohX!nEg9Sp13ZI_gdMgJsqIC@_yHsprY4V47pYv zW+atr!-mn0@UHAc-a5>14C6cRrIKSodG?qBuT6tZ#UYNG`JMH`r|xZck8@IN3g#dB zoxs-mZ7=U)?z^i7I^s^p9VJ)7B%+78dnr_rQjPm&{y{p zM+~6Nnc0w+8xVWl@9s^(aNh9h1+(C=<$XVU^Gu$VJ*tm6v#(ab^bUULJj%Bs{#x-O zmS`uv->$jGjg}48jCVQO^L@}4+8+?0k!PU)bYP3~C6(fIQmZfHQIQiR)An-4>?Y^t zf4*7P|12t!YSocUUTP{~wIjV*K1S|0=~{t4hi7?t-qI}pBK4YiPHL8;3$xUQYar`2 zk3zeNG7E1pw(%=3{GaSnO;fsw+g3<77FcrmF@m?*o1)z+TixFwW6}{VJC2%<1Nw~! zI@*!msp~@QDg9)s{PJ_JEzVIt9_u)ls$!6^qGr-LFJP>ftlLw%Q-4Quj^9Wyw{q%L zhU+#X=f;@1_%g(7Eu{Fr7nJ%*h{-C*#me6>xKB=_ko2lQNHG4}UVi=2T>9qpXNyg@ z&+DYnW zz9`WYvbxnmLB49p_s)H!%sS$(x-YhSlRGQw-3tQ?%h=0&ymnFxJIc&d*5kF^E$|nIF&P&8pEw zIiww(*bse|P1o?VP~*E4`6+C>ZQ%{+6UDs7%fVZ27%fsJmwf-RoXWxzD*vIqkPPJk zn{#ymm(0%#`vsFD1k-rBRgRN3Nfu=F;U}D$W;AS{x>FuJ;vO9C7&IR57B0s2j*X^a z54q&wZ~jSi8VAzcKNm|62c=wG`yw~H$M}JEfs@_nflc$#;x&$iHMflH4`UUooz!^R z6`Auh@A`L1&Qsk~0YO~1{(M~%Zf>prsB%Seh@WI6Q_A_rUEB$ntGK>C0Ob;o{p1a# z_>-^wkhy(dIEO!ShU;8Z7MJ2T=KKn#p5k^-)mu*)&V3V6b9uv9 zojcpTIu!7n=EtMA`~?*j1wvw(owIdC8G{Ys7Zjz6h9-F=>3;MX< z>P`10*ZiorBb~2%(<@6qZ;X_-u^r$X5M`wO_H|72@S2v;`r?q|u9cvRx!>dF@)~oG zjRbSW8pgzB47R(MrBgc1jnl0zmft3Mtppe4UQ{MItyOfZe&^Oq=3;8tEpqQ6m(1BA zK}*h#l+}R!+jejIxU#5hdnDI61M}pKnw%D28BkTNA1GaPT-Y1tO@E>I?IoM{tt8!v zv#WV#2e%Z4lAq4jlceJ6+;+NIuVm`dk=N;5qxIoVYJ= z_gi_{?%BoBV+G423uH5U0^O6*wnqTXod=lPa#MSS3%AE76*eZF(f6`H^y*9r``z7Q z(u+&aE1#UglF2cj@!N>ol_D3-)RtkqC+td6`BI|#4v~A`rdJVmLy$n?6#0{Nz8`V> z+M#f4@kW{HF;c74`tf7AbLCb?EtY(e9!H9u8_3zi<(nk+IohX3Ce6Z{=W=4hE5)9E z?2lgW(0UpPb3uy%N#4a9JGd_F8JG;+abL)+hF?+e_^8%a38^Ec4aeL$8$VOsi=01_ zU+Xh@%m!P*<-9YIrXCf1GOI}MB?p;6lsM>@{TnXbzV*tZ>{xGZ-9%^SVDUiZRn=H& zHj-$8aEQ95P{oh&(y5rw7R$$@PcFMp3m9+uG4rYGxl<+wf2p@z>%FF;fQbu*kMd&< z9I4iLsyLJ(R1vAOdgRgdUwyvK9jaZarAxNIT7`v0r^+8nO)0nrv@G>)Or0q_Sry6W zx7}2%YefB!o5v&%qsgzpthkz2(#<4N1J~LfF)_!uls}%~?ncW~=j>lA@5xMF4KV6p zJYPz^Js`~g>|l+l4WsbZ4Y?~sK5;gsOugLH%zrBB`Xic09c4CKtMenC=Y2^u6`OVm z*1@AkwhMEfJY+%9v+FnP7dGoJ@fkOzb)(;z%zGiI|sEA#&%TxW6ac`mDT&wb< z&mT@xa~&az8=VnJ zo@kHE*jRL)&i$-}kUP9}>(39@FJDUiLHnpM+$%AFpX4bkOD}01R+cc5@osKwO6J(@ zI}5FGH%4cgQ?0v$?Q?QA@x+?S80Yc6kg_v1Qd=kpo%h65POOS=VId<6%IE*im$U?U zp=7t1=u|kZ7_}pT6g!q6j$Hio-l%c5*^FP2MxG4Jv;2#tBs`tEYUlE#JKP{qGE_&6`FU=zS1O$2N+Gi_@wz2o4_ApWNz9QfOrri# zZ>+R!M_Rg0iqM5ytluRhZtRQZ(!RsCb@OqS6W!xovZGwPpLB4FA^c?0)n&^B?Rr3} z@$n&_XsYJB!W~^M9qD!&Fwm-h`b2n*K`{4`lnj@a=?V4|m83gQ9?+eaa$7!_NZLnh zN$bYJTgTay5f!%H$B6;7GLDYR+XYy7MM68WOly9w$<`3f?I4rBLe9EjLzVh9Y|T;F zv5)W1FqsCUiDvG}y|%VhtshUGYX#KkQI8EV7@qhYa@+QadtKi3%cP1!Gp}vBn1bpa zI{LZ?ya>F-#)rZ!ccpS_uJwwA6)STKFY6};ng;x|A zUztfOMy$r>cPP_W>Ryw(t6d9`<*uqtY{8OpCPJEwTpV|s}nGYz?7oBiclMmhX z(o9S@ynvhZ(WUt;>HV)HSIzLWmx)^}`s0QkMYTWKanaS<<(y~QCTZ>Ls&4am${Z68 zYK7?Ts??a`3o-_*(|?9XrzTelGCi8&lw2q(!?ZM%n~Dywkhzol9<;c$@)v!;KJ)r) zuY;4{>(&diaukL}bPsZLM!t^JF32eKW;wyKWuL7e2tavxC-JrT%T+%F*-wsbS>zqN z9%a;98SD%BXbSzr1S>#qwzL)yF%)gKAQKrxu}qqt64QGh;mP3m0P(aJ#~w+k;5 z`Ap`d=nE~PBuah#{lkoDD0lqGFuv@fljYy^_;Gl{?hip-VW`m${XUZZf$siP%a!%E zPmv;Y4Ih^*So4{kt8yr$V(_xxWn0mmJrrM2^S#X3)BzdW~W^D@IW zxSrA4=3D#Mqea2)!9r&ka+KaX{nf}F&lRUV8FYTZ+;&Z_MfasSqo@5j6}_|98FM$L zg>R)@oBKATYCm3HQ5DMfpf~o``$qGs#At;Wy)jX{{EpGjcyHh9o}R7v9WvO;Xl?s> zX5>f1A0F+<2tMc4{*onrYmqfyMuqvkSnfuP)3&vW!S}s)RK@E0wI#Z4tZRL?cmP_g zevnn~iAgFiXGiL`fb!z1E3W)zUG$ZW$grf4+?%FP^=acLcIEY5WaU^G`(FO_0njN| ztx)3CN5#Fj9#V`Op^(KFD08p*M#{3WakdtOYIohFsnKXkCjnTdI~BXzyw{9Q_$-VP z_}Ha|-mxeYfH_+xYoj7b@=N+VU+OwP*m5ld44Z1d7?e!`cf*xk!(-vy^0@ZzrTrusH(DEXBtcSc~nzq%D(=2 z?#h$#wjkSl6@&4RrIx;nhTcb>#M_-I&`2-hqousDo>4pQQn=t?N?FdxEFT-DYVNpN z*37>>Ham;&N&N--73!I6Q?lN`Uuti9dMqsSX2@=rJEM2}_RrXz{DxIT@5_SZX>tff zJg8~qhd20~AMuTEjtula+u6C!*`D^m*Kr~1`Cc=nce}92tKBKmj62MR$*$IX)HI8c zyJGnuqQ$903sp8kk=ca;^VrNzP_)yb)8!;wX{P{ZNtDZGWIyO^h!KXG<6 zdS8~^AK>rtJf|ihTT1yyQ2@h%xbE>$y*mrd{vG4@qg0;k-tw_$kW%!w>5Hyxk*_8f z1;U4#KgtGK%hgQQ`X|JaD>5+fRzv%# zlJD`)k09JMeKI@9RxV4^QnN8^eP3;%XEG{stC}OZjL|@AjoonB(SpOG9&4I{-MZPd z^Lu?p_v%<{My-WEDhD@Tec|#%$dX<$Th+UF;!7JX2CQ2zM0x+U`lTSZePT^VQ#wrs zl!x#^hX=9YwjJc&!X>FQMtkJ%8M&N)rp#JM27p@ef~IrEVE2?`Hn&)j@8LiN#z6GB zr0JFHN{;t^?;kM|JeO#RT5E`9_|JUAqeuo*KD3f!G0%eCkX?1Rd(`WS0y^h91 zexu5XcS$JA$ipO`bUr$|n$fR&Uy;_m^zyUr);G6aXDD;0U)`{7&B!0^ylq~)qV0I# zAa&^_T}8dxDT{Bl(fTc3Z{@Yp++MN&o)EdLXcdhU1Yw}o(N1^MhAR76k&!|=mTlB~ z4UVGkv_d0lET&1F4q5UC&D64K8};-_w}Z-QzXIQX6z@n-`98O~bk2p5#gR zb8Mq#g~n}oj&7#A%+hJTr83BtD`$@RsAN#+5!cg`(&AQR%Xb308=Y0u<*&Yl^5 z{LX2r{|1WH18%F^^6c01Q-yOs^WGUey=H0Z*PAC-aAzYDny9JX|yJsBCQTzO*tL0+VouC2< z(cEM8xt9ET2M#RWsPRRhqa-w zp96{ER4TZK{pxE1p|#{kf>5Kn7_IT+v&xl%Wow!fyAHi^n%bYm&XJp)&;%gz9&=cUr+Kb>io3`f&~)~0(_>zV==>2{Cm|HQSDw03S52HL*?(fFW0pMegcN@FVQhvn@z``eh#^f4Y{BCz`4*ba#Yh()fw7OcR5SX zowK-Ct7s|ZI?CU?wX@~zv&To8d>_ReGuX`0Gp_MvI8Ryq%ArhTC7}KRvh^4#{Q%0hs3yLWGn@kw{N^D;WXtRIiP=F zKPvtRY_&z|x$=)iIk$XEGzEXUwiMrW-SEFcw3rbmWT4qnjMDnERH>W)#WcNxy|>4) z^jmwiR$JSj_zqqE!^a{YqH_*s zJD>X;#YdpK2cLYH;oSee=BES>RgQKPMt}Y+Oh_?J@*v}%{`apF!*bBOfTL`@P;@-1 zy{NfsaHxSNyc&WIJ%09Y<;TwkeO7OwcZIQYGG+Ut;{tsR=r*Fm4e|VYLq9|1nK6X8 z9IbuNaYK6Po44&ESni*bgr8Khhj! zw(OH#2Y2k*BOv+|7ld~GO`mPY!oJELXZiOv^fF_dMJGDKaA2!$5ITw=&J}F_ib4JCU`we+47$IPe8r2 zzuh>pv{Xtlb{)aHrtb@b<7MUpB*XOPf4?L9VN5!I^3n_)!#QcIqee0^U_WSyKhx;y zcDP?BSCcy7zoO;s$t(KsVh|x2WuI+z;r{vRGFgnihlh6qXOy|kR-LFI8{y;>CV($c*y|9zml(PfQ8fH=+pxeW?qrpiD2qW93YT~wq_EamMigixsS z?`CL&^n-s+{O`Mcf`Jqy^yKpkd*TD;|MvkHK}9;w%uE}B3pwuizpq3q4GD*D_9#Fu z_0Rv`=MZO0u?YYFzV8^e*YTA9eXfw#1A@9?a`67|&Ee?orZ4E3=p>9(n=k#l#x>gx zQ`+l4hVFb(=QWmV*)iivDc5~@PN{XgjBKsy8%4_oySX+RYEN4wO4*tjOa*!4VLH50 zBd=;Ec#J3EkGy?uzQq58qJGPaobZj*)W|Uns4lI(zDBVz+8k;sB7J;;xU^o%eL4CZ z*6}VCzssHlmG&!ymVKCtA(y|m!Izs7akon-V5uu+for2?x6~GgQ^7&rL!NR+s_Y>kG9S<`ct>p`^CWXky7h;ww~tu~ z;p7zQb50IkztwJ&mH3spy8-dz%X!OEl-blofFVES+-uR@qc^zJ(#p9wZT-+yw`w#} zW^r}y0hpq^^(7&i7Vbsk{jKV2)lDLoQq$9oh9}i6Ev(*DZsgZI_igX#oabt4ZdHA& z;H*P0!f2$f9tV`E{ndmNvf99*_Fg-fTE<6zxvZhpbz2>guQM<|KcqR*Hxe{ z<3HOu*SbvR&9r=tUY|WR_A63>yWk;P^ccq-da=*ZH2NJqWl@pf167zm@<}`;kH4xk zBj4Jw%)c)?e7&Ao&?ESU!{YGw$3|3KyXC!~7dYf=?Mc zRcxs4Ra|+csAPRz&9&;M;i6*4fq;Frt&ur1OKDFy+jIPm)w}$4l+Cj?c;YP1z z`^r6Lv7(}j@(OWQ<$UB5zh3tLUeAq3F)=gFo!)eh^ZDSSF#Ai^k{6kgw-@ZSFNLcc zIXwx;-;$CwQtW7LnA&H-3J&lW#fPdz$t@Q5ta2tCcm02=RQEgdF8hdS7llRpwH4w- z=q8*`xnr?EHAo;O|3Qa5^Y6R;;laAq7NfBp)Kd%N44lr6RQU9(3kPp#Z)^oI>Lpuw zC~dscC1P~!&b4>!(X%ve=`k^v7erR$Cp;C=ZmtFOX3;?44-;S~%m%%seb=?ke!Cd%aH>RU$aV;>HwC zNrP3=jz8Jn35mg61AC=DMqe6WYRbR4$BZ#b%{lGph^ub#diB>2+G%coMjDk8Wp{fm z2Hk3XU+9pcE>oac*s5~s8FaNS?d#TSR>v=k-YexQBok{)(yMY~6qnK?N9md|t#{fqC%@HHIc$Ky4Hr4-9I)-E{&y#wFiF^nZK(g)3h$JI$wS3y+reiAc>V! zlRw5FZ%RUYWE>3IZiP>ex6)T780Eg`s0=^!=l+2M_aePHf~LCMqMXDR>^7T7Shn(J zt&>ZS8)}zH&eQKUnm+RE&b8ECYhf8T<_i8i4-*QH8K~u3TH4ssS=DzbV|`K^;J7aT z%s$4G$>4;;8Ka$xL!xT=J{o9}&MHK9u72XM2^S7}p~{j#s8X-f(Z_xu?aiZi|= z<)bTH`$1sCqVt#R`u&Y#q|7G_-Bvb%CtCA;*wm}sAH^$L@l9_wcy_Qf1?wk&(SD&|$m!ACd8| zcSA8NAI3O73`LVpp4X0FlbPrl>c0@oLI)1S(lA9^7DRCe#r;2Vj(-l>dJKjK(WGW> zrt_UEqUg1Us^GvrzqY@#;-mAA-KVa_#N`Rx+kT>Ady~`Z zlX{k+)#=1p=MiT#>h%9o({atAar|kQG_v^Z1}HybUDj-I@6>%a%usu;uY245Lf(hT zk2*-Nlviq=-%sXP5hUZEi@vRb zoiyuYDts(@tfI;?=JwBbVoSU2HSQ_OdV;n;<=b*Os)ENj`PvYlli08+lMh3Fk8`ts zPn)N{_{R9PC>P13(@iULcJ^6zGV75G{7Luc;atxMJ;=_mPU3zs%wNlk8i+R}-JnL%N*<4Bz6Hw?MnDaUcf^jG)a z>fN@!bc6oW&y_$hs`YdA6f^=0EIC3uzExH;xP44FB3gv_w-=sq^X1-IE(?()DBk#2 z+VkZttg5lB=Cx5@@!BB5**}$Upy&Vp)$XJcCqxjWBBPBudpp0R*?sSf*4ekB8@~4! z5)vUZmsY+U7;EMy`WI>!zN=JKw*HK(e9TJ_HwyQLHPfR1-*v*#{0YZpQ&pb%rO_sl zsY~M8e@KUaW*!r0KU@}_o2j{Le?xECD6Pg4jch~PVVc3NuA7LOPq|VgO#X}r7+>D7 zi8*W!%Kj!pF7F3@^~$AiU;3Ki*}6gv1Ho|9;f<48xfNCFPk*jYv23zWKc^lQ%JcNR zrXUZtCRhnqT?gsoc0bI%F)@jpCZw%*`*w5=j>`KUE#exf51HwJ~R4>y4|LLJM=-Z?MJQvR8QXdnw!CnDZY7hK9da4jo$|O^3?owHNRk zricDsnYTE>9EJ`I)%wB+=z8GO6gy_O4Y+n>iWG^+*o+e|pY(3%nExYi^{Co|1R-Tq z51G5ZmiKFwnKkG;tT`${S+gEcdS|BTKoCU>(VOvnWKG<6DOkVO(_)^UA|9DGh~gUw zLfh=^rpLv-z2rweN1~yxJvIDoN0kwuGRt2tMp9EFvYmzs#_uR4+J2xHXjgeoLaO~< z|JyrZ@iV#U#LHv+6w51I8`&;vqGo1hEvJtMf9(5Yl;oIyA!#VutjvgyH&(lpd?2N3 zpcaaTSV4QzO_L4vq>O0X)(t3#3?sj`+!%Y{tWkr)=Gl5BGm{>V;>6jlYKftJJ-ewm zpZ8oIX&`-$pAe=wazMX6JMS4)QF~Y6Jrnw$wVIPb6v0bB8v|t(I_~f?iIYRLa=J?% zO+Fs|+Vt+S+Ba(<=iB+zEY#HZ^Og^#v-lWXZ2e0vh(qx=3!REs!Zraqp-);HS2O!r zb-r(8o-eqxZR=)>`Pey!+PLU+`6qH&5{+>ayRSye^y;iWTWXsZqZ`!Rf00h$@c-4v zPO_h@`8odd?2<%+L%sVIY8jUH;U@3Y-7ZH5^Ir#Y>pXmVF!2XkaUi!BrS&&dr&&Gc z@`+Te)O>eqs5Q#*^?44`A+x>PU%A}qYi(%@S$CRk)2P&cA|Y;RWl{Dc*-i8MrlZV9 zqq^I-r8_C&J!<**=z&YUyGN|}-7{*D`|IS&mXPTe_WCF^i3#6YoSvL!PB=$R z2%Bt|8o!$JI9Kn@$)9FcKo(tGBD)~f5&4xXs<)zKce8&HM*!t1g`tiuGVx-ry5{$Q z5lGvfEnRbde(@?*<*oPmCd(7=(;vkcFi3&fr5_#TcXG%bh`*G7VQUbpfk?wt5Jff# zO$Svzit$tZ_1YHqY`WdAxYn(u``S;B9o715>v$yaxBiENp#^5)wc3(Zx?Xa*6Hr7t z=C#f)AK{w*^UGI6tS!4Ezs}rnE-G#{^>IZ?!I#G;W@Hs)Ut7L3mUo;wIJ~v3o$NaC zFsHGyQNVJRc-@ILFuPf4!p*O)|Hj;LR3}B*WI3Kr%Y3P5ENP&EU2~=Imwd88bnVHa z4X=ys@r3Ft@}m4!n*6q04e3S7EkD~dQ~eD{lWaY%t$WMfv9X~L=;_si-xq}ZSyrV#ylqTS9Xo#mUkSSd2(rpf#}8Nc$WWVw^+ zTk;=09TPRM|3~+_GcT{0Sj%F?g3Z=G-&9_f(QPjh-5tmlarN^gcfWP3LMYq2chaS$ zUETHr6tST4i%&2sy;q`aA8(7y60;jXSBJb-j+^<0#@AWb_0B(zh~ZEMm;^ z)gRL*j4rvPFE3Ltg*T1ktO*sD@!Ev<;ZiKkod@3aw|3|=h?ungWCue2m45R;c&*89 zJkDcAOlD%G{Y7N;EeGo+a?4`Q61BJ0E2a~2mg}=@D*geSzLDbe6`?&1irUU0P8Ew? zL)sqNBUcT|$ayvrYloEo_!c?ZUUx3Yr)#W_$vH9-w6?IlJ<)=)^8V90yTeO*CO-UK zG9uQix1ZW>)NE`c73SW}uu60@_U#%Rt!nok;S+cHpS*S5`J`X2$o=uVPlt3!g&#Dx zo&K%jvUrf|0+>W##b>dBf#hRo58PK6yplF^(pBx6>y0BW+zWkKvv;yc=f!tV%8l$j z-G9u8*!+w$9Pa5c$41OFDu|C6wsmV(sbRgiaihh z8dv`*a(4flD4_!^| z(*a^*JcVa@&Pd_^A?!Wia_-;1|1+7H4YMMr9kQ~r>x^tAWUH*QiO7niMG4tmv{Y8I zNu(s3LYZk85eg-e`ah4;_5I!7$Nzud_y2l)e~;^TCFlA1yvH$K$Ln|AOg5<`5RPr`MZS{$E}vJ=4wZP`;wc_roK?2YVP9ryp@F{$-qg z!eh?0#Jj&7DiM$A>Mxff)vtv`eH48&-$ainxRY)W9o=rr+Ss1^%S(fIcv?-Xef?ACW;Ncx;PpZ~)JQ2PJ)aryPxo)}3ALoIQA zZOY1eA3kn(`+BRI)sNWxIWhL#$DMVc)?$9os z7F9J}l$@!X(#UL{S+&szTYmb~SR&xG1^e?R#c0o3u;<3XC#$sz+xhQ5-OaDZqO1o~ zUvRuSTs<~@Vfx%VZ>72X#!#Sw75mcH% zk0R@>>AdM+dfD)Tp}B<@oN~42O?)(<-`sOQ*0&zP+Vq7&B@ZWS^B{kG>WLMvMr8EY z3%oq^(I&Y|bu_E$URtJ4xHIGP_9}WE-dXJMf2FW1eA06Ng9-4A>pQ1Qf$2bsR#ZMUQ?L^c(hc-Mp?#@PpTlZhANU z+%ce!S?|<2d2x>y?i*k0?x!pBZ&@Fo)laP;ocx=xdx7cG3b z<>~dx{r7xt;rD1r#Pb)=QX`iv%lw@7{X~o<@WJ7_28jtD#?Gw(LP`lXFj!f#_n*E& zw+c?3O{qLCderBoZ}-N2>s04p?|W(IhfH4k`hQ>j*)?^?>w@R|ZI>BOuQ+?4ar=r> z2EV5~+O_6o-1@`s<6jq8%q}rqo?ujUeb24`99&<^efR!8j;s55?cdQ&GjjCd%sv(u z3X}RCJw0f^SiIH!HxR!6O69IikzWhH9d_}A zwRw5%hEI3D3PWWMWrrRy-l`kGdrP*SFc zmfFca-@~Hf5B8L9cMuZS1&ERKrNw9V6XnNqKbm!`8b8R(toT^X10F%Yug;17Fx~%` z#p}1uBTVB043h~_>aSV!y0MOtS%3fdHvZbW%|=ITk0>lQnOS@`M@P4M?^PQg?N)xh zfAFAht6}h(x=r@)?fSOs7Za1lhrdm1-=q14ZDUhz73BD?w{B+ec~M2&kL3J}r|ZA< z!UzZ@uu@6t>siJBz#oc*+)Rlt)cmW?uJYg1sU|o&6n1j{{rJ)3n82We)(dlAoj*P9 zPl^ZHRbHr$i2$C3S4HOfr_U>%rd3QW@3J~QXwA-N-=6zH#dy|!ca+Vk5$7(aoI1Sz zw#mKzEB*3*t~|Hp;DPKVlS{7>M}O+GYKzvDR_%5s?`=~FxIDb#jL)<3^0(_IzNom_ zV&0X(5Cq=LfN@cHCR?*Ce&w8O&8Xv2;VM(T^V}Visqg%x_db_TgR>k1a^K{bd+s|M zJ?8X}sW0ZNnC^OKNqg4^m0$?Cyk6CO!H{?Q71Q4gYM1qH)R>@;t_yO%d@_&M3OePo zj?k2JX*6g5^q)NrAL16)3GV*t$oCaTU++n|6H)&I1+5Mh6{Bb&NPE-twf?+kZlimy zB>tjZ<@{OdUjgS=mv@+Tu~-s0pJJqL{GZLg@!-J&*hZR@r>*wyG@B zK3d=KT4LN{>%Nz@GS7}2b-TUI8TW=&PWL*}p^e+~+ktV)!M#>|YDA@MrQ5sX(HYy9 zO?=b_d-45F>65Ia)Yk&{iz0ucMsq3NXa`nP z7b>V-1iDaFU>3wt>CaiU>pw+<4>aB~r$k*>_}1ZnN?s!Gyyn+8r+ZT6IQ^gM#L)8$ z-hhF$Sh%AUN^WXOJwg7lx`Zs}*tiu7qT)m3U|c(L$Bul^#_NFE3tH`|7okM zE+2I>?Am{qvHtnDj8zf4XVw06sU|JGpugWs`h&OSmmDy?zb)U_Dm^%E>9f4*%88-8 z(VLa1*K{@8_-0J==FR^dk`Wem)39>UN6Qas5w+v4l-9jvfnpFe-Ls%5YJs**(~sTXmh=4k zVZC=*^=+oC$xI(9eg#fBO0yW!ip!+mFpOHy4D{RBb{QtcE6~MxuSt_0^En*1vYAqW z*jm84396P8)FsHJ$KeNOfBJM6%>XHN1}IRd$y}LRTjWp>z?nxc;9zFHi3Ai{O;Yx~ zKsQ|AfdeCGy;oOz)eU%7C*FSiczf*Ve#vw=t3QYa&cRv8X7g|U``KKyGEhL2{%!gZ z>zrTr$4&ZYDcb$_hlu!8`Ja;MQb=`y1z8!T1?lgWHb!KD=JPbIMO~X+*t~FM%Zo|y z`B4AaAVt3~esaRMAC-n%7!ouR!t{Yi$q$`R9ERHMly{LvSy}Q*#Y9>VVQ?AnyqQ!+9#=%-9NKK<&n zuu7IKM~{*E<_T;C0#nKD(e~=*!1^( z=4geu9en%i!ifjV*P6ZWU-G9z$>pV2FYjK0w1%FcS7suoMgxQnJh}Z_0X8Msv*sl{Y1={Njh0bLew|(^c@;9W(HlPc~4`* z^4f-1p^5jxG$ZW~wR&j!cw5ckqUIvUgpS;qytboZ#I2` zD-j~xzoBR_sxg-E`Z8*^R0dz0ZKSRJVGGesD0_3jc!bIvjT0@A3B+kw4heA(CEo+V z!P#HFn3!ofT|jgt>E_MqxT4UodfGwN)ejJfqD+m+=jLl0dUzLY3khkC-Uh8Lt`0nM zXzoM(PcJX(A|e{?G9^p$Ap~8eP{b}o z(sSCCHSNsm#EZBV?FPLq_xQdQHEQJum6_+gfl6gsJJee?qeu{`r5h-t-mFHAx`Bu* z&ptt(9Sx%F{QUOv?64GaxZ5xLZt`}~+P^`i3Lo22r^d#O8=ux{skEmRxC;ZX?jGo^ zi;D}*9ivS@hv)*L;_Byb-sB5kJH|L@?(;LzEV8QAX7#uCzIN>z6DN{+2=%^o=OZA& zWh1SlXXshpqFXmzM$!x#IMDsyeargt#h)fzAAfU|t%H2AGWgNuRm=E0_GnYHsDb}< z?b78MN>Kr{<>#UR!3thQ-mqHl-o59(z7j8=quzX_dhOp^McCjM79Z)XUE@jg*fu|Z z|8`_##Lav6>Kl5MU!||?5abjR9bH3D)0c3)mSiVZInmHcW#*-RfuX!WFAyS#3zl;5 z&Y^HD3YFX~UO>@z^^aP}A25b6Qob0~o2{Qd&F7n~-k)r2rBaziW4V!3x=zEodI84c zb>#>#fbbfJ#WE#zHTKP7+O(!-%Rjbc8#c%hN=~kVgroWhwN<}Ho5uS3VNj9f2qyB8 zqJ7J#gd=%v-4n$_^cB z(dw;hIAl@3b!CmJ)iKD@vp3%L^5qm!g&n)e$Er=N`oWLVrVYTw8R5-T;qN&aUbFA* zv!|<^Z4S-Ni$E)9VCx+rc*4-IZ4E@4j9Co5yZ*1QM~gQL;jHLHRm%e^z*O8$DUb!EM7@9q?UNZ3FZJ!Pm)iu&rE!DxMYalS4$g$?p9 znu1o2bO6#J-junfJVp?*EhzYj5vi2I^IK#0lQEd$$*A zMe6nqHQByj>dn{oCj`e@z|IYzYU* zX%i>2dCQiS5%Idg>0|TsX`dg$dfFq+A)AMmMHV{c>O1uJ1HQ8fqK9^Q$h{dc;QcNnS;tn-Fts?UiYb zV3}F4rd<#j^`B&7`Rv&M4Q${o+bKR*~wX;Ksx#wC+9)T)6Ore4fI+ zO3eoHmUb5whvc<2n}_t(%5QIKl$I29Vhs)<;58-Qt^TkGo8n23!x$Wu^j{wqg5i?i z79CBDnYMeG^&Pnk96z*eI+^*8n#6=g#Cx*y_-sB7e-MU)M#yy#`pGh|kXWM0=FOYY z?rvZh;2rf?ebd}R5H~}g;Jv(GTMnfss%XrKX=~a@y1l$MjKPHjX^P7 zD6vG>i3;s!%y37Q%ZS}Y3|fWwZ&o-~%ul^^`eMbry~pgY1G|W64B8T&5S}&0Byp1> ziTsk7;l*ZP6HIr4esSA06s*A6omMMW^pHdf?VbdNo-Dm+AUfAKqxf*&|M_rogG6|9 zVj*nsOtlg9DNJrS-Kb5QAua~Nhir|tn>O{QFXtLVs9X%vW&|0JGyb+b!o~+={s!O z>e*df+G@nek$2i^Yg>O%JKZ9Lh&!KK_{!LX;cX*^Z!!=~ZGZXm_Egy~Om4Ih*HRbjfu zPt@luhom~!ZQ56cuyuC0UOm;?swUCbK;xhWZm$v&6iBk!ot&}3-Ik;LcR$)Ak+T5` zm-^+pFkTDyEh~f)2DjlX$rciRp6C>@+mjA)X>TF{Ty)7iB!L_P;kgz@^?64J2`q|y zH*C=O*T-#re0t94pQWWTk!eg@z0lmb!PrF+%(K{B3+YLpyk6l|7>ji5x#b1-5v?tt z{y$_$-=Ju0O+&S{a21P-19^F6?PRfH=IA2&a^Rcy?K{Oq@3@Q-K&m1ar4pi+Gn+g- zEJ8A7c5Tc9R}RPB>#LV4(l4zoaHU#YS>2GSdLLDL(~46Ps6<~7q+!O0+Ut*q#~zT5y8WHn-h%GuetY2TIZMMW-8E+PS11-UjvhMFN-FPcuB(Jy{{ zWZdwl58S<@j{W<hg^H{LZ3i!5IV4`A!&A~DQ&cgR#lY$ zAzV8z(qH7T%C22?@bfE6i`pSGIED#OOo6HmMll3MhU}~?#pKD88#TK8+IRxWcoOX) zqrBw%yOA$1Ew3q~n3@h)SIbAWk@!>Dr;k1gmL8Fjk2zpjL5JC^t9_B zO-{E&2u*7utqQkyqVvdJ45~npJd{Xc?yE~@KNh3P7m8%Ocm!@iJsq7&JU-*fL(y!T z`{YRdnO!GF&r~XPHmGgoWK8TPLJ3I14-xWtl3!(>&Zx8WlNmzl>|k<kB)GEXxPK`Oo*w4Bl09h7g=Gn;vAWi zh`UK(_#E`Jc9*rQ4?M12UBUKRpq_XVW!m|#F4YG-N#wMO5~oa4M5*W+s$*3mS}EPu z|24Tb#zDjuo6*cgG;#~^rFb`}lATRWH}S7LZ~^DDcWxjN?0gk303r=mSsR{p!-=hu{G)*WIXZPRm|H;zuTVtW z-X`>O(zCFz0Ah2%NTF4vXE?+WCE$7lI0z(b8(J2$)6)_Uy!_+iK`hD)XJ<_Ncq5`) zPV^(6*GGxZQbf)am`K7*8ODHzlF1y*PWAR5qOzU;*V=BgYAN!_2ssTf>fO7s!uP;+%0iQEGkR)YuZ~ovY5mWa3%Y<3+KjnBKlVF=P2y7D z?%Y`0`fGUn*)_;wxl3?I@?#0Q^~*+3vz4@Ca7dGz|~^BT&ni)1NcThz7lyxG9QtZoxOpP z4%e{cgfjIFw|YlemE9Z49g9TQcn?K}_RWmyYgNI|hQ|`uRmV<9B8Bu`0>d6RGeqI$ z%&v?D@b;gO1LZq#|Nfc^Qcp`1r|1G8K;Otnxay-K*P~j#fsEdPu?dY(w@g1VwI=XG zYOJX_($rs()0EMvh)`Dr2lPO_PD9Jn+{IL>?Ed#wm7+Gus7S@$5##gv`D46C?jn=~ zZ*m}!yWm!Atfy6lZPe599Mwa!qTi(>hoYA*OeyZn(gHm@DIv1%S;bY$Qbs`a{5Z)S}> zxl__v0;)j(E;j<21r%KQU5BfrXE;3U*37QVy4GW6+XY4j%TdQ?$%9mOSDRx;JRF9~ zU8QVS)KoTRD9B6JyLX$}v1|+;Ti@>61!iJ|M@H^maw%;&(`l4LmS|RF=v`)tBrE^> z_Q9I*aU91)ObjUw)^-RwV>mx!k|go;v{|GQWHC3@8Ya1_WhHf>m4SxaNzR~o?LiXU z?1228c>DIptz)|e4QmBXVmCPYwm81&aqsro8e1afDTd%n<>?#Q8o3jolctL@4G{cx z-?3vyp53Hdcx1%l5?L?N9m8}~WdLXcLqG4LM(VFD>0eQ{3?MRkYhUq`rapuSs0R7~ z%<1*schTKsp>iqtzy$xFJBCFtfH{ej|7rCOgP2h@FraC_&a`gH@K?f(&4L)%O^U(l zqsD20RYgslDpVUY9qkY8l6p2X2YdO=kSS|S30vR1Q>U1lUjN*^JAw<;+{C0#M$GgS zz{;)Y@UpTDB&vcq@R4fc8H^+0+Yk7DoathIMURdO8OQerI$8H@6?LAgZoTYfUI+|3gm zOrz%YWB$Vh5dBPsW^0Td{lHv*Z$0flyH(0TSUMV&wxUMZqEn~Q3m$E`e7WxdjKT2- zM{gB>;m4lS*fPMw7Cjce{CT^$@`ul@+O!dMc&p*VMLs(#Cr4?vebgyti7-H-3-KHi z$ePf8XUT?JrYM#D5+m#2+A1C}${e&g2^GnQj36V{PvTfLZQ2xMs^5|E$F-PUif-$# zV13t$vhQ`jWX6kpI5WPKrlxvwG8n`q2OgK>lAm6i%%@e^EA2oN8K}jW8!@wN_|XMc ziy65K{+W=Nm=knxLR-CF%rxjR|EW7Da*HlqtXt$y9X8DONVFYEl(KEx>e6G^;$-Dk zEn6PqtpLB0k)6J&{VC$^)2EFlqkqL1#Gxs}Sm_9r(`-bugGYef=L#ddOzr zRxhlt=zr|^@vr#!F#K2)=hvtI{O8Y}U1N|k8rJk5R5H>L*qamYs?5?coO}2!|s9CjYRhg7Y#_*6#PULHA z4^kTImepp6klWB`TL*F(aXtdm5#5Jj&3FW_PaJdgTKmGS zq+m9gyCq_(Cz)5*qtft)q$Ced%^HrM5?TMH7pJ(ij8_I)bw+6s%!;|j`Xn)V4?{fi z>{Q{o7xs4se6H_lz5(3ErBZvvd3QO8A~P@7_5O<{3Uv*K#tdz#Gh&}<4S}^p zrjqDBHPGA2>+AV;Ea7EDCPe`osZ{sVr@P6dLfl~jW0hUbIK-YhC9~srvJEminsTWO z(7^1qAuMv*LldCs!k3MJyrXO)eA6@H$e%3;`bkgbSKeiWc_J&2{86O1XE$$>2w(w3 z;=%V2$$MeyeEoreeXm+eCF1RyH#5M)xH&!KxVIXY+^r+Oj@WDu z%yM(HI)D5Cs@~lXuE_!7%c6Qz1&kT0j$uYI2W$M3bO_Yki$3@q<+um^|GzZ z=q3r&IrZuiiJXz^YA9C(wlpz?QfCpol3Bo&yY3y~a!lv_hYvoOnfi!Y=ViwCVM3CC z=&7;ZQMF<-Q(s7ijRSp%pb&bM7A^*EwHfxg=GWWZcBf|6klh5Buu4%U)@Ci*w5jHJ zW6K~;K_RfMJ;JEKVEgjyB5squ%PeffxqWe3GB$v!$wsc4?QOdPyMoMJ<{<-7;uq*> z=#JJt{uarg*Isx-Q5Yov^3Us~3b*)Mli0R*uMex?!14QuT67`9u+`Yyv96Dp^m%O3 zS*_9k^xc5^F;kN|;MRk5()*;xxprA!z6KrFN*qQ{3frvL2rT3#u8iC}+>*S>5cQi( z>V#Wg@cfMQ?8Z`bkV!EvEt@k4ND{zaeflI}{z#5=4Sl?aHOimbKRxa|(?Mr32e&6i zsu!`nDkbE+(T-b%v@IL8Ze5iPr`F}|Qdu3&Jo)naYF#9=fzGNC;DMZy(zj)fS_iw6 z3hGy+=iMp)svo{ydY|RTT$97bwCz0qX=e!>NIrF_-3_7=CU87O0ZnYH`r{Sqk3UPH z_u(0*vB2JyyI`@VeUs)AYEWHEY+lWaR6W(vR&M zH7fi%dU7!ak)$_?S;AsSIZ7QV?_Yx3-2jT$wY!Ni@s$I-ed zC~sw9y}5b?-o;;v?Fy!S#?Q-Eb)J!+2!1_daK_LG8;QplSACrk8!Gju%RzMtK6b1F z2t*Dck0gcwRF+^dAj-+iCfS5RU_kOT7Xzi+aMmQnZNk&=p*v&F;LT$^ADTBDeD#TBzOQJT3^}l)hb|PF`iU!J#H`I?EM3I=uHVSx53}5mCpdHkD zhdtJY7%cDk+4ky6p^kQa&HPs{9OHivAlNmBPmqd3O0|xv%(xOq^of6x#DC4YrYQ$; zxH>1Cl#95ecug#o2Ot8#$~fwiB9>f+scOWW1TIEV)nl#&{>62qjw5ITR?*krJmL`1 z#7QhHr%e5*$?8Y-#4HM6jfip;b=k{*{8C0%v5q2IO(vvX-YV&FC6NWi3!zp3^$!M9 zRt*WUnK5(bCP0{M#6=Q8!_@nxHZV?oF?5oObI*A<9gxzq^PlNR%urEYGJq!lOIMcG zO^@3i7E>(c%3iC!wWQ?H!AvVClx<~HQ43R3TX5SnbG4bg@tb6DF#{FkcvB<-yNSo| zB@yjZ{C-^lS|3_5BfkB_Qo91?$aq5&3Bg)`f419xYo=^8>Fu_C`;37jof@kvIbr?Q zmg-GkTU;N2)SrnCKK^04L@S-`6Dv!V3nvtsTQ9YOL-nobc;+h-Gcaaz3N_$=UF+%T zRaSt~M`X2Bg)dOsp{+frrxH5A8nANF3(g^j#&bDhCU4%nS&z$};5K1WQFP{UyVy$9 z49qbB@<}ro%nnA7g+9P0^AP#HGu73%HWvDcAFKkIC4chd$$ExE-b;>6cCn}TvcnXA z+yaVExO}jPzjOco4HUk8#UmuR2dpYpQX_)1!r;G?{93r>&&ePkrs<^7;-eVf2+5v17QC zo~uQNS%Bi^%t0dqG^dzQ)F6NAvxmiQC9na2u&b?Dv4Tx#T(f%h1fH*u8Qc7l)o`_0 zZQ?}GG56`o<{vX+I+^*7>Sk)HVtr!~9O~;&k!4f!uZobe1Fcp-Ot6E-uG2?$C_3yPf1^;SI+1C`($<+nRYB%bfb#`n4 z`B!OXM%$6{T6T7gs1zr4T{V}9geh-s?FeL$^N=AaHdmVlUJ>hvHCtFw{s**6V3s>& z&l!9@h%2N}P%D*02&tH@6wqcM=MRcAEP6`G)Pmiot~|KCkIF0DpY{~ZoHW$S;wT6U zuApPA=PTxHb0Ly+dS}E@Mpr0&RLTxXFJMiyd$Y86dd&2IPFKrpIEBGC+}i9ic@hU3 zpymsoOQ6zGw$a-^&Li?+UBsPJ1E3UPIjZ5hC;0_#25;vvk&P_jf7NQNJk<8Pi6fgJ z)fG^%JD>KEv05NS!gTU2z){iD0;oO%h^Imldt!2sn=V$?OGQBY zXe|>#wm@WX`jgZp@UUaNV|@D>Wcb1&Am{{LZ9?e<{3^QpHwsDP zN^TQhXMg9pAgt|rv)k6Xb2A#>g z#R>iN_+|rv)X~1TjvQsENMMCK!-0^qDwQzc7dEXjslPm9@gXT|Fh#T`wz%UD&B+uR z@Bxb{VGE?+otIltr+xdQuH}2{Z?Q3RD{4zHgVAjsI>9l1Yzmg_{=ortFHZ5t+SR;R zPAvqCLCdJ(ihn@B5Nd{k^za3WUt@1l>}bB@R=oxdHk09N-n=^D%FmpdLMwnW04$bl zb|UIGm{c{=PD}1mA=7D&e(T+tg%ev1a^neH9t0>{V-Tp?q%=hPMLDxV=*Pg9Pl;glDwk~)*11ndS!P>jmaO6 z3^r0Q)=eBY2(|OUvnNjqpldr_pM-;i9XlR#cgmzGQ>QNB_;!tNt?krI>H^BGiy2vj z(~M9xybKxPjPvss`Y=Wpn$4f1Ox#YQo#~Yc$ zaq9GGjW>%Hew|LDR%zRS1yf9RH_X2fDp!qRC3dk6RhY@90F{)H?ZT=E2za0$nM*wa zIA`SN?sMkMp>Sc3vAICcsgzar?D57opPk)m+kh&+uI_)Y_bF>F<0>5pVuth3s>(=S3KW~4RlT%4ydE&`Xm92XVA(Xb5a&?^{ROE&$ z{uCB2W=vsiR_>Y%nJcQhem*{@}AB#$1h^oLt3;pFor6aetu zgu9w8b%qAk&5r2?mW{d@$k?>8T8_uQh&Z_ZdTTqPr1Hqkq1-;V49{hUT31YH`IV^zER;$#W4Xw)SUYVtovo@$q>#|#!ky_Qq}3)qlbD} zau?SxFRw8?M+h^DUtvvo^e6Y16QA@P)yjCGf5|wg+yDmt#zw|fj=3IlZ546o+1Txm z?hQ8wyMzm@P*9pyFiTUDD=UUruV9U(M?jg{uqA*W)IdlHJfYP_iE~&!%^@7gmy(w0RJk69^%NLPqqaLwtFi)Eq~t!99{nxMa`F zgjF-RD9(_`B>fw5$>3h#&VFu134~3ZVKsw1C~?8?3cbKQNGKGCh&y^M$*C-LSlr~x zIM=#Y|NPL$DB6xKP#gk9P`Gm%TY##ty2-b{Ucm?+v5nq{?YmrZ(c8)}-!OkF+O;E~ zPFVl>r22_pe7wSC)bQaJFkX;~l?mcv8o|#6DG*3aTs@6v8(=F-Y;9oXQl0Pcf~26( z`+cDzvlBPPgAzBiX!O>M50MjGa4q=9S#LVQqF>p|^%*rfWPSQnixVWEgqRi}QBsjD z%#DWP*rI*=nwz8uatcoj?t}!r|Y%!O?6L4+tBdO?UK)*J_%+?xskY3b++6VYb@j27$;s+W=2bn zPCf_?wV@>NUun^eOn9QQzj(KL7{5V0$;CyKvJVmpFR@C@ht<( zqi$||u&XCDldI=<|*?Oa)uljB{DsI`tLEn;d=pUM~@uGb#^@~ z%xS=*zP{&~qlW`>rnRg0JSpar`bE#PfCw}zk6#OjPJ3*=c^72s{M{+n}z39k2StlGx>F$;_sok694m09lkiVOWt(OLZ;ib3!~n9WORvhHHsV{h^Z!oRr3W)L7PH+~}eMM=WOf)zwi&codKNK6hUHYVhK~v8QOT zdj7n@yvOq|Qr5oVlnC3^U`WroO&MxRvC-RqQ1~*t0@HwV>2Ye3d-nv= z2-T3b*B?HtM^+r_STC?@u0)ACCfshQa-oh7uj2Zsx(2kUDJXUDyVP{I-8*)WSoFVd zZ4XnFXzaiK-hHa2aO%vv9q6~ZLEUEkJEX?~a=_h^pT?Y2jkMFU0$g{ zJlCV6e5zQtz*{qgoC6d=rWHp`tv>tRL=Yoich-{LM&fU9v zLQxI*J{U*>-VZBIhOMKZ7F10*J{Wp0fMBY%jnx~lS^2+#Ln21#M?sAlf_vBi^Qm>zH=jvLbv#QrL$hJ7x$iTWO4-O8<*k?G} z=S&Uti!hOXFlIb@dK>DGLX2iV=+N*Z{f>@rMyo2LESpM@-TzO~*s~>ql`x>V0j8T0 zPKPt+ra|Knt}i#Ul48Y|mp+~~8w$}6t}RR_F|VKubzNOu1%(4hin)7fUWUo1%=q6y z1*_fM9GJtsDE`KD9D^8EJw4&P*Z;ljYDIaUe+H~K32!`i=1kke87-Bc6aVKGS*MW3 zQnnF9;mW8HBd#-3TN6@^MADz1R`{=ZXi4ZyAB3Kv2b~d_PE8jaccpeat84ut@H2{l zt!PA9!h}+y%|Rx+k7h0+*j0!~r)U4wGz}<-@DpH2KO~HJvYN9jgcMj)V~7y|%=G~p z;Bn}_n|#@;6>uf$NM%A(`_czfrqKc;`9os>d+vM$6u^TFD2f` zvJxX~dft6ngfJMLQWRNl-c)YZtQm|bN&2)rLsgMA+XoRVz}AUfcBSru1&ej9pQ$OI zn@*oPm3Ac9fL972YRsnOAY0P6Z(m^-R)+J^)PzjpE6{zdrYi@CA32a#>WcC z_V!|3L$gg}UDx38z?N=zbW!v+$rA#bscgI26_nS_fAu@|?AeaR#cg!arsqsClF?A= zTQDk^RU4gNtMa!Y)xLm&W3G>E{zhm?l)NFJe`$3BB$mp@`h)`I3o-Y^bC(%Of)YMdQGNhf9=P%eyj_isjq0-bTXqj%{!mJ-- zF^&hwuwVu@JO1=m7Z5pqH<9zi0Pn4Y72=8jtx`C{<5D{QACVJQjxUfCBUp zr&+we1(oA^FoU3N3q0`OfJguQtSooPvp`k}w{8vnd|Bwsghv`2^Si-)Cj2yU=fRvG zUG$Ecc``FLGf0ofXXfnL67fZx>JL|OAYdCXuy5V|KNSk8?lP_X)2p}%3ryz>vM6Au zgmdR!R$K{*XKN;xJ{grj4l@WWP|gWmu%Q%rj@;`@e}>eB4wx<1Vg=nVUhK_X5#lG< zY}7u_$@zVa87rdx3LOL(yJZiULw9D9Lg+f^Bvj>Xz}``3fuK3*()HEQ)oGWwVQbZpe5i39n+ zbISN5w)A+^ITE-*%riVlNNr@~Cyq|Jc^(bf5RY^5;PkSzh`5r4PwY zB6*_JU`G%5vx!QI0~V(j7-8%wY!%DG|ETgp88_#>uuC#!^y7t`m${bc2q#f!z&9RU z5x_?Oiy58!*q!JCd~W*Z@6FZ}Y7p#wWmg^64p`K_ckhcgKcuTvRsN=R-Fo!cjME?i zQVm(v;{OD7JSpwlx^?SDeOGptPH_#*stM%ioCtG@j9Iz4VdutMgI}3UxuEbg*H9!j z+p?jo-7<3YNxv442>&3=f?>l5JvH9+Tu1SKSXo(>v75*|y)ke{r=6#(i5@ArNMWzt z%c2vB7KZQxOpb=9d5Ck>O60RWx$&M@HS_!`4e;SoEZEm$ftEsA03ck}Yuwlepw}K2 zp0G>l2Pu3035G#OQ)PE-7PPV(EN*21dEm&~m8`VZIy|&t_erp@`j;KCol3j?f(2U%sCNInNVc<>-bIR= z^Iu;v1l&LUaebBbhw%x{2Y;2=W&`L-r!5qtG4TbtxgPjv>hevo9vg}1KmMU}N9KCN zN*LQ`4z_{`TIR&W1{A}ONoTfbagc2z6g`ON9lt8-ZPbQ*XIZeCuJDJq(6CFsJb7=k1M~Ha$^VK|Cw269Quwu9mQ8 zA%YBAxw4n^BJhijKMS)J-y*eem9+&DVirOT%64`igaM<1c?N2?7uUrtbKq-!wgvZ$$#^ zoN;sGZ_z@@eRRotm>sfHG8e-WAEJw7IqwMj8-veEf{0I3YXTw4gy|!v31j7G6+(SBIAT6GB zenK<)7!3%P*C8|5wy3R@ZeAKgZGHw+X}mzQ=jRsuzMVU4ENLr1yC-YB>(^PIqAKcK zA)2E?t)*IoO${HFjq?4LfPfU?Yf{6AT5#;^BF?(7Z?W!TCEZ>To52JnmcEy;F+*$L z+KQ>2aAUDQ-|QG&NJ%7&VlX@);X@LDNRLJtKV`W*cJI#6zv?An5tP&;F^|&H>e2Nr zSrtT1aG?~CNZ;iFeR}s6QpRN;j#| zWl0jndl8<{?>_v^fhzi5Wvd%zR|xhFSh^(VdJPD;Nre9i72P#qkK4UD+?rtP=ZDCH zQrsc0I$J_y6HCTH_lk7oSg(En$rh=X1bV?v?3+|(2B*X3_jCLnZovYC&#fpAg21=& zB`YbOp(Nl>&j_%qBc)|2YAJ+!`}E4%K8iz##qgqoxWbvTc3|O-`C8geb$m{lj_*k0 zj|XQ&Lm{-SUNxMyOPel~h-Azj$@0BFB}ahYp^!`T2n~4L)*JAfEx3lx;SborLYnGD zCjn@kvN!!JTW{tqGNn|~cU_s?f{zm>;(K*@R_kNQk0%bWYIri!r^KZA%Yu&qSR4#k zDks?W{?FE!KtbmRtn2%nFvdBf-Ea_Xj`P^hA1Biq1Xcga(w#ZKjlcA(qkv++naprf zt{I2{U2hw?udKZ1m+ip>|A0@@Q~^w{x68wK{I2!u+xeAROj>aIUHLr^dN)V$snXFS z?1`YTp{5(Am~>jYZ~GJzuPG)I)-G^xs6@7H9@flsoHXH*DXaU!%)`Ii6eZ?<+ZEWy|pUpl0xEicn{_b^!+lz_*p^Tx>HV)FlIW!D5S2_FF zt=C+sJllNto$Q>Pjo{Zlqf&op&*i>W4*037dN8v>pX`5=K?acm@~`ByiGt*^Ix2+4d&Ud z5MbMQ)s%u^yU(3)6)YXjB!NP6GtbJs0X(jhm(GJxV2qywewtNqX&|5UIscWrhGsoo-K0pZe3?ym?bWNPM@`nY3EiHo zexT!!Bcz1Sjy4Dm4Wg|scE#sOVgRM5C)gtV(S^*Q4pZ$4C{=G77G(VOb#udI(51!R zkORwdd;DGQ{L>Hmd*Ill8qjyuH^;Vi^@u=z3o56&+ zPS!yst-YDouv=VQIgcm_dCQ_FsvqsFaGVI)pcCi>hmI2xKBZ`NCli*=@_jvR9!+oR zqymyzX%$uHK{^$-mmdK+}A=Hue->hE$)m%1Dl;8wE z-?F7vYVRhCkcg4b1Yv7!=ILEtEh8=`!Y}_7w6<*kLJ*o#+u3d;yE;i{wXH(Pb{G;GCtrtHvfqeMIyp9(#>(`w+wSbDfuv>@&yhnxT zoS|wj73Y+cAugDABlS&cK@ub^J32S%u=UF`*y=(~Rrc*pr#-!vD9v2IbTISs&fJjy zD)Lcr@fV>-lJ*Lu#v@h&y^+2F($l%0AI17a&0Hc^s!15QgA}JEv)Qw!5pUe;FX}i5 zPt*m|pW0hhq%ozSf76UUhag203nXE5gpK=tOqbt4FI3YyuPa;0aR?9Tr7n7ZqmeR! z8!ydN;1jC%b2ky23v*~pv@PCi13jF1@LMaNaJ5V+{!)S6t4rGcG|n|9PtSNTsIOrC zoiXcn6DP8T%G&Ca?W6~X4&sZn+oDF%foh>;o9ONk@M$5=q1++S61h(YGaqj)pZuw* zi{snpafF3Ee3Hh3nX_j3+z)a1_PgL}(v2HcMJYqh`rmbkgv3tdX-RG)k{E!HKBH`g zW0VK{DbkgWBjQJtz$UxEpViKYOU!mFSpk?h7y?20&)mAKa4iW~g%3su-*)o3V{%Mq zsE_I3uHCyQ)2KUx4n(|bt|aF>9{Yl4LI{!2T`VH?G3{WLs+kX9&k|}~l+>itf~F4d zfPjX<76%80jL9N?7)+a-y<;zNxPy69RA=RCBbgyJWddcsL+K$ZN~${j7s$ z1v(i_Ww}$s^f91OH|Um>S|?$@yhxS5FER(h4~18;%K|Pz&gVPOEJu~{0b_VV)gB5&(>#gQ}-w!dT6HOcRHx{WI+^D>2EA^`$4~dNQ!bj!e zklEi9klZunn?Fb%BOGXAN6F2EsRxvVitR&>n~1j-fB+fhz29 zm+`TFTegVEfUGq!oCD7I;iJ{x1AgEmj0hljE(`EEn1vkuznGCkpHt0(;%yE;tV2yiQ!4x!8P zI!e1SQHk_&sX5BrJ9G8DO9%NOacKy=BQ2>MS@EPzR=1mq41{zj0biR(*hnHEU7$fh z<3jo`g5?L%AGsF|py||KZ31ME#+ZNFG4bR9{V2rSgOZ z94R-^e8>$LKwGs2riqg)Z5nW>8U~MVE2IlLR)qV_U20Cfo?7T&sh3}lp~{jiY-6w; zAre6i(P`USX?-En+P)*bViYu`8SpjKb79N160YVnG6JKx*Q+hJZRVex2qKFB+H<2B;ndP)`>3(TK|(f@`Wk^q2-_CA!`sSFZBfnM zqpPFh(85TyXHoGTAz)OZ7*u#CLZT9sZ(YVCOP(5Phq;jN}uhBo-<0@#8bso6wvsLL0bta{nwcM#}dC z`Kxk?1wXs)-w|3JEjT;dPpBg}CN)zqp|K7rEtE@K*2agqQcOu?<5Lfh)R&&OtL3Hb zr9W}bhgzHnB6Xpe$wEIrp;RxyTcl_uPUzpFB`0Q&*>Yob?Axf(&V-nvk=S9WlR-g%dB|-o$W%d}ct{B#b*L)5 zh1!ypR+}^Dec7F5X=!PT(XJB2jldgMigD?rxNNq=990<#8z9?KaE*=a0`-e1(T~~? zaBvF>lN6nX3>&5;^;R07@;-c^Z6$>BB9;Qr;w5boWwUar*SPcgSN{B%EQW(yb6FK& zb?3uJw;_urP>eF@=cXs>Al%Xh9qim$G4K0nlgNGc^j+0eI&J^}q@`ERsFylCE2>l9 z*jAW)&=Be>Vocs%T%O-RR6Niz5uZj8t})hI_*2kN8p2M-v3CHPlh$OQCdq7|S>bVn zgf!;#?2$V=P99$v^JOF{$Uw*^(i$qX8qv;zBqN@&UE34I_gN3Rt0joxTO!Zp8^%kS z5UWVK>MBQuwrmy0({hdz!kMZnt?|+W(*|9h?9R}VqP7(k6-kX7{g;vRI=yA6 zWk-Q}Fm@DjL=TZPJoL18)Xue^+#+p_?dhD6;xR!~r)%cM?PkxO{q*d-hRC06z<@f^ z-e$2(K_HDDh& z*lMh3B2BpJ&*e1?syYijl$4;6#1F2o*LuP8Ngdkc18!74THSKGw_COKe~KHjMV51T z+FV&B@66i}H^rCY;bcdkeTZqO4Qs4dt;b3-0dcNk(Fukeh}cEm?zHI*!peupWe90a z+JK=Uue#M;yM{Wq){ftCOcjTqWJ%YQ zpv9yUMfn+Zr3F#{NNWJYELaN2$Q&YzBB=2~RVc56GPI}quBklV(N)dI3}SIPShR^< zz-ifZg94&h`Fdb&c!eUHgxz-JxvOM~hC*?%B{(gw=ppBmP(jo>ZQG9r3x&&79ax(r zNvh!jOyLbIBek{T>r23@vTga}M~B9N1OLMX_#p&3K1mZ5v;KCQytOd>lu|C++9j=8 zy*d|5E1wZ=^I`INhW{mzfJhCJU2?z^17%&l-*v;p?;Ia#5fBP0n9UrBj1IRktm1W=K*zmP|EZ1C)8BlYXHM# z(Uo6SsgKe&N~2Z?$3!Oz=_c&SkTGMn<>}e~RVB^J&TeiN@sQ;EAp)VqI#uyJP%(do z|CtJ+MzoQPf2)Cy#2OwV;QEPh%b3F>_BtFfnMlG(>9d|><9u`o@r(4#UitM-KkviW z6O(OMe);l6;Z99lXTYx3CkEIaF<&Wc7wxsDD0qpcO5T&B4W$>C_5#||$8h;;D0koC zTy!+^@SRJbDUyD)z6@AZuli-TZ-r#dg@i;#VBnkxkAnxBi5w3t&1dGOR^lep4(!zB zal-&ypmbG=vz4Me!fO|&)PbB_t7+dxES+w!m8uB41-m@d!I~Q_-4)WXEf$gNW(M%g zWiOq;I&>hKtsQ@;wH9WCP3E@la%i@4@4kHwpgfJJCDRz{LB}W{^}7KPz-p!cHpK<- zf_Ut3yMT)R^s8F2&os?1>zh5tSuqwi;^$sELA&275PxA~)-QhgWkf#CI;m(f@O(9eBU|g<5C;$B8 zPq(`RHXFYO|F28~vL=g|NN7&u#X#1=9&f>`txv0%avMbpetj`II|K*WkZeF`szT}C zOz169I=+U!5)*;o=#*)Y8?(>bD*mr!NQlK z-X(6Zx7Vmyv!=q`$7d+mEZz_x*QN@zfewi8Ym^#A>-8=aF{@Jh^S zKO95H%JJ17yJ)Az{eALQp**gf6v8zL>ZKFE+w8k^Leyu$TYW0lb%pQTyO!-T3PC?f zz_j)C+n1fI0~L%HoV=l8)lS&lG#k96KMl&c^yg43Zy}vYu<#V_-2ZoVOL=k<1POYm zvP-oJBykgdp555Y|AnG7Wxj5$O7&5lRV$6VkVr_8M|4<4`>7hLKxgwEXl4!$O;?db04EYkzM+&M3`b!b0kW2gc`kN>(*G9A|Mp1 zmi5$&)^Z9bPHsI(S}1kF!ootSsS?Acw!vd45`I@b;zQq14I#7RE}@q+2Y>XgtagJ^ zN#(6Ya05J5WYDlG_SmMpv=L1}dg&xDIf+z?pe9n6M!gTJ3$>0AmwK&V=3^$`6Y zwtlUZQ&D3JiA?z(Yt_%n_Oeek-YI%7qOcMWNCO9Wxb$7@>@2-$F*~ z%^4_p__mv-C3+YGVF;mZv{A(WIJnC0NdnYsJ8kJY>$Zs1a?+`ESNiuN6Bhc8Y7u&# z?O*w`u(<^Dk$HN+UO=n%^q4}(SD(Rqf2~uKNETAMEv5Vm$Qy=_`G44Z^JpyF|6TYl zPnvleXp|B%geF9$iVPK*L#QMfB4mgRJ*9ynHuI{L`?{|4JU_$nIX=hdTfR#^?CLZt3HykK(OZrz17u2M z9kFOcyr1mB$h$NT$M%TbTaFfkXI)u#EWPEevX0w^b)p*N;vU|K9Vy+tR1@LY@uH3ZYb8IO-&lZtvhU5Ji>jX zv<`gez03S4t69VAl|*zN=9XaZ(d&A)2b_BENdM#+dT~hS1Pm+z(P-aI|DBfiysqk7 z{*W{;z&#QD1k}%o5v(+p7M0=j8U9?b9+s01uamqJxIiX=c7G~SY*i*GU*Yshfd&j| zBobhejz^-n{q_F!f6%@QpcZ~=kEl!^m~Cv_+$#Z9Bl!CJf4s;_kaMJ93ZV1m0q;yC z^l{DauK&c`!IQ(25+z2(N*;s*q!W#V!T^zo3H?(tg7ki!KZ8Dw8lhIA_)io?&_eE^ zxVuU_nf1!s8G7b0=zl|Il(fY|eP|u+`)TDIwki?uwn#hzrpJ8(r~~LG4Y!y`ffJNy zU~1}tef}%&RW1T6CAGL~%KIR6gx2LBZEbW=Q6guwzC>KF!oM;{Zjglt2 z{@)9*z{ny?LOZ14De#;~12l^I#DB)9@l@WB-Ux9PFg}KMgosI2119$ zXe9jDE0iX1&Phu<30_FPk30$e(>6E?Ca3xR>AvrK$%TxEKugG(NXsY9gm}Q+ymx~o z?v5bsPpBB7xI&;Y!tMn6r+EFzdn9ch-GA{MB~1VX1WGeeujJmkYybXrNTyJV<%2K` zx2Wt>?Cp!#z~Kk)A99EI7SW^>6O)d9?69hGlP=DPbYv`neL?$9fK&&ey5JN0K*=PB z8)b^{(L?1}b0iG$vnLhgvq49fid8Fw)4cB90AVjO3ZxR1uGv%`8)Y0hplmWmf3GuU z2_;R_(rRk}hNIu^9FfaWk5hY#iezQ95-Uk|_PP}x-0~WzMKh6BftkIpsB8)n49N!& zIK4!3UMSym9p&gx6F+t<}iOO4> z{>2CLCt+$rZ?O`u3AJc6^pjpvQSlY*Y~j>JZ=<~b6*}&)Lc`#n6n*@-64F3KsDxh&(eV|xF8VpgJsDxTx@DV#j z?AgEJZ_t){4`L@F;ZIP|BPluQ3&Z1*f)EsX{#>RS7KWfq1oA=oicq;=BEf0Uy}ngB z79Iz6vLY~{Wujw3h(*`WcLw5qD=aLGxQr?*+Tma4w-h)7k@bT`L8ecxi+(S|DE_!7Mb(#MFmORC{OeZx1TwLnD~QCY~sD6lKVI^p{~A>anqR=XOGOMr6%wGLUh!02I{MZvlA}MOEms-SJV+}15wQV1k*+AN7MO&a4+(wodN@~O(^hBE4MaBBktNxKVJu~Ju&ORhmProbzS)z zO7@HJsoZJse+M`fS5m7Ne9@iSEX4jg1B* z;cd-#nYTsH1MzVb(#<(FX%Y`%6%gNs-wRO)6&J)UvY&zDyd%OCh~CH=qs;GM^i^^) zB075c>eU{?&ZOM}NX(S~By;Ckw*gPo7V&RTQ(^qmV-4?#yzd=ITkf;XP7wJ-AhVQ2 zfT+Q|1X*RVDl8po3c-IE#wTEXCT>Ohd>neZc>+#FX}`MkfQR@ZjR#Afty;B;XKVtj zUzH_iZrIzm=L2UD#brD64xJV#?XQI@g7Sobh zhH8vDHcuSD^w~Nf5b7W;Xw*Q(Z6S^HLjd}2$qZY-=XjVX>ZD}{sECYc?Y%5qJPzc2 z9(4ZHz#@#M-+-2391=_(WMAqql-_ySN)XH0>X(vYnbLn!RX5i`-SG0|DBvT(-5b;D zkdvt6$iq+@2ZzI;itc%8M1T|^y~wmtk^YWdMl{Pvrvl0#7XW&Y!8IY#Z-{6VmKUd= zl3f1Y+S2mqi4*jIbVe&@us(Y)Uc7h)E_1ey5R!0ee*h(CO{+j9kZeUbxFXn+(V-5_ zb0wk}ZbK9kU^>7H6j~TZOmrT?M})-;wSlJ#4)RWR$%xm8;wp+I)9p?)wP7NQ0eq8C z%h^f5bbTW&ZApZFW% zKTx$IA~0whf@nwO4ecHP9fZ~c1ehE&LV-swku03F`|LX-pruHrh!QYq%R`&H?!^)Z z@#~lsXy2BLmKF+7*yG+4Q``sxP<@TCE-F}zk}rUL-u*S&i{`5=|J(`}JO-wdtwXj) zuoVI+kn|UR8WisE(buI2nUHl5dgoIA(H3;u!KDdQ za9z_cs6$Lg55x}tVKxp6Q{eRJsor;HG0dLwcY@7*uo$j--;IYDCaK9HlOaMk;IKds zQyP#9&Z110@n66{X(J!7r(5jvkwc)6NJPC!lMy}xDINwCp-2x7ww3qe$G_gT5ndo( z*F7X(m~FAb#bsfsB=)y@q=8dRBgn4+O5fq|e@8Zl0tecONGwYVF6ph#tDyp=G}Y^V zirlg|9Y`nlQOX4I>UXaH1>CnEvs2{~Vwlv#(0>Raa97~8gy;Jf)Qw-5>~R5%YE1Xp z2yZ2`CoYj_v4fm(Ht(hgnTUaxwR9*bAm9gFaRK-cMBADI@&OaGfZiJD`;nL&=X2)! zmH_1S6usty*MQzwDkueS`}Zs-);8e zLxjDJSupppmAWK3GG5PN$O{zQcc6bkp#ArY4qSmmm#ER+fe~;ouax@(xBvZ?Lpo4~ zWnPRUjhRLoT<86SM7IcU@OZAsS?p1*K`EG&S7oA^3i5Nh??O<31pcVcYCF~^poN9B zMBva6R&efdZ3kiGR1ypeVlNn1{17K-&;8fuP>v<`lPE5s&_RUqi8?dTN~kL+UEk4z zkk*n#u-g@!6SyosQLpQF zPI1yv5}m?-VT%yH6H)#BH^UjDUrhC`KSakqTBV3)Evcc9hBxp)_g(t%HpP4y zvBZ#(R^tuh3dWLw#~Y4l9g}wm%|?1mWAPaGaG^{{h^K%7icq*hSI|nJsDvp6w;nz! z=TYC-Xxbcb3(yBb9Iz}339>-o7U`jPG~CDA^C0#C0UXihxErGsKv*FXjoNR_9w6WU zry?lgC#V0XH1o}e@i3&|jz-G6Zf2cq&zboLSPk#QgBG$Nxp zfP|orZnvoSYrL5v)S)d&K=Lhxj&LN1_B>3whoWOL!m~EiTP<>&L0=?cI{0Z($jTzeK;4$8V04RBQBWQ%0s2OuKO%rZB->#}R+&LWl|mt0Z|}2Q zAQ)oLkc6lK|5{yAT1$7xa{F5u zB1b?>$h5#O#qa{?Cl(Bic#Wl6R8*Don_7$LXA3>;VR zL5nIY*J0m)w=M5I5v>xd5aTO7gB+*jhLSj1>uQyLZKps8k(z#T`St-$v1O*d?sx&B`Ckdg_|(xZip(T;yk822 zLm=<>F)8&Q#BWbePY*SpK(bD{F5=XK(m_VIt62q!3mP0!&#&MMYyyRqg=Dh3e?1r=__iCSK*P+lY?;EAt$07fynQjiZNZ zwHN}r_*%xJ0MeE&BB|l)N2V|A7B+8^g7P71OhA2?0uUy8jc8SzY6L{Kxgb6@9+RDJ z1K=c#12s{rxBY^!h5$uw~0VM1Mkosjc2$V6yk#-!_Oo)v_2?jKF zqUnJ{d=!O=)(qAl@N5xhb$#3s@Glnt1&4>&-7Di>P{9F#KuUm%?XzRgrLeHeG;Wi0F2w z$Q%JQlpw2w*X9&`L;$QF7yAvw>4*QsNqBkbBh^mg#dsRvBf#FzQK%pTj1X(?Yjttv zR#%b#`1v!4KyUHnm6<1vDn%@8h6`mVQqaHru2!)!pKoq0{(t(KU#E{02?6hy?_{ge zT6`_uf0On**!IFa@{NaEg5%K$4dJxE3NmVGjp9W9P&uI;j_yA z>G0n*x^WS~S4ARsoxbj^A(!vtxvf%I8W>j3p+(CicYjeis13P{Ma~{?xo64AjMyiD z6ag!BED-#IQegeU$as(H{%lqkR!$Ahed+r*g^b({HWK;Z7_i)Dd%{}NQ7{?nA$+X?@~A6K8T;|u;Dy~FeV|F{1UM%E83}0)$k6wKLb~5p zv;P0MFpY8Z)Wz-<{Si_c=*JWSzof?Y=ej@8u}jHucqvUbV)j+i=QZA2Oe!?%DrLWa zy+Z;K+RydB7B?~RE7Jdh9b~cCSZGZq_yB&qJr`kp+DGJM|7;WTkv~*C+9%km)-5gr z?K3?q7GK?e_G_G|`IEDctS@wt_l^;2|9(kWdqFteH$hYX?lnKmL-QWct!DgQUt|OV zJ?K^P`qxRF5#aS3jpN!!y(9h(+12!@d)v3suR9LgT>mE^Er1pur2mNz?n}sjl{hWg zYt8dw)p~LH{dM@N%9%f36>0eD;HwnVa{hB(S)K{9S}_wcf`>-U?q;#{c3N!c-mt>y zxYPB`jY}G%ua=al3O|odzPFWM@Jmtt@-+|FzO1p}9h`UCL|4afC-c#bTLynUT3OiE zUb30_@S4VG3t7d>4AcArt0~`vEVt=h$}Py>Gq>#?GizbyOY5gWhc{7veeb2ab9(LZ zk<3dupKm@;u;G1c^N?3!X!eG8gPw(d^oQdm!z`9pKJGdvVzsQcCUICMU*LQHkB0`I zJa-APGb;^B+?wl}=$6Pok`S-s|G|#w5a)`OCF>~^7tiSp3}xHDI5|&VoYe(n%CqCo zDj9q4nGF4|nXd00@Z!?V2$bqEBXhxT=O{=S7>p_y=?DKRK@5$+~+t2i+dw5b4`VYj+Ah z{(1THX)%{sv4uHD|G^(Ux1-P2m@*COW-ZUI7@$yu63nxlgMVYe(#%;IQ9lSRFO?|W0=C>(+yUXWonCsY*f1K*x!oK~aR5OFk z1w#wBlgs%pXr&uIb#```ufJbBrK?r<^8MWJ8T;Vb&3IeN&$ZpC3uVYXrdRy;rw>WB z{b@O3@O#IYTTaS$%CDgZ3-4EXF@4)_x;${qe5t`n14;o_@XFBF9U)P6Xl=fw5L*~N z;c;#Jw>m|tE%Z@})3j}Ht7TrnzF)KYUC&kGeqm8s>b)KKn9>?+-n&ZD7rd{eN1)Zn zdG&VQbBUW1Qw8DFWt3;l-L_|cLKZV~48DqdM|6f?L(Iy$7C^s_x?ET$yBWPPrqMhjm5= zXuDeY??V@8r;%@1Y)+*7Zm3(r^Sv$^%P1;z+<}u8o;m(EGBWxu&i5IXK6$ITA^#ug zk=|1x4?_Q9&_BGh%a|dw$#Ln|RZ%f*+RKH*Eq0DxxIEugZhmC7M=5hm&6`?B$EK)^ zw|FYbwXQ;zrp#;0JrlQ&F268Y+@J#ryPF z?U^SFqxXb&Ed9Okyh*a&#e5-d!NJ74fl`n|6^W#;nhndJ|17!Old{u0&!kMfJx^3) zvZa7FI7vP|1tJr z!^2No+GZz4DX!KoA#{|Rm6Of>X#LJR`gWAz)s9?~RH{~P%GE&+O*&EH*!$!!;4Z#= z`;{1Uvzxn^V_26MuceEX60~TPYVt8sUfRCA+oe#kbclP@*WP<-Dg6n0nTBX=pslv1 z_q!ieesBnOOKmI<*Zyi^A?7!E`|611RV-FPY=r8qN$N%Fsxkk&)Za-kkcXC35V{u7 z^7XYf2^VnAaKMj-vLzQd65|tT;&ZL9Kap)HT|SKSOec@!;qUuk#3pkY8E*LVjK@ov z?35`?xg*?i^5LmFSWoO~{t;(M4S6T>vU+K*3>W9yCHOe}>zLw)8&7VZ)(av#BeGep zsIt}TCu}#i4Rf56a^i=X#NKmEIEb5tQDkEu7YPsRyqS$B zt{%*}IlumhQUANlmztB4U%%EJx31{??a*5{rBuWh$C(>=zCy|IOun^~uORP?@=Ui_ z?m;yLCeP%%D3PiuH}+n|=I7dtXOqVxh1VeGVTh?!=ne#-ok&e0 z-&vL8LWV_Q8YHRc0JK?^GYKuB3n1QtA#@z&F$|P+TRH^nc`I=KLKHrr(^#AEmXwvi zWI}!F7@PNb6#UMj-3rt6JQ2^nh45ZTBBK0aK$$I5&S<&E_=K?&CHJ7hM#g#)@mO?a zteVRt4YLOijz-+Q#p?LEFc6F#{JFc>ns!6z-!1-V?}cVP?ih1VOAvPxJ(;CLkWXQt zTtiQ?utYYRZ77iNeFq*f>?UvnguO+`d5|^~K}--p<63~M?3|oG@h`~IAe;oFi)2SA zX=j`%1HtD6dU3#_COS+2+{nTZu{H>PpJ^XLv27_j9*U6ad(75*ar}Jh_8+04H#6WM z766_1Ni#AcS(Fp`f8`--C)o>L>Mm5y7sdgekuo+BVBNp}El{LYH~CQoglk0pj!o|7 z_5+^5%1f3RZL|!%cP+GPUrF^w%i4r^ z;gc3x#!q4m>qZ7o3tABL4hf%Jh1qDBD)j-VC(1nZ4LkX|R|MudXA@2!AR7=yZ|_szd0&Z%!Y1kg}@LxsZ&;#g_4|;gJy) zXkMBPZrjYv9EEX{8bm(= zlmc;e)tWWct@9JYs4NY<=`uno1)LN=l2^8EE|nq=4@D0X)cvTD*j=A0DwJ^0YtVwd zal;0KgM*i!Pj;Pjv(RG$T%7{hNM}(3=hIivfW)NCHt;RwIEb5PkX(+n zF3f9!!0~CnsiJi!XEOWT+>lEhSjN?m{lom~<3J&60ky{hBiY_Xw6BjHKMw9@l&0Pd zjuJ~TslRA%%4&XMA)~m?Kg!R{Oh_#^;fL@(FXgR=485PtUK*kM8FuY(+ z#A5+oQzHS;vWVHAL20CJgOb<<)$4rFm${s(h3roiI2wn1a?Xf5nbnO#aCfs^YYiC6 zzTVzMGR+vnbmR2%^H9GKMD05>BZF)2-p|^#xnor)PhQ(9lJ#+V2C}HaXaeTKOS> z%FC87Hw78M%F^=V{%yQ;OK5Lw)pKYP9t98~q*ORi6Yme+$7^sZgHix)seut|WB46r z>@}z$n71oW-MD#k473BZG*=JSr^s(u9A#SuHpp$@qV(=%Cvsy)P)wB@_p;qoXkvCF}HDG>Wnv zg9rFZUa7SUQBCf{c%?79y~HjycYgo=3{9AgIr;bk^c+xu16z&X7@2!Lgz1qgf-YPQcK3Q7ApE#mPqZ;6?dGRTuJxX z4kRNtcFajpb?8e>PiQ>p$MXW+8^)mS*Vb65)bHgBQZ};7Y?fuH-K4?N$|c`fPO)txr!wf@{eVg8}Ty6ap+9o zP4KqvV%T&nc~mgV0x@?7M>z`J&)sl4Sq|fIfUSMnhAs3sb+HG{W}v%ybu~C`h*>txu!t>c=Ypf0uLEqbn;40 zPDbb1bE0evda-tneF|uTL5bHstp%g>bie@_(tr!+U5zR^Ir`ubP%+jDJvjH#^k9lSeY@5aEryAq?bu;Spkc>)c8yE?Ln~>1;;rqs^7VoUea)c{`?A1 zew8s<6Jzz_p|H?!Ss&^-v$L~kpgNF?!&qB$33#Q?$4e^q_8COrF3tGs2^sf|DS62V zN45PrEGJ&}OblRlrk`%X_s@utb*(S&hsdT-$D7*thuwRE81VrL0+ZS zq#y`^sNsQt_JM&!ghkzG=k8Nhp4M``X0* z3KO61j>x(>^@?4Kb;pPi&UNjaORoNR48_<9u^XHIZ_1F=t0fC*(JGO6E>i`v57lrV z@ff)jgB!D(T{Dz3>WOItl9%_jYu z&+$S&VNwOml_-X2MJ6ndl&eAk2?bLAsp)AGI4t*v533Qnj-#2+1cWU(WItF?{K?SM zVa0h-BlFQWz|3DL_*KDhF(FVv!`Bt$0=3a%c0E-cPe#|Wh+^78S`~QIhN>g;O#UnB z{=5s}uy04MeJ|Kv0voT0D%SKwqlFD7%lL6AA5+nID-6b?dd!)laHbz7@(@HACLY-IQ~v9<7_sBieQqwYnK2q46_lBe@x;CiK+Gt#KU5ib5)Q<9t>7E z7WV8=Jb+(gcw{*+knKL(?@13ln0{jupLQf@Zd9z{PCe2&3= zYQX+sMv(N4SlRTeff`zQ9F_M6|%A zLZRF@Z4IA#Ty^eyt@1xHAEikckAnyt@!`##qiADn9jP*a@x-qh&3I8rd1Hyl1fjZC zUdK#daAsK_A|;rO%R0AiBxe}yq%A$={NKK}7Jub>^xv~Vp+8D&AoAQ4e!EyO!zW##;ycb}DG3x%?ejrJVhccl#9 zexxmKoHjl_c`&~&T{SPaZ3gDHw@)*_HqE+j0QgZRgu}^7EexjoMMQY38tsG-djz3*lYgtlcftA^Ck6;OS1AP=_>^} zoh;#3FO2<5Jhgp`i0Ba>PO%e?`StNd7w-8vi|hIYX=q4!Pz1L%onr7z&vUFO9p?3; zJ`JB|ok%x7!epr+nY-ZgbE5cTW2=shXr1>*-`s+%uQzecnjbHVsFB4Ld>S25el?~J z0`gVkymjLPL9M;tY7G>!o<=Gh^{c*qMzyB@>}XQCs<~(SrO)dFbU*k0J|^olIxQ@k z*OsyUGecZexVuCCSDkY6@=S@=)>G5(J984Z=c~lZ$2#0BY@4>M^}&Wl5nlNw~>ezc3x*M67s>1jdbbro#nDI`|Uol?q~y3_E`#3$O?x3R5Q zwwSl)B6T9USzyS$#pGN3bHzc4SQZc4?&+Gotr9NwNfnd&6^#Mi2eYPU)XUZe#2>6Q zQcKbkk=L5k4jygae&e@&%+!}oD_a|l>bqP{67AOF5C9vb7*{Uo<4=bMIv(7 z?CS21C#Ek~2{&XrB%c;Zo9xSdbxC{Up`CKn*Ih1-?dNU3^f~IKGuwH2I;mOIx~Nue z3%A}&w?GcflltNW%b@ok;*RRd=@Q0;GsP#EIOM&&T+Uffmsj?3$Ja#aZ^-n!_Gw(C^waQqMUL*u{a$f> zkx7`@i+edVS{WevYDzA&d@?cELnOL)Al~#t&aRQQV}8XKD@#}M8rtqR{d{*nPp9T7 zYW4ioMBrB=&%Cgy+^rKPS9CM&qRoVL#qHE}Ic=3oHXTu*Zk;fXQJQg%d-`kh>2E@B z9deRG=M;`?8me|{+mz5a*p*q@Gqa518cp+9b{~sMjth!bpXq;dmLpW@wRvl*m6y98 zf0W79czJ4rM5Bhl{8)I|s=SA<*D!HwNORYzSP9lW&{yJ;;L+m?HMzjIlq2?%l)NB= zvDv{1^RAP5z6s*I6EVkbuc|$uK;0g~Qe$@QpkbsT*GS`sa~-)sZ6cYXv0r~H)_w{- z&A_S75SQQ^&z*I%(<-ULPEk~;e{ZXlzPw6P-r?c_57qPWQl{&;+-{JpYnjeEHE@6$Sx|dR$p0@*V`UlY^@%U_H$^lll=eG*#26Qd+vAfm z7WXyU8{`C88_unaUVq(cQr_mb?DD>ju)JG&3JP*nQC>nl@5(nQ91eLlT2jv2q3wQp zoz0Oh&tM;Rzj#&tMv44hXQAXF+dS+2U6)eM`Ud*OO%-)D_GfFc=^b|7rFmnh(_ZEz zv!9QLPgw?Aaq6nF>PR*D%pu<<1Ed>Hr>|ts-fyw3J=U(PG;x+*|$Po z<;;R4zgF{M#|X~4z=Wa8H=`eJ+Y;9h6*uitt?%~_PlrQj`zNi*f^7949qlGMtC|OH z%CD5CvZs1zF(@(!Ei~mcRmUhrm=(9piM8lU-dlUJ)+JM-#uToOd#&YPVm4{~b;}x#~b})Ak<&&+BaSIX8JpmWH#=&mB+d%GA>Sc zlU>NFMvi zC>fjVnl$cC{t|tS+4UfWjxDFqE4Zbnu{mIiyd zEj?R=W}Jh^BRJIppZPc{c8ZS9Nwl0!Khm11{>EGjE7*JO+YhrD&%3OC*qCwY$)R0s zEs3+D*oL!Ai+<2+cH2xm&WW4TcFr!g^0)Z>hC8G-_qMsTf#ik`N;2}^5WpH(>;A@7UL4|Rfj9$*AB@z2gm2b=~Lv+ajwCk zYT9)ASRl=Tr0eKrGTWq?+YMh+?!Nb2Tg)LvqHiI9{bS;tAP2=hZX-J*zO89;UxSqD z@+;Ri8GNH*R=yJ_wpEyX48PbvQlr!fp4a&Kr5SyN~%P`ENcqhPQgnCRN7E z#`hFNDV$XN8Yo=Lby2-`?l)YtiBL>HR!(j4^jJevR%=N%cl2n;re;nXAC9l#h>c7? zreuG2@hEJ`T@%{?*yNO?u3D0$Za40VHb}A84F>(+2b|3Jv_Z z{=Zu26l`7CIoQdL5pxpn&?&3a?#J(YraSsZV*7pi40Jv76Amsf|BxP+Jl*CoCx%|G_(OK~Y385Sn42sa!6xU-M>G?_*&VJHb46-PLdV_dVy?p7} zj+tFOXNx~E=NlaL^`CN|V)`UhUR)%zq>Rq|Mt?_}m1g;&mBwrfobz^Kp}9AF$Igt~ zG7iqW+F9jS)Fi?!x>_pJ(zXvi1sb<9w|4dP1ysl*VAuQikEg{)<_j1olwa4r8yDhL z;pe|3oW*ti4+G>2Z|un3d^ewsUT|3inM53m0j(7LqWStW&(spmDgyw0X}6}`2OzX5 z@UC8Pv~weNO4^@{GaN!+i5V?M=Q;yXx{Vt*UK-l`VGpvCuMoORfH*_LkyQY|*H04U zhWxDghZc+BSNm4{n=BOUq)SMlhy3dQ{=@(D>q*U}rONxFKp0<(@nT?P)m|mJ z-h=fVq?8bp0+{$KFrjJJ4NEC!elMm{FFnwF(E}>0JNlj8SO9nuO4dm;8Gm+h3FS|X*s9Xt#ou`}b&wOCBn#jGoP%f9ojXx+0o5(PK~*i|i^!e11rA17!h ztiT)?VmL6e(n|-Lt@5GufKK}nS5;iwe(_+uj3*U=%F5vj-rpmcWDY`quyf9 z*Uy7_=V)xGe-m_ylR#3ftqX7OEu}nr98TLy=QYvarHs4o_j5A`6s^pn)o=yH zKUXlvyoUNWT6>wt2b&uij1j=*EN--Kx|o(~ra><%DjEk+1o^Hn3%Rp@-%R#;4uu$` z_O;6&oZZX3cKhKk<(L7uMe%#%jJIWrD$kdu%O4%Vs zTl$S$XwNuN}&|_>o;Yp#~Euq7U zr%=SyxRx%Ve31MvO}cU2|7|PvO2m>&+f*Vf;vKL50LwD?N7==DdL{DGG+QEQI*va? z(rAdwZs5yci1zf4{OePNHymY`P`*E31P7j-ac@aG?QuCT@s70-xq|CGdfKjay?G(! zECS9!G5&+2OCoRl?xd#_?D#wZ^Ni3o&-28ID;>d;OdHaaoS9~8#!||ur|REx=;nqm zJJ_N&R-Tv@cuiSw>#|sb4?6|T>nbld4c}G{6dtf)X-=0FuKL=ppZnxkiKiysae{$j z$@a<|0_IpmAL%e!v3SY5}_}CS!f#n2#ugd z&NUXL88!j;E0tMKZ;rqJYoC18askQcz(+y70Uy5( zeQLsvzeIaIuD1qScO=CY9dGad8F%@uuA$YnFNFaXpPEyjmSwCqs9v9Z|B?70;e6SXfFLk*8@x^GqY46LwLk@3D zbZr=a)%k70LyxukijC-7~*l{ zg^SbN?YLSCcKO}~ui-$k*U5QLC1l7`-ZR=iXVe>i;7E4&0Eeks)S}R zljg#OIPD_NX|vM5L`1T?o0+z?cl7O%?-d`pk-Iq&WN>=RwbShE)td6k=H54XNC zPxhA#j<#BIiy9fGxuE3fdAY1Pr$k$OVuy%^QhHKyh~(8TUskGCkM>lSe*W;~xp;Q9 zWA$0}BrREHkB-nTddjnBZJ{KEay`o8`CGk&@kH$Cv*-kwXY78%jAyb#ch%O_@#|N0 zT4->+G^ZnJ+T}e)%O@Ttbn9e3Y_mqKf za*R`4V}f7WLmzh8RG%GA2h^}1Ur+j<77|&WbGS0BX8)43SNz4|*0WD-boL9dE~~wI{$SEM zsb@*L3SZ(XB&wd@4-wsDr)}iPs%rI9Fo-&o?0jUM5)oZ12i(=m}@9Pms>x7UvV> z^=f0i5gZ(n`bbB_?#*4n_!k_hM%^c-*VdR;o-mjn_cl4ikTuYi%Szj0f4-8dbwumM z*=o;36XvFx%dMFisl#DCfUAZ2`i`t2WV@sbmOBLVUc2~$HwQxG?wF?K+=!o*Wifw+D zyPGffk0@Q{eyzvBQlBn+%Vt|?eT`t_up+B?!eohfd<=_@dZvbMKuB{2TG?vFgH$CQuz-5iLs39kRBbz46)vi+TDpvZ)UD07-pe#pZbvZ$}>531)I z>eO7>82M9zqsHks7dii4d-9$=omoU69^u%h^cY_CsU zNl&}?ZFjFnm71whrk|#3OlG9@aT-4Cx2`X}Wo2Wn`8!?Tv`cB*_0D(u=U)Hh&28d6 zYraFqkLnwVjVZ4Z`>KOOzLhUDIl_D0q!`FIZc7#m2lPVie)0;Z6|3w{GIy64da%on zYrn2M=o8)7y0F=prEdSFXDX`H@nXr=Grt?usqC_{sXO|F<>*RsQ66QN-}<{So?0C4 zXKiF!q^U1QRq!3D$(ZcY4wDx9-60Qon(Dxct}AneH)=(8uE@cI6O z64f&Tk7{*xPoJc^GYdSA)(hjTV;gj^XySV0oRFF4DH1)+*BGaL`@MGlwa}jzJ+CzU zteHGSZvEEBiz#ROSx61&e+BkyuqxS0eSG+%&|+BBvfH&(GnS`ygvollojA90N~L#J zs4x3MuOJq&pRWYxweQ{YE15Soss||e8QPZ4@AKr{RKYtTg@J(~9dJ`(?o1UnEvEkK zH#(lrSMVKEpDHk^JuuSn;x7)p+=MD7UY!!oS!WFyF)=fK-aP+I%gMsJq^j(Qd%VIf zcPDzj%2Hor;>WD#vW+~Mb55&@wEtlLBiy<|B^QS(+%p_wYIm0n9WJjaJ6*F)D0ATH z#TJg5XX@GKjAm2AIdasJWrB3f1+)u4T@Lb3ZgmR5Y53MH{_CIDRVVA0PEM^raP*5*BRML=Kuav^ z6MnV#kFFotGgIWrTFVW4uzW&-LRlrcn5EQ5M!zsI^M5nlK5jYP(U{C3f1!16UH@l3HST#QP^^N0Q;WSg4V!idPsv%Upy4dLfa=jPwJNqSi^KQ^>hZP-0; zK6F^5SM<_5T|XZeC;hRct3YFT}l zo3bS168q}TPF4g>1agiIg!{H%>U-m)BV>x$N&B#MU|EWW^YkswySADR+9pY<wC|`nc1t5o@6>^RszgoDXZJWx`HUCG4j} z%0k2>>PAV7^CuVk(a{%uals`vAzgoqACJ!blQxFgEM0+#E6HE?%T^l~oEc{kS7h*H z2%MkXReR6V-eHQ%p-za!^P8blE7qg0Ah~h8ZA3<>WX8ByGs|+fu3|ZVX1T+a3VxY^ zZx@|d9Cs>Y3ppom3bR++RQ@hrJzWpyR6~+qKT=D;Gpl2!R4s`uStXAe`O0spT0_1{ zQrB{XD?bigAPXauwRSGZJ>v$KRbNH#a5YzrS<|7$%_u8YM4fuUng+F5FHYs7g@#L3A;gAuN%j{D`dtY8y z=l5Ozyl(B*$p-N*$xU_&xw4*>-~IPMSmh%%Aa;|3SR7t{tJTelXj$HayN}fPUlQAeP%``o^5a%h#x zNLqZ>wTaK-tgOU7(ayS5=$tK^230NF*u!jQsY#6wdx(fLN(nKim$TGo>+~4My0>Ji zGkeGkIJ(6kfE-v@IIs^dZ)AcuQ9;n8)%wlP7LJboi;l4=vME2Qk-m|UjuP0Z%eE{g zDPq|qpovw>Tvb2Oe3Nqjwn1&!wnq&%YA`*X{mqHQ(?2({VH)L+&Ejh~`>6o$C|)^; zVUi;#HQv;7W-s$6rvN(KuKSNSD(af84v$6Qi)ma_4%4aav^;?Y8AqNFcGe&^!Z})Tebvfk4|k*)IVI2H8;GZXl!iT z!s{)KF;|zDInLNjX(po_o0HJkCuEWMbk4CRBg9EyT~h0-#5ln#3FE7~xcNKpNAc-h ztrNqQH~F|7>4v5LzU%z|0`W|g2>s@!`&E%BF@RHdfa^@Mliq!7hfl%A6|NgNw6k&>rKU_fE*U9$={z>guQrm8er&#=9$vTJ}FyQc+FiU+6qSPuc16i?WUwFBpmc3orf`p|Ahq zBu@eR`8|DlC6$kxTb>HJWsaZ$@IAm3bU9xSpe*4J%vOv)0f$%wym~ORIywwTvp^{Y z-R0|TcY2D{^J2q7vOxWX-~|&V-tD4+g}wU&PJun-Bt;Onrau=d10fl7VDlsI-@ZKx z2D-D0z~ofAFMMQN_f2D;?0+XphzKV+kYim`hNf_ zlK?`Zd+t@9t(BDuh=J{xA67z;Oo_$#roY@wC?5E0?Mb(w8 z2J#?jdOZD@qX-F-dJ*hx!xnO_+?cWQ+P|<6$_!G;Hknu6oV1o~^ z&`m``Aw|@>^C7rt;~oC`4M!M8KH@X95Z7Zd#4Uwn3Zbcrhzy^9Q260u3S?$`%7j~A z*==mUed=TsjxxdTkI)cWsas?Dy8*r}nPP{nOY0xG100j}PO|O`J6Xnb8Zv2KNCLm_ z&yTHJL6MqYd>Z`0|5woeyrsN7@hZc8m`)Y7lJ&LWmECT`q%J4=lwI{g2Sh6SS3mwa=4Gr z#9Ub^V6$dROr)u1n$6tST*2`C`OGmLb&Ik4BoOwwC&sNLz-qH~pISM8Qp40&`qK+D zf3LN_@XxVOM^Gw{pW+W$&{e(X){UfhS#^@XV96W(vFT@eRsy}TwasC^(f z#qX5wo%(#`QBPCD<)(k^qPovDanJSju_Ec%Ew8fR^02P`xzl*@7&m!JF)@!GNmYHn z`g$1)0j&rm(3 z4p}x%1GC1r?MZ5jsadei$MCo5fR>u*7MDBN?+d$<<+Eg%aH&NJ2Zv)7FCUZ z=NW8@7HWWoZgWCO7H1-Y+1(Vhii0|6nsy5v$_9XIBCW>)SRf%>`?!( z5eOU5y%9c5W$MOU>!85bjBmDBU#Un+nBw`}^>j+1a2b+sclR4N`o6wwuC2LGeLWlD z(mFrd(nhdf`hT%syeN_oWR4pJ4Q$yW{CX;gskZLL9B=GXe=j!YmY9dzdV~!8T@IXl zKF-tgAy0f&LgSh8q@;RJyXo!=xsrogNH*h2(|Sw$+Y@qsT&N$BzRKu-J0fE1Yx6LJ z&e7p$!;hcUQWcxc+zVFc7xPXO%r`o>I1=Ee&~@?9afMYG))@X;5h8X|iZy2>m~1Jt z>yc4jTjSbIH51!DE0%Aa9bxC#qxbc5iVHy~rw1-2kVSn`78*-{wG$CLnI^eOL8Aw@ zz)5U)S#iJDm-l`r54BCLv$}Su5Ai>M{{P3+L&`JYlLmutd{A~_{+u?cbR68{*%QsA z-_xP@)@qb1&Nq_1xa2*cK|ekL`Dydvn=FqWW^`pPp>VCqIJFsWY~RaW$){9Itf+Z9 zucRdTb=c)e791KNrn@j-BG;Ee$}EA@1cCXfyMWfH>>{Fp<7e}aZaqvy<2dE33O4Ea z?PX33KBBWNZ~7QjE!;QxeXeBPH2crD0!k_;PDIt-l#uP)RGwNs{M+?{!}KGU79opu zc8re#hM%{%7{6}WT8Yi`+J2?@`1GMnX3ILqwiCu<24&+mpG9gc>NRS%9;%&o8{N3c zTKX$T6V-^72oY&L#g>(v+*KBh3sC#gH zxmuor`qY!;r)?QP#tBqB9*Gl~%#B5~5aH(#(b~fx^QkN%rc^C%Z>s9VJd<{)}Y9zV>yX^KpJM(d1NOg0ql?Dwxe5};-N`%?bMdMxb zaU_Y(C6-C%=5Z}AKVB`_h?JQ=v7tk>EsRvU9QYcVtQ;%X>j^aimLBXQr{U7Sody3D z424-%$4M6bw&ve1vrg3Dy~;=BE|$3GUOZ8qR1>F?dLsUi+JK53x3YS1x<+hJi-ze? zRYNvciowv%BChpp2iUtNW%Ea7I-?T3GprQ#DoS=Umf0lrS+~>~e=0f>);CJS%`UB9 zj1s{);r#F1=>OwCWH>>CJN4d8sGof3N%<}ka>U_7_X^769$H9Sz~eNjr5~}IVz2n5 z{Sy8w6(vh6{Kvl@X|kdHdA5sgqGi~1K~ehiqfo4?T=)1^%H^HdS=d`^A+_|FJ)rFJ zZJ(XH7o!LY<>eAI7!W94_J96UFC%&t`kzDpGQhu4N~OeBF9uu`$~LJj_l^F2?DKR) z#!cQ~AgykV;y(`;(Hdlc^k`O)rTU6H+Wya1_Pu4#{O`*X`VKu>Qo2t~O|4XZd}}ds zr%=ARLTQal)^q!b5-Iw{X9Si=hbBA@sP9G??7aOS_fJR{=ze(Qd27vL6hXclQ<~J5 zt=hOxl>Wfs#kR(_2p2QUlRzv;e4uORtQG z`1he6v+a`5`;;FR!xHb`>-AwpN2!le$E;~=-xp=8k3XY~7kZ<2XWPk9qt*;2s`$nC zMLa_lmT^#~uVgsl`T_go!RE|Fs&h^C%d30TlN$FK&SFFODSbS^qa!Y5+BvJU%h5$u zh>@|zv|f=J^vnss$clN^X;?GKKoo#PIjzI5jCtb7I2YMyVEIEi4`sJ z$YCv0E#Ud?JF8N{1x4bN=bu?L^uzv%eC*x`yQI3h>=Ox!+Wp!Ws=M3wywkkVQEg;n zp{&e1{;t8WF4Ok>Uvfu+_g=kfUHS0Zg9nC^c3<}?jb=7Fw8}dCazFUB(W`efR9(o? zl$j~84RWyd9?#XlKG2v_trrO~h znNQ>Nq6(cF_xV~0F^#6qYQKDYTyPeK^ zQgb}XI>|%3$4k(uO@7y(-6>`Dc6*|Z6`d^X2ry(hjux7`Znb^VFxFFgBDTrBR5GSX zrY_O({Qb-t&D}*>rz3|`@fnYu`!rjhy}%+cV;-z_PwDJ@7vQffJ090HgAr4JA(hm# z8f=<7b0fR=%3V-on}{yU@o?iYLup>tiIL)x56OLI_cCX3J9jt@1b11tmKJHo7`*aE zwR-6rPwUFqq-Cquv<=+I30Yd&t0yJdUALW2?O2jH#_Hx$;vUzB&Y-FOm9e`PxEEoUzTs!>Bn;Bi4*p!!`T-k$KRKA z3|R0ti|HAd+f=GGOV80vI4N6O`(D03y=VU-=svg?4J$OswcqbZes^z}HTGc6FP(rL z7u7w|^Ja1nJ3JfCkG|WVSntrn3ieWuGCfvXznW=3Z&_yFFUGAhSS<*ntr9+fTcl^k zFgCOMQ$vpYC{QS_x8Ha4>^zp&wdiWGd%VL*(l;#{ z#&PDQ52e@ZMJs$C&&e?VS#P9#{Zn5K-`TCw+j9D3w@Q!qFNRnkG;g~5bqg4A72cOz zD>I|X_n!yeu6p>d&jNB9T`p7m8r{e-xr< z`Nc3Ie%gTtw{g;j2+F$rzDj7(U2oId+8bmyy)KTAW`$&4ycYHey`?SjFbO~_wbIGb zv{f?VLSt*`U|G(bSv%}kcXyyHHhHa@r+Mie{q>X2?q?l-6dd5}e`^<~jF5~nubgM<_B}bKQGFi<@ zih-T9V#92=Iaq!F@?ARHF6D4{W5va$z5Rh=@^=L_BZn^uikxZ~VTIhd*3#~hKY!aj z{;D%Q$6vj!uSpr@a{B&q({7`*kSh5D73JO8nit)rMccL4H@m`tP+}<9bI7$xEZw>5 zx{u2iBb(L|6@{cfL=8hjkL~vZ5};6qBUat%*9iFbrrO) zo>8Z}lpmh{`rzPP#e3^wf@a}>oSkbXZiVqf))isJjR7f8*1|wQ%VtC4ncW5Afxj-# z<8f{YIkx-n_T;bIG~T9!BsRb4e|hHAF@w9;{QL|OQ$v@Y{k440-Gi6fK3$kD3hcN& z^G8^EOP1np?8x0yiw(o>wFw(;J*TlvHtSB#J-hVAnxxZpvMauQ9dk;4?ONXND|G_% zmXigoK)JT_&x*GeaOpXZd8#4kKwrW?wfYe6#0L2M<@3r$k2}r!3B8R0cVGM{Fg$OV zm>TJ*Id17JkYd{v^}9;z)uxJ^Uwyu@al;4o??h-c`{FAl* z<`gR~sl~odlI`$Wm0~F^G|Bz?Hw=3~ALRWd5=TrNxpn4qj?lORo(2KV>(7?t$c9@~ zq!_C%VZE>E;?s2;7C}=S6d+I6i9e#78MiL&In|++8`ue*Y8{32#J%#HHQdJgh zFn9}p|LdMye9t(CzqkZ?7C)rgzi3;GpII#^C@d_zkCrhkFOE4beK#RM=DmOS?p?@R zk2zx8V{_O+|0sCUpzpIDm%V~>=i(%y^i_3qd_ps8$;Jb1&s(JD+Wy9Vn(niUD=NIi ze^&@Y8!ySF<}BO(4!^yL^K6#yG%d!j7gMV=W(1HM1)3l6z;7I4uMqrtQPF-F+rV3c zRi?`Lc!zwjVPX)Po2!5|Ic*dByj8f=(%c-1=L6a3eppfO$(!Wvn2ho98ZdezHZt@vJw|rU;*APePi1J&r8^hqEHvKxV50>SuKvdG+ky!g zuU@^f!W>X6-8oI04$->ey`Pbhfl-2p*Y7Z)`)NDujo%_E#Maml+?0Fip9^bkZ)fuU zmTMt&<6TtGo??=OxiCTSmODW}=Fvce^l z2X^Bl)?k?`lEDf51=OqeO)X!@2{UG=amU-&%+76r8B8R+0nk~h0biXPo)->qB;^X1 z0M2u(VVfL{`HF@9GAZzzdKwi3UFds+U%bn<-0Q;U~Z=GP@z-o$uCF$}^8Zd$(Z zpSMto7H2#xO&6C;dJCas7Xio3WLgp^&++k3_vJ(Z_t~X*VW=ZuFr9?-Oh%?RynV?} zq2pabFf`8=_|_f&+&3I}nTPitVrZ>jvt|!0Wi?>dB7x~C_-3!eSQqq|8=Ay#IZd9V z6O95w)q6Kx7HLxq;Bo}kVH*sF7 z*0Z7_@b&JR*bF{v4nBy0{j(~}X7}X%xn=?t$NUu@E@U%}D_$C74mDu&FM0UoZ)hGb zYun1;gojRy1fkav_4R#@zUkeR~LrSB3a@lNM_Q*`bd1x7W^G>(E3IQ5>N8+FEw zF5#cC?;<-+x@%(gZzU{R`aeFK930HZRfiQ!Bn}MbX~1-MLAJ_vtQ7$7Tyg=)4KRVFpq zEHr=m^blXUdDZV)=Q%e~1ygJs2=22FHDc|Sy&yA*I zA}hJA^tV^!wM8(*&m+u7LYNLC1TR!i@jPrD1wT%lk@>giXoPQtF#Hj!?D4FUEK5a^ zVZxwd=jSJn5bb-vDOmKvfjJs24H#h` z0B441IGXa}i>xO0VN8wDStWF&yQL|(;elbKH&nCC+GwW-A8bxc_SVRC&io36OgNf5YcY8Md>=cT*2Xeh<1N z9^p-eO22kK&wv?n=zh&?mot>NlNWZ|N!YSs#sy53AF?*UvR6}G}FIVBh^cVE;CyJb44F~Ozx`}Z;ojI6?xm|)XM zaT8tn(VJi%g0m2<(Gi#rh||jCA}oVH341dC=R_qXLLjIIjkO{fdBjI5>qE*r|u(UwM03#`_ZpXRf^kLV6h|AJb`ud&iX)wCT97 z;KatoUHHB_NEOH(2Qw^{FsVTWXFKMOX~yVm!hEQGaI3hLl9F=bOa4PRQFOo}9)1~P z@9i;Wfrp>}F}!<3#l_X&uP}4gtO|Sqo%6eQ?=EO)FoHvpI(&~XlOqoAOnokw6!Agu zQX{L56);)AL9K)`_$KK;OePNcOR%@2B}p=hZ?Hwe)sis@EgxKB6z&oCR-+z=u|{q3 zDC{61K?GX?ehaJg>(v^&59y(pIJH(#a2K|66%L*+FgQZMkAklXIeukkWf>0{P0Dm$ zv0?=*N^UyB@|iFPd@t7VFgSQOcWoa6_fCcc4*76mghv9FzpBw6$v_&KNV=TVda_MCN1+ z8GYbpI$=E`YMB0{EWv&D1x{j-PM$4fd_#yE1}n4?>8G(L(S}skc3K2iP=)a=WXpvi z%tM%UoP6eo1zazp>>9J_Y{w|zRT$nG9L#mv0|-?Z(3#E{*kx5>UJ+dQc=>1Lt8f-U zh#!992<)@Splc?#aL_!tUkcb=nwT|n-Sg*7Or3)f$5C=%QB_sNpcMaeV}q8XFk{hz zbBQF%?C9DmYP-kiYfYSdVcMK>ve1Ipm{P!Y-@PN}a};E$X=QowDCTR>NzqaLI2w4{ zwZBSRFZK9CSj*Y*?EdKH04EgK_rS4yC2d!n7nJ~kvGm+AFlbNSWbw!E-sxbfgCfKl zHESwDr1jmEfE_z;+C9rK?Zm6L(+Xkexgi%KxUcDyz$y z;A`6Zvlj^n#KA8s5{eROyekZGp?U0)J1)-Lvv>a%n6QGY5`|QhE3n{UQOa`$){tsCL*f zMmr2sjE3671Ia)P8!j3bQI~2TT$aG_TL`}za?HX{5az(tg}x?qTvGyc#9ebjg|XvN za4=q`46R!TPzdNCG_Rqv_K9{0bSv1FRWNh%-ePAYd8}RstD_PDny!xK;*bvwd=l>g zm&V~EY=CX0WPdE)r~H+TUO%Ss!&(R*p?ToIbpYES*sCw*7ug9^QNE?oB^Xny+Tc8+ z3WptP!-biD0j{kId|4UBkN067d*R^(GF`)J*A5#wKlzpqCr|j8_U6Ghb&BFWJ#WE> zh;Riy0-&QOfpIGvupf59Qpo4kVG1Wa7-1lZ_l&4;!1VQXRkXi3VKxC0qMIv!-#3}V z$Mqb*86Q4RenbWNcsWkxIAn`;FXujMfA!&mA?C2Zh2fDT(zm0H*@k3!17rUip3}@| z&(ZrpBPl=_ue4(Xj)9|&61icV395IXjFl!Kw$CizF^L7H`^t zt@{=>G8#b&lY)aM@xE?aK$FcY;ow6Wp3zR@@x!LSFZb^;g_IB8*PR#<&^s_d&=1_P zFqg3c91JBfzGC(2)o3_0hNGQ?^a2x1S=y=;C}#*i98I9N{;rJf4)Q`yq z{XP;3h#CzVy7%YLYM30|*=%_*c$MyY1X223n6hahYGX#=QAfet-{GNE2G2Cjc#}UV zPTFPl`)n{s4Z_-1<>HFtpm6taI$_AYO1yeqKU#H{Gnl*n)G#kRf^m%}92_=Fv#Ky5 zwgMs^n(!cbE~M zrXY^zieR`3V_AY%)y~E#OB<2g@!)&~n;tgo;FmvV07gRVJl9mpU?>kp#iI=0FER;} zj?e#wXX>tjFAF6zH=M;oeHF@}ldpT4Ibc>9S^S~0NkkS#p3u`kbe15-gd!8>tVDaj zn+vvipdC~QgY2CwF3I6B_St`aIU{TI=X)}!Li^>u`S(1LYoO(G9bgP>3l9JuN#f4S zX@$ZOYH)Bc61X_yJBziTpsdLKUV_8i{61@?^x6*fPA3 zevE30#Id?(@01>Hb_`ZnW0{~+ndV?g88#5bV?vbp-c7$!;Q| zqKeqGTJW&yhgWAC_9Nyb-^M(VN0>j>T;l(}?FwUz(-C>A;D&%aA#Aemy0dEU+~6iN zJ&ffY>WGe`R7~lW9~^iBRc;8GsfxpYOnh zuL7y0B<3ekW{d`05U4jmA951-dAa*Pvzd+MUPB)*%;Uy+gJ~##`O9slaUv8zfC~t} zTnDZ21=IJ4W$1!2>p2AsrdU8j^6*^4ew$*;0lBwkJT}VGBmYElNyDx3@=qV9C2e%)|p!r?P5Z||NRcOV)!mAn+oV5_g>uw|8FOyAYaPUU~jG5O4#X(kjH$1u44#3>*2)I6w;41LL!&hzvwR4be z2>**I&w?Hb%e^?ELi{#hn-wAto{9J6%a_aSd*+>Jc@v4?P6lw8)fWTy$$Z%&cS3C- zmZr^F60&l)M;i>2tKhFkM`j0(eAb1@^X72*&KS?5jK_B$(oBuil~THYpndJlA< z)i^f^PvLcA>E(ZnHOh*b!zXnFc{QELU%!2;LPk>TyP3w?eztdDaJGmA4Z@)_MSi5` z!q=~-_vPzhf}~BZ`=pNAq;Mjd0hf_`^1eZcp;QOIt%$nrZE#$GLt$_$h(i43c4w$@ zj&$*>aDb$oAD!l#q-O@GV=@;!>LhV3s4OcUJXnp4n0!SE(;^{J#RjWPwKpXfIRGkF zaBy(hrb%Wh6FZpZN75mVBM+Zu^mea+YY3f^7y+&Vr$8`S+F!9#>oU(IRmT~}A-Z-V zcK7!8YeK^*ajMHe^Rb>LQKk17F%!FD26~VVBHVeyv9e6H=P%+6alqT)eR!Yub%)y# zT_GL|Wzjj~V;P8hYv;9$Op&rPw`{6PRgmDh5+ei)RIOh6<1Ke0a>J8u;h|46B8#w~K5db!>o8w&j z)Y%WxeNKk!@Ke}&#A3W-C3xdiFxY#G0e1vIoAYnKXxn;vE)pDR=e`Kazp$V>U@KgK z89y30ZE^3$2(zl#p}wQjmm6|O94Xs|%rkJ?R6%&Z?)jHQInn?ne`!q&ja7lV7`N_r z95UylwTZhCvdi$f6zRd;L?Eaw!2nPI;4iJMmW~G)z8p^2aerZM_#lpcqyVxGX(#U}jV~I5l3?B5hmd0VN0%FNm zodFi8#;DoK5az-QEWLRM#Dw30WI|>i5MnZ!kX{O#LO$EDGWkd#9;PL`EKBmlAoQ;rVLRZ1r)7l)fgG+>XY zPJ9ra@5hT13ybE2fvi$P?3i7&(Xma1e7-4_fm3%F*VLK9$lZ}N>m^y-kg|Ojg#Bub zoY6L{hYPk8g1})guqA4FPhNs3axZE7&wtY!?sXdKv%+!Dv%x?I0iY5S60!cm@h%D} zm(6F~vf#i{z9;5Ie}HvR#e}X)nd7+1WYm@bkF0Pci`*1_|NcF>OcTd(Fi`F>y=5E@ zVw!qugoNZPg5Q=p2P9((DEUSvI>Cyqyv9E{jwvS$9VP8f6KDWRg_ zwqO~)bafp&eweY(cw)=eVZ1!m%w9COv0ccq9=9jEJHXx+W|~isalU%_GQ{~C=q^e; z9D6?gxlLNS5N!c?<_9!wF$32cmrJi6r=w=bzWH?alf{N)b<%^ow{8UiKv*G&rl0u{ z1_8Tf+6i-*e(1->#)`D!qmLkQr+5GsiG0mr>^X8Zd7g1-p84_+Ac%IV1J9UIhJ#e? z9A;8MSNiK$KE9p69x^I=vLKz7JXpdo;`Bo0z9v-9F=L4fN6_F$OdXZ z>g;lTJOVwsL#1wGWOW-;Dt=l78Vrn1C$PK6&6{w$)`D4#252)Fjcf`1 zRTLnbQI;@i`A*K+Z+qbQ@pzoOn!u?$8|3kon>KC2`mA&?=EKGyV25iuu_#zAw@w2>8n*-#`71whLYlM8XBVxb?s#$3Lpf~g<^8G|@^`1pnow920I9s*NN z9>Zj@&vYL6aH_vr-wsC;6Hx^B;0M@WK;jsLWU83qgtErMux`j4bZG1@f)-5`g-#+Y zL2qXF4k)k1f&G9DX+p!bcy)iXkAefZ;ks`~q94aOswomNiW0Up)_jGV^{mGm2e}x1mUxG2Uw!m93yv`?_@w z!-^vfh~Gp!Dh}>BXWej0Ukhe#6Y+{v0Rg8GvVGae>2*o9BQLYeihPs^!`-}D zYV5YWJ!aqSm33(j;fK1%@4Y&#iV+3y*nQAJF6ab&VPLd^32I0K_uAaqxfVGoS#A@b zqv$Ps+d9k`J9+h}?ny32`CQJyyA!s$;x8>0U|K;Xn73qf-{jO+okl+C;purUHhGS~ z-LsV3;1ATZPLBMtXS&4>48*jq20tw)X9R|+qaiB;#g5qbDRwr*SaG`JhW9iBYq+KNI!?XIcnIa~o7= zt?N@01VwE~XKN5JaokqMCgS8qT8^}RUp(BPQ8;RB)QrFjBlFPEGjmjES{9Swen<|Pmz zj)4|WFtwGac;b8^?_AyPTT^XJbA+cQ}?IXQ>}l&HCm z_Sax`Jn>rItM~tb``Ax38PE%@HhQkvyd}UNjW~d}sFCWAT6n8YfO9`mED^gdiHW)UDP3RAr9ac&l_N4E8a% zx)ND|ul2Ntc=)s_#cjSL-BSbHqm&*nT#-eo8>&Rqt@*ZiJ-%xNW)a}|w1W{OjSuh^ z52g@+#>PmLHe^3&TVaijj9`=^aM)wgciJ^%QVxM)h7d{}6ax^A6l*cT%tZfyPo5k4 z_Fx`?TcUy^3IwK(+|g2qTdqHAEvevCrm6<)Z3uo0E1npc+begMn`za8B6Pw*4%iWe zxx&DMowEW-A%n6nT?k4geh8wyl{+M%0Yo5Gx3!I-KiFYz;e5L3#uf}TA@Z!bz-uq| zrrV$mN>lXqf>(_P=kZ-xr~mZLM?r>(x(`P)5x;`aIlB~a+oQmYm6CGmew^t&D3qz_ zsZ`>CK15$}bMTtJIkNP8Q8YRo`yg%yex(l($r`|W4;-(|j>ohS>GJLtVs1VZ9BqqF zzXItEx@lFgDdV3^(PZa>1q%XfcFuh_4UqEJuhakG?=bwk`D3oNg6|x}rjNQ4OXKg+ zNt=c6exOO<{ItCI6nvBImEd60^f>-g|F@9du(77jF!lJKC_AX(m>WOQ+O!DmqEIum zO7JganD|ZXwa;I^cpo!2L18RRIWQKHVjl}-R8c?&Sfg7(M@@7P4a$)aOSgRma7Kg# zJ)UCu;J^QdYty#S_InJ*`(5l(^1I*hy6^x(i{7D~vI=}Xk)rh4={zSBa~w8+%13a3 zEZKG<1T&q*yb8QG@b)-@^!d7S4r8JS?>?vC_NvRsdMaG%=}5-QruryobTi9=*4`$R zL2*bvqT(4@B0@fahR%Cz+_xBqyGYH7qbONX0NaztVCYQZB-fX|yzy1XxxfF%=cE4y zft?JdPoGYfeb{=c2M!!4{0>@&226x22WoK^*dyfDBz6#Kz0SLN58`b{U2^ofk?(uI zHRc$Dm#hKnABhEOgXD$S5%SVQzI_C=M`576+kHbZCI&cFBV*-85{EZ`N6gp=XI&*q zj{YZdz~bM=e#MFW2=@5Gh-`D`&ZXN!Ve^)?MFUOH!I%nvxWOY3lshstK{ znl+bDSnD}3@itkg7JxQyM+pg$S~UD3Dqsa@?f~lPAvjhrF#?sB{itC}}=0^tSn3kx>q6kTf41r3u6YXI0)_k3Sl+m6(4|6sOG!eLej z_|0(9QA%FhOY?4y$K6c>WAO7nsv(N7G{XYMO%NHQmUR=ZkRXwK694qT!=~TEL;R3= zMjbHkEtWP8ylBAVp`;Kf+`c!unu)E0;sx6eY=d$7bqqpW?3ND4*?y5h;~H_&~d zvI;S@+x8!l=;ZbiMDqLnY7Gbib^lePgQroDJ2=>Ym&^({-Piv;I185)>6!DU9|1@M zJo+CS3?fJ%fbu(jovt(>RgNZao67(E^}P|BsRNhF8gt_vbVK(2>#C_0W?An*0;=RR zPX5H4iiK1d3Wx-B@85umD;x*hS|~I@b`BN&fDD0nLU0nY=6jpQWn{V%@^;scf4K6p zwstVmR4(BBfU;4`5H2VJAB#uliA0aaGPGS2%T@uz57=5$P|z-xhqXp>mk{|=x?^_Z zhAYGFfDews7q^Q`F>-%SEYvV0f$7i>-1mggfl28wLZI835|EzU$lh41_lJ4A18cf| z&O}PMpGA3C7_adz4GnKhaXO36_jbxWQRQYc`oI&4`}Upg`VQ^JDkg?F1a2^$%&_IOpqrRdpoA!8^xL}ZR-VO+g|R^a-hM7<9sw5IdGH_xL=dV1lFh%sCOZQ0wl0g?b#(z)yM4(I zuLvUcAO%wr^B>3|j7;$8)R}3FkK-JXUG8gqnkGt&(r#m!iazUn*O)qkzC%|%RXQlM z%=%Fx1Gbfhf53DTICVW<9Eqyn)QP=M`5R)^O3KS|T`XL@)Uo;qAiPL?uZWBQA8jX~ z*)pJT4b(U2w!rKVCPtq3h!zgArAVn2gECZ?07xV#g;H}T$WVN1d5mrnP=r^@VCjQx z!O&67R*m6e#7ySo3#ODrqLz5^@4rKId4-vj7kDn+CUNNb6sTV}HmBmU!53n_j>;Kl z=dh*S;JiVTC8KPCIWXeQzn~oo27C_JjlKH`!VBaZ?a^`BWD6k3jLZ6Y9|^J$ZkUt+ z;MCqCx&)qig9^XMh7AWvSbT1E&~j=xYCcyXow&bi(=4y&smIk!!_s~ z+$2pPfxJ=h%@{HCx=mQ>Ay%;mD$%F}__vtq|7ZB6CVP0ev8y4AXs!ZmOu%TRTGuVB zu3{1$%BJ239mq#QY_~7KYMn?1N@ZJU6`=SGU?gn5{@D{8^no%P9e1Cr|EkJ%b5&!uIkwtF4xp2Uck<*0k#1%cvj!>LDX0~*H;Vs|@^kGY~0 z4dF9`v33Uso-BuO1h+xswLr8=*MjDzawjSzi(yb|vU}nft%7a>g+*wV_X$DNtAexd zh9|yFd?K5tdKMM(pg4v4+=?noK+a3}P<{;H<1Ce6#7&|WkKlKS(|w=r6ph4(%};Yf zz&=4Idt=1_+n{VDjeE;)+q!QRo(dqd7sA$INBkrYM|f(h1Ki>?&}AsKAYla&r>Fq* zLv@1hXP8`^xNa>FHpE%8E6_8`^Rd*a_85UZK{)l_Y7F3)wDArtSIy_6G4xar0V{-# z8mQqXfM+Z?e*`;kY!6&UW(bY~Qe)7O4P7oni{ok(Zx7`-=)p=5DVR@V<+wC62rL&- z^oZF$aFCLo`){A=t3b(=yxX~Af0-JTnf^ra(g$PUNF~F*Q;xP!d=k(x&f!j~S%H*C z8L*WbV-#`RS$)aBjZm|ZKGcJAp}sUe9qE2Fu9ug8YN=Iu2vbx&>^evpNbvEXCq8rr zxG5SFMgYO*GGz*x4-<%3Cg*a* zap}kfD0B!F5Ku(!GGI37xbiNrNZ5+Z(qVE-+dl2I;Sh!FUdUQx; z8UNHP3r;&bTO&81axlbrsB7dgzyYQ1vn1aoQG^FHB5HsR1E^$;}VDvxiICw*C?$HV^xh2KGtKnZyVY z8Pg#Bs(2(72Ia>GZxUkxh8x=IaT{y$DcMu~p_&7F3W{2^-v`kEb=netr zP@{GR7d8Y}-J{0%VNarMh5-OF^2+$+5yzV)L5r2sNG#P?S0GbM%wxrv}30Q%G&1-DmWK`K7 z>m=;JeM`{ldd>NYecy3+)<_j-bRUfnb6o0|a`@FuQm>)SKm}*r4ZN%TDNC=D5Ddym zsG@?2PQVcW(IpLXl$0(XH|;J7juKt8#Z&&WF87p4)0ghdO{;vMq+X#?5xZ^v+$-vH zt%ht)?5nVGe*ET5ddizj^N_GdcWNG{ulbZ*$^Gh~hQfjs_6jd2-TrfHj{F;krl_lr z9!wq{)-6ijel}nR zM$@^dtcmH6D^{;%1ij-tw$xChP&fgj+{VWc$5_xZt;Vs2U%Cz8Ny30p%sSJ9WY>A1 z>*%+A494Wc=nKFL z6w-T<3Iw}>bec%AV)!FpUh(iy)6uyHvLo3vZ!VpI?7QL^qX(%VGKNDJLTMiWI#r>( z{rqgc=}2|rZ=At-OFDIEQ1JkL$E2i-;h)7YInG%_GmBrn+6_uV12g;xeIP0BfQ5Ut zr?~?%TnWrNrD_&YfnZ;oJ36et=`a`^7?|n+5uq9YE-7&!W*PkvIIeNsZgY#)HXqHm+IyuNJPEiF@KRYC8O|2S3%^jM3Vdj zLO^870L#2$h@!R+Vgj zrMicQM>$b?pb|uch4bsmV1u`w^VSEvk++a=6dXj>V2d9;79>Lbg=^!8Yqm(Lt()Ji zgVheL*JCtaQJjbDumTD9W8~kmKi+HxRs7`SE;vkaIvjFcKVg0$R0*Asgy|(5zDR=- zF;o!+jZR2`G%?jNc~wI-3M!=Sqd!e+4N%8RBON(v6eOR*d#m>RjgiZ5AAgcoGb7Xw zl6B7I|vaO zh#cnxGYY9lu-VcnJb4f#w;_RsPCpcv7>yzg4iGV)b-&{9W|=I6aj(BI&xF=diy z;8CK>iil!#_Gl8pjwzYo%Sf(D$R0WJVQYakgMKOt`9fhq zK^f2@)NS(ZAD>;Q%W=!0fwI^E`_g~Dvq!Cz8bH8NLRhSdx?lqL-$rhT<>un`|aAOcruBg!gaA z>a~zL*a^>w%NWZpK$&onY1$o?1PwzfblE?g$ zS8G^Ugb98xRS_Xqf;%GWuj@9kmsPPvQu0pvVdz}y<}=>iBenO+mCxy~C9ayV@vZH1 zcHo(`x*i9qATLTmrZRl2_6yhV6wciG-h?{EhBgI0}1VKMTh`g%_|n`s??DM?L0u2q9!* z$R<{G=-GgWDD^_R%tCV9Jy_3hU$FKswviCSWGafk6#y1aC~nvQdoSiFQW!f6N=tLT zEl#Rkx9kxY!+Q_sNKwo})h-%X4s9H&X!a$79<}P9m+S%4wUHoDCjdI-Xt*@?jMMo| zkRdoD+d?iMffI|G0v*iz66;t}VrwAQX8V=*!72>s%YsbWG(aU4@< zMr>E^&OqfKe|``Rf9o8z7z*v2lRrpf7=`3YTnN9Miw^4N<#^L(c6M=C4IRlfMoHJN z;fRI0*b!wU%Ft;>H|Wv;mtH0bz*v1C)+n3QOFIz?Ij02L+wdrDAb-9uZm#%nGfDbU zr}AkW9v*Hi8zE6T>ieWC1Qu0BDWHbb87QFXH#h*aK^t&8I+|vvI_{Ovy#(qbEgd4(5kdy(0CC#-}JaB;crO*e&ZjfRGC)E)= zaOW0ZOPalm07&_3d9tSSxpU`e8u45Kxpk4p`L1L_Q3|nfA`l4iMoiJl5fsl5O4QLb z`ULq~kqZEKEXl%=Dd6V*c&A{7Ahbl>>^Qw{Hb+%X#`OeFr2oMnL4qETJ2K~?~n#W&dk{da@L zXPS9P--Lp|U91#nIzm9%QQs4RNOJ=mGSn#qXbLJ*6%<9P&^Ek_MgM&H`RU7U34%)c zcFBP{c?IcIE_{xOJ?)q50gq|$tD3qxN$)8gLzRL0MU1c;39A$q7K)yzeqIvjDNJfdwb{Ztbui=0LzCiLP-LoJ{B1#P&WVE?;++ zj2loIlsNrJ==42Nx6~EO>SKl2L{ExCM+yY@RAU9t7>;L*hUka+y8rSxifU@KaT=8CY0WySi%xE2FzcJ7!X!E&)>Gtp5z1v|Kjbl;J3hM}Uh*J++?Vxr?)jopy;-PY( zo^qV?684{WG0QRM-M(U&dQSlo2&JJOq0uL>ff2Mz94l(#;8IXUSGqCF5u!ZmI7i)} zJUP!AWd?K5bn&^r!1vjpxay-M2`RP2(K-o4oYF*7JKPRs$)u@gaU+8UKPR$k)lL?= z;HR$FiC2KQqY7FnZ-4*C_4Ni$ZEqhcT=HY$;0L8lkQ;yUC)fnnhppqNMU&JXt>QW@ z4&Wnk6i`aRR0C*7Q9qO`ntvYNAxb0EFF6FW&zUg_Qy7bPa74}SYM-2WQ?c{?0_5Qb zio=$WgzT@67kc~pGTT#&?uvDyV-rOlZ3TtgIWj$Ap`jw^ZAQ-tO)rMte_bA0Z6P~! z+nc7YruLfT{JBfF20@Jnco~fwB&`Ym2R??$)`_F&@_RbM6y|464EO%u zVZp@?GXL{O{_|&b1QQflh&JXH7J4$%-2GC=x-C%ozT?Hnb%S)^L*0CLt^>gNw%^Do zZS`(gwvvX;2IW4)*W_8Y2d1$7o4Zs1vLztc!Qb@ABQC9BjttMpU9)S?9=)-QDeo3g zLg6k$uNW*`9YD^=JZcyJ+^EcuILed(}cFQ8~&Q>wSkGd)s=3-*ECRQ(-E4;)`a#apc z=Dxu&y}s84E8^OwVt+sqKdSN~DsrdIb{YPb(hu5QPc9Fh0~mDgjFw(W;Bzk1(UB1k zM={*o2s=0`Y>L#53xMR(Bzf4c*n-I_FiX_C)Cm#uESHmxj^2Va`WtXLX^$X&XQv4} zmFG(YPeQl?vNF-*+fm7(q_1)8=a!-b1W1xc6PZZb2Jp5vz+qe7q@ke$0svzA$;}tk zX+N66^-C^8dWZ(tEd~Q{^@zsKPj~K*3OLikOL-Ja0yETQ2kIS8J!m?HaM=zHS`#!C ziI<^9q8|YQ_t~VYt4q-feYsRqAl)ovfIvXyfhAf04uLW)&tU16F$rR zH92FNIJ@o&!bSNAP!%1n-Y0WLfBGS3egrZs*{;zDX)4v@AbW~D)o>|dK&gvEaf74hOdiqt z{?f0&Gj~zV0kgTz@5+Ju5=_7^YT6{hsyV15!f2nb3K7$&6ANhklzCR0dmNvW@ZM*{W~DLw$e55d5f?IN_0k018s=+I}TO{;w37M zFcY_LSjwqR7omRl8b?agCI0&_UcBG|t&I<9M~6ztZ&NS??m0Uc9q#@4-mbClgUgo9 z=7rox-^Kq)77@Te{m-Tg=#;#A{TiCiwK$m2<1;lj!{`?7Dj%|S3_nl}=4%lETKjPB zhD?bnH!9egNchMG2D?vq(avf+KzvJVx zhAg{%AW1oIj0|MS*5g4i3JUrMxpjAp;i(Q+{J2=G%1z4@+GLWAifq}U4JAM|q*GO3 z?YNqZ{;L2^SPcUk+*y~{IhWKCBy2zsDJW-$LBX#F^g@&UR?|{qg}A#jVEhr2>qH~~ zI7tIT!~Sl!aeWq>K!BQO!^RCL4s%#UFO+4@og16Oiw!$2VS31+{xM-BELvpopvF>S z9KB;lU1HI4u8I?ta(bW@Du_z&$7cNEW64tr(q$SFZfV)}?_;rw&qbc*uo%yyOnLdr-&-1%4xS z2h!4Y8(XMb3LGdtVl@*_Sq;g``7|mT0vGowJNzM)_kk4oU~a3hOu2TO;6w$supC^< z%PRt5E;YdVn1YO`-Y=FV*j{Iy+Xp-hpYUg#l{?(*ZVkm4HL_`DF zYttX2`%6qrBztn?Mr54@&Zgqg?rUGb#B4Hs-3Go9Td5q6f?8H#OSBI;Zx!Ge>02la zAthDCF-58&r3YJ8(NFvq2O)LiBgmrcnh)Iras7gn2_t{N-KrzZQV|#FMkI>tQ+sP- zbk$i;PJciA4dTqV+q$&C-;%1DC{t)fv|%mdv%xS5$T1&XVi?qkego=X(@Z$L8>iTQ zBv0wbKmA4R1z`IWAyN2z)vrVVc^RI!6%-(dW)+65*ssD&cE4EAJ%EXZc*X9U*j6Oa zr;Hitv$Q&XHA|r(7r*f82jeUfhhyWKXz#XmHbNwxHU)b$&%D~XA z5G*HNo~040$g|!!yFPJp&n&UbAZ+6!xCDZ}Xv2&`E5Uvi6+16c(aWdP(7-?fKtSK; zJj#2-D-bOPI6vB3BsNuxyr#VOtoiY{;Q&UFy7ToBTwZtctKNIGsH1oueEs@t6khj1 zm!0#Uk*dS|w>#&JT3x-%Vy(oI=6a=zO?Cr#57LMT&;rz^vsxdq3&$@| zEan=%dBc_?d3y^YFD1PWk&2evOomCz#Pm75gb`x}3J;MyyZMrd`<-3!WH}Ht1>L#o z;BD+=Vxgl3f!81Ci6D3zqT-{iU<$H2m!5h7dJd5&yWJkhugH>!7V9cSte14LZCL1v zlN;DoD*WevVJ=h#+ssbbp8Ck}kh!;>1gFbHQ&aHFrQK!0mw^*>Z851K-h2XhFBK3F zpmsutYZ8;z{>oK{YEw`|UfviHYY95BC0BAWOuQx-!>zbFYH;)R_6~vBj=m>O?)#{q z>0U3uM;_&XcZI9$NdeN*4ZWG*%TbrY67$#uB=GbG$B*GERUtr?H1xXQzzRUFd-%W< zhWj^8)l_9A-XR%4WHc8N4H1d7@6`iH9p6vXQC?z7#GS0H(K&f=fssVMvsRD`;w(-T zNH_uM_VJ22yQp^%nz8@*cO+cLA}=cG%5h5umrXKf#8LAs69_3MEHhT?9XWF3{J^xj z+FBrtN)HU2h&P;gT2H{yH6D+F2w!U-*S}x!BT`LsXzaV--j|*Ix(V8&vbqHP>Vzed zVkT|_s{0}|MgaC!qk!jLe1gN+2c?D}k=neuqHfc10n{N3*nTUhUqO6UTX^-jd}?|+ zc@7X4x{Ljg+z(wx=8IBe!pYh7mI!I_C#Td4L2aeEYuDe&rVNFvoM*Mu>ieyn*Ymn8 zzAa9j_sH$wE#?~8`nRQZs>cN~2Wo5Bv?&(_RL1O3mu=dz8YyAPrXS@ZI<+-fU%tLH zc2ErNZ8iX#%21p-CPr22AdR3YQy3qAzEfI^{}2Cc{EdAq6h6RN_ zZpETY8d8F#Q(WJQTeO^=51+t_h+8E?TUV;dciqrbG8X9XC^wLmmW%0$durV@aMr2g z>(r`&xG<4m=g3QO1F{)Ltdjx<-sZTyi{sy9UszJ0=`}PjLN~asgRfEz-6lT6TenFc zF&oe8!?)TrJ~TyXL_6uyx%=WnB6sZV6Wo{z{x=M(N&?AH-G*T6k&L|Z>0XXtc>u!;DD}`Opj5?-QlGHH6^b0C&aY(tY6kw zvt_(C{KNExPpoXNpM3c;(lE)n_i*IYnowVu z9x)uO^yQP0mdwB_tDo}K9VzNo&gYKID{?ItGt&Cv8K{}8Hr459jMA!F-vdL#mW}z34S3g%FefpNJ8)j}%+-yy)qdA%AR_ux zAV+SGWR!uv`J(FZ3(hxu!!`L*Iv=J>_77b)M)l^;z+yh3z?ogFkcY0))_vD+J~`cd zs-rG8!CI^TwxGXKPRREQ8P;aUEtzU5Kb0g-wk-H2cPl8a!L302SlwTDhH89{tPX#W zymW0xzp>qo+ZVF@rPlkdOi8SnnQ_QfCUW>nj#a1f<#pykf}s!O!tqDe&u?`>m0q~T zk4OAF|F?PpJKi5HpV1oV_%`a)n!*&di#GCp4g8g-Mn=w#2u)(-erWOs7lImL^SwpEaqFkQ=L$Vgxi)>2{$|_Ocb#H4CAGV%wtu^F zHEC%IidH&-o<5~cBQC3%>jiqnRE8w&H1t{Cw_MT{!X6n23{?gXxjdV)AHvKu?YP3n z$UV(0cv5dBVPeSa-&pMfnio&5@w#8>roekQ$#Li_ z>5uG!?fp4+!oyGzW(t)mvbd}@(>@vOyZMwAVmFqg%&Po4(EIIX$FTMtL4IXs^3rIX zXNI10fBKXj^sE{4=*?TH%4!~UO-nm7)pJqv(DW$H%EA4)gJisvQheS1Z)T z#eF!Qb$9Q!Z8AI~onFJ;H^(h3%sb!NSqd_p+{9LmN1VK&xMk^NMy?(w9&jrTT`YoU zl~J>uY|H25P1!uMc3bcMI6MDT);%u`lOH$ahnnumt9y6ejlEGW@8h;=@80Q>Hahx` zmrOr`cYo21H?;Af|Jse!wFTdOe%vdzFwD=-Y<3<0s>fh>aMV`rJJO7L_^-&u#(v{l zv^7OeTyJpDLUVLS*YQM!GXV~N~6!?nS$@Y(sxv|A z#!T+>;Uxhzw~f`iPG?V>RQ2VfNu1&InPaxdij%Eu4&461#W)_v36jFxV)M4l8QU-O zYIQ)R&A3jq@n^4O+?iK{!J4yYW%|3T_2hW!dCHsS^E0AANNJHmYj-K?{oBRl$KN#ElIFb zd3qfN)?eea8+9LlPm=QyymQK-s%TZ92Ct=1R-#F0fb+?K!_`^3ZQFz{$v#}GyG?KS zoAS_!9~+ozMVH}kq#>r{KR)O>xD1*E5wBWXDSa6LaRp`>x`dc&ICAfJeLO~pbkV`r zR%6~OHhV?B(+%RNDom4(z7(f?p5HZ2=9Jvdy`G`@1!YF(lU>$lbl0(d+!c2c{Peer zXlg;6^J=KLi@smu^^H<(F45i;Abr;8`&FT4?JQO^zlU)8qLxe1E_W2hhOaF!=s^nKaJ@NuLWa#S|&F|keu3aAKf8G-ys{dNo z%~foN%+FUyc&)2fH+nUa5=zS=`y4eD#@66VOrTjaSC)zpiRr?C%O1aY_0qur2n`sB~m(V4Pq( z;_{snLl1mP-45R9sJsR9%f%rW=v@AnvQ&g+_+`z8VojE}i@1{5jniru&L-a#yunwm zS(@4VFuOiee>5XS)pgwZ)krbULOIS^XtJ>PM=>j;?ZOOc*74^jwue~ik3MO!i!yrE z8CGMM>K&=1Z(6xAC_NMdo%T*Y+ds!Tv_ASwUE*j^bE#W6Cw)qB0`KW_L%l)w zz3Jz!{u*RCVeI^p;jVPt{$%C7TN|9alk6(po^uUq#)Y)ZtJcb}=*J1bUt9aZW&O_2 zS2a+cGPh`M3*92O_V|WXn}GmaM2kTzYe%kiJi=*T)<~N~tY}ECm9pE`rn#O4Yg-+*-VZk$=stJ;DS z>ju|EkE@l0>}OeP_NU!7Mo!}QLWZ3Ky!+d5xT#9&bh^~h8wpE){B$G?#0t5_t9{Jb zaE`Iv86UCmZ)Z)N`mK%5JM7om8+EkDadhcjvu!@7S86gvrp|g6C)Ldotm=<;R?db3 z?kPuZ?5=dgV0@Bhk8t?zK)dR$3o{B+AB67Q>sH}+Er%tTY-gm*`l4S3@~6eme`uSW zF0jr0jUgH0;z<(bV>9^=ZhaWyW!c|;S06j!l=TNel^dN^iS*^sa~%Y=V+&tWLob1aJ(QV(r;EwjcxD`fTlY+WMPJh{l;QfRvD zqPk+WX-};eL`?6juIx~H@3e04$=J42*PZ1XcIeh_)9Wdi=9+OdM-bML2VIxhMSg$y zVW#ck^4=A0nth^ii^hIN_7>Ni+<2@Z;;9ku`|!m%>lut{&Onu1izn7ggKh{Ysn#cO z`yY@MQyxg{jnmcT*Am@2xPwEykW%T3mjhplf?}@_08=@)yRpV%7h~^omq>G zN1Apc_%m{&bp98}lmG2O{O=3NP|YDOiO2|P1B9iVoXt2TEF)9{g%c8K8e-x8aKv)p zknA$4_qdb#i`kNVI&=t~<{VJhw>h-5p^%PWtQSJH%1f{~K&e(=59EyQzy28P!fp2E zx;kbFgKJ7gxsqX$@lF9BPyhV)X6X3B|M{9L$H4plH`h497`V$gfkrcg&tq-^YU2A@ z=g_y!mG|Mp2XxyR{s(&P0g543AYr%2ra%b96?hk%G1?3k?S;R2s0)s~4f>Cink@KobSS)8Uz(Ac8Ugj=t9xJx-gBlJ1??io3ACr!4RtAC2opLr z29%3-Fc{Z4{;djECDHmrgx&G&D4#%c6~AL8u74sAT0l)+q!9R&3Pli+U;mMGgDe9b zxN09&58<<+7w!qWId3w<#Bt)vpWM22i|gjh)|M7gLBW^*k#0kUrm&`_rZ)}bEcL(e zEQEeBV}gx50-hMc1_%qiAYmfkQ~+q3Je-niCb+=w5Hmxw`(nd$0QXD(Y({ewI?xr6 zM0*Ma5mpFFG}JMpb@0OQ3K$TgY7vi(DqP0?9+Slo<3vgw`*m0Jv`_L$?)vH5YIRv5 zDUJ7!HVdW-l!O{rFO&@knLWjQCA$#r{&4&E(}ufUw!b4bPf#$7$+E8LO@vT!-smM) zi*LU%R759|Egs=XlQ3iZ|F}HSM0onSZIKrRj;Rj~uC4Y&gihd4u_j4RyA>lx3t&=w1UJjZS#$RJ<7M+W;TV=}gvGwtx$@)iBa>AU4N5_v0-p^8$vp1TzB{u9- z>rXZJz*Fb9l^CZ!icHF>b~`x|m1N~>UGXSv&+KE7UH+Y@lnmxb)@4c_L`i0c|W`QNrgYz1#uwQ9Y<+u)<45+{!DuYEs$s`y3MJ+rJqBk!MI$Cb`W zW>rR?o$Pxcj4!+NTT8|wT^uP>IjghlUG@c`KK;U}t|^j0xS(56Yl?NesP+7OmEkQAt~{L zveR8+29unvKewHn**8#Dy7!>SCK1E)V-3A-AN72Hjn13VQEr(~n2~WrA#hD^tcsRj z%4@4P*Noe0UOG5Cv@B%B-f>jkKB}CSx;5(5%{_^meI;DGQhJ5E)jno$TVy}70D+|S z591kLVo?4y(y+hq$4w?{hw~P#f$;Ij2!RECMK{Gn(AEPGe?y ztBvUw6T9%g+X{MxjrXu5taj`jl=P48Z(NqheSBf~*#44|xphhAkF2YDyalticBx99 z>-)yl7`*WZk0e2?za%^9i0bml%JMf_QuC~L@L}Vu#Dm{P)`tcPIvvs1XE_T0d?MIx z;2eg>t@>k46`SwcR%>H+WIy@@+ON&|AMCwnSd;0xHXL=SYmJ51WnmD3nIV9PfJ&1Z zT`t9lfQo>0l_q`YQbL$~gA5G$Q^jjMUN?+462FwbY|PG{SI0VK%N^|ycTR{# z^Iq%gM_}@Y^PU6q5&-zhKkVuL>QS@vQzdxeIJ+fu%aJ1sj)e>Y!P+oP!S?2rhDo$> z>BF?jqdb#8V~-$H_eY}#p8C@Pw{sYpO-i4mg70m+8Vtr))>e6&Cob4W7Q%@m#M1yT zZjAw2k-=9j<^hN2MYkqsc~{Hd`$R63vw-3d*O$2M)(q{=V6D4)cX}{^Hsv9EyS{#( z|z< z>?{uVKsQTaO(^eBCoU<;z+!rT0nzq{m1Wg7}hsfo6BC|Q}Jv9YGjOh(IIRnjQV zn=+A~if+Um*z~|ejnb%a`agu z$eE~rE<(Ii21Z;bJTly=DY@XZs|jUdi%Q7Kuw5@2UFt!i{h}eaSv_%Ns4T{hlh}q` z902QeaG7@*U)cwiUrf1}`xqMW0<3ANc^#p9yw^@9EqGs)wlyfE`84v;FJ$Q*au@c6 zeUw!8rJmP&XAQL-r8Pw1u8D8@Y?Hy+JN)O*knz&|Jt^;JJ!<&K2*$Q@kvBlguU}gz zZmSu{wr%KgYv4}2)=1n|VqKNF{vk=u;86HE8cw!6k+y)lK`I|g+Jbljw_XYv{Bgvx zq^vaQo>AkKT?!Fq_@aq-bGR@Hy<30%Qx$e)nLUol?-At#t_aYzh!4I#?e;XlZ6n$ZJ^AYmpbFm?)d|fD;KxG+pg2G=h(JG9`|@0d<9-(!V`~W318inc46@%pk3;# z&u&!z@#|j$%|S{jw%ltnc?AD)h%gYc^XUU+i;Nr%r$+E~tbrwnN=}{5&~~@*UsG`Z zW0&l~%{ z_|;+lC89a=Sz^l-^6}X+^Jesv3rbT@xG~e?@*-L-fAe(*fOf#%Jv>NB?0i}(kL%Y* z3#kK+DB;f=uG~rQ3)bPjv%)CG-MZlS?u70#-nbS_*HSVxu-O(4$>dnKZCP2rz9m8& zrlD}csDsPDXfjx9r=wIVtIHH^dc~03ro()X=7Ixx̭z?0h@)Ll5vSF`z^1Zg2 zhcd%GHF_dwLch_)F!#2Wtvk|~V=N<#u~dTwHC!%xFKNSCUSYXC()AH~wK@AX16Rp0 zLG(zI>J15zAKD&$ZLBKnTMuRQIoJf|;G#LN9|YS(zc@k@zsFYj@O-QqTAzkYuuS=+ z07Q11naO!fw9r9{UfYN@!(^}aKEr>AO0-d>boBa|Eq1>OR_K&rju9r7DHAJKcbjmB z%Is^Dk?fiP<7AE|-_N7ZKAGGS<6i}k%lB=B6C1Up86RB5 zhE>d?y{|;$JEqtr~XL8b@wG>DhuSgD}cpT1!x z-chd^#6@qrRkH74XSX=zTx{LE(pf@|BIwL4>DFn5( z`pd)q&uy^`uoR9gcUoeW6_Y6Al-a&t_;ZER|85Elp(@_{HQZ zhJHqSC z(v57E>awLz+;An=g;T~>t3>;$;S1MNcW9!H?k(!+Yh$i<%?|po$I`=3e0tMbMaebv z4XPL_KNt|=>=5Zva&xwtkDvar^~5wg50V6dN?s3wVnA0CI^!z+rOE6sX6Us3{>i45 z>9aHSiS%0Sr{ZhJ7V|vQH7MKTv z7=_x7MQQ-I9`;y5?ogZh6ZhGOBXym;kbwvHQE)3(LnTNHLKl)T)%?kc|=an|4mPPf{SyDL3e6ek*mN)#;DkGo78L)_}D%jJ);bV z#oM|FL0c7`QJJS}Xd8OuNY^%(Z`h1!Ug>IsLM;cuYW+lLNfCc)^ejMug-7$&hYjo6 zK@`{4LULbKmi?_?wE%wowPnbAiP`BuPiH%-H{Jfkt7dLLpE~R{QZ>g_)u>2QIcHuL z(&NDD>HYb=EE?I~S)@`lSYbOYGIg#Pt`&y$vU506<`}Tq``%&tc~{I1yhP1i%rU|B zuezY76f!orV~P56Ni{5I!@|>d)@ArB(-)|=<@QMedHKe{AfN-f0#2(_Kd7oc$9|ip zkaYH;?ewF7G3?7JPd7F-I7nEY7reS^S;%|l6m&f?!HYL@0KYW1-E)3ThR^0pF)S4) zI(zF7NqLrhdPf2p{K`m$IeDjuxL0O%KKZC)^>>=hY=4c0`Im4cO*6zC^B@f~CMsoW zd5o=?_gUvehR{@@RYT_}^Z@dHI>CkTc<;8coCCeGHglnfGE`%#KC5)Q+Z!Jiy;OR4 z@9?q~v~sdHQlcuqfB&F7bxW*Y(&weQJ}V>e9AT&M7gNxk?@V2gphAh#J>t32sgovW zy?k%X2X|6g-9fu29)UK14wmE^|7^15gbMgx*9WZ8thDW9QgZ36JJ(&sd74;q#Z zC>@@7QGJr)7ukG?yEv$1%v*RY#?RrKOqPX4tHb(P*`*B|l!#{b*hAJd*c70R zpP0^cE#kgu4%j zmX#7~x55q|1+qXh%uZR|vU^D{^uZ~RmTcMc!f^XQNV2`_GDZp-f}?i!x&BUdmVz*K zy~z2Et=;4j_dGaJZ@Y0OVuN4PWX$c4mnZo0yB?uKJqnu}zwztJbR$(zJsWk+DTe-x zva?>`UVAfB{cS2{k3}u^_uLM+VN%yoRN5T7agywL6dWR+1wPD$Be_b~iq~^i=koI( z641}JKy_9oP*#~P{6$@>2ZEo6uY|q#H7dl>FoQ^-Jdwwo8o?Yc)ly&!^&GfKlNH4hiV2aT7#8mTL z{8V9#j@(O3>+acVWCv>9YA!8t@L4T=Q>>(TD*BPYA7PBxm`RGbIgmew5{ z`J#(_$e1JOe+?^km>moO=o4_dbN?#*cer8({QVa?=9wo{^v2fHG7~mkeV?xy+>iHx zzP`j%1sRsZU%R`zNBFwl;bmZ=M2X(0y!I9t*+qWWNPr2-#Q_#H}{?hXF|9@@MkptuLwB%f(d=8NExAfQQ zP25&Qvhx&##r<4RgjNWo{|@T1umKNfX*}%%G%pF5H$etY1Ese79hNj;hnfTjYCs~W z@#ZiQV!ZNqn@^QXQ2J~(@Mksx4=w|kN_5CmKvCSGu;CjHs{3IBF}ctQGN2Di&qzRv zo(qg5;^jfz76#Pn7uF9S{3C2lQi^j)B`0$BIrv!GClL^hWP|@W9Nz=pFS#BIAR6}! z8}%xRU!(w|I`oyT9nyfvZK zcgZsc=s)d%wOJ$;NE|r@nyBa7{=DhbLLn4m%$7k`#GqtJcC4jCK5F39cd+7dXo8)} z;l9_~2K6rW%#HBarMx+I)J$?SUPU=!_5cz&Ek}Ubg&7(F{revpE>v5AG6w<}H%)LE zBWSUZ44gDF6Sl%?Fc88l=71hE>A(vC`VByI7tfr8UGn*SC#+;Ei|1>rpcusQ4w%Vd zY9d}7NHx`dtnAk4PO=o?3tcY(LvAk&oIfd@SpE@yHBB*L?JpI0Mkcg|iaxI)V2sn# zG-4I(izN+&>?=9lt&Fw6M0?b=>tz2lyA<~k=RW!NzK-0Mr+uFTLX$e?Zv4lRDQ|;q zgV*0r0*_io@4=z(|6J0#BD2HWVDl?oNy^T5Z*nYbi52m=Y;50`!ta00>!nLj5aPei zCub0@N0^&#iY>R048HyE<3LN_9#7}nhSE;l#-2!mx2uim0od@U~C8+ z6A15?*iv&l*!5s&c);{NwT}Yrc25`iqjmBZBKReaG0RWz+`^(0(bHKoO=elyq3a_3 zJemm^t z@bn@QgG0|fGddo56Ej^Fu)FuPu^|H-nQ-JeV?(7@d08xgM?LKQYQdKhbI5bAoY*y+ zjIR(x;_`^EwfrMvHmWX5q8zih%IxoZ)#sSL zCr_mS1RwhIPc0&r-lu8Jj?@LwTBq1=i|^|*IxA-0wOEVR-p=W>-(r~yFVhO0n?uiz z{S!f5$jpE)8trORXpx20$QjKi_sw;^1&|Xnp)#HeP!6LyiEEKHWX^v3={~=`EpfX1 zIfp%^CQ!Ds?Ozr4YXUo@JK48=>ju=0bkWb;0|C;#6h@`$l%LlDSg|_vfL?iItcoTb zQ8SnE^Hftf(!w|pcZgHq;xV9Q#h$H_F6yrEOM<(Yh+l8a{vfTzU(nxA-L@;n2tZR3 zr^Gi<>}ob|r%rJdFrlT!~c3sf{ObmZ78AvH`0w%pm!GX)X8 zUAG-Fx5B)4_zZ!4ruPb{+lppxoKZ7DFJiZjeB2&ha#)C&ezzsu9Mi;QATV&iwREGT z61$-w=J4jE{QUfw7Tp)7UEQ|KCy0x~pF#iV04xIr9Y zvGg^7qDgQlYz1YB9w4CU7tcqj$q8_q{kjJ5vdxQ&Id$D*5#10h=0IU)=;+49NFR#> z>duz~7e0qR`=ceHk8MPgI$%(1?n5G%R~`UmxZRWzN?{v^P5I}dK$1|eZ@ z{_(VYJ97U66lHA5tqm70V(Od|ua<9^(u~#OkAu58l31r?2Yagf653yfOAHpql>NFh z-PStqg9{2RK%MbFlws58{DcsnQ~NrYmSKVbCt2_PjzzoY(OD+0r=3NDo(CksVvXxT z?PBZ`OUsH7QHiw=SW2$HJ^%?0^R-q|TGg6nyH?&>evrK!FVG6xcyuu)%A$f08dA|B zBCeQJo3&AV4Xa&v>_jShHM;##R_?e#jX7Dl04z{jXv(?wI|K*ugqn=lyZ|J}qrk4R z+6a1h+y#3^6Hfl|pK#4-IOmhvt{tS#`afv}v27QLZ_bK&Hxtd}J`dld4@fq?vf&MC zaY>bQHO{!2_%YzpFmNdLsv>=Q;-g`B1IVQqB>o}=fAftw!c z*VmKM|8`4DAQ}Tb4|Dm?mvb4kDH^TOk=qP=S^)@U7mORI$pJ@AFVHR$@}qA`xW>FC z%ZF*vj|2C507SFR@gnm-hrj zf9L8q+bk!ZjRK~xmY$*?>DbQDO~YIX_UR2;T~$2ssidPcq|ZgHt7Gee_)eVOT%UPm zaI6MbA9lN<(S1PW#8BY`^|v!0JqN1UXA37YfdULRU@jr1aM$b#f^%a;3yvmkw-XjP z8PO$vRo)kD1Qkx!S-}CDu3JG#cTgF_NaL2z{E>Q;cOZ5a%B#ov^=zd@X`7{`Fd>5P zv8<`bCs&*sH_KKTdElsHt{F~bC1Q^`WN3xk?H(qa>63q_*l`FSRd5`_Dis^cSS=9Y zMsu|cr{B8rE;d@$b&|jq;NQ5I4+ZTzT6nh(X!5@a7nY$hZf)C3PXOiwAO2~$Nigx_ z{6K~EZbZFRg>1BMrjB|qg|K=l_iN+h$egwJyDmQBQ}|Rcd4U8)?DQ^H%#EZrwnU zY#+n`b9j0l?tllK>3?nxHuu5{cwJqb`OB9+h}E9t6_Xim^{t6xMLE^sqt*Txm2;Lo zdKxkC0P~zQ1(%WVBu4GGQX*~&v|k6Uz0L_DZjG~^;VtFy!T_V$&od)j1Zs_-$(Q3- zaK(qyq-1ha)eqWuIZCYUUg>`7C03D{OXKVha-MFyKB^M(KZPOHzV8o9?~FWJP|y|; z`1US@!Z6c>K4Z8*K7xD>$2-k088h_tUFR;^d+AUs9JN=bs0db-vEuZSdmBG-Y(CUbm zN|thOIRH*-PAmwuW-yJ@P2xg{ioEQMevG=Hh;lYfqyb&k4Dn%qU{NSRMH)*}i&|~C z@vwdC)1}*4$3Q9|uVUR9`{0f5#D+Vb^)YZHfW_T^7pOXu2S}7{-18CqStEAbpJeoI zoP*e05vM12|Jm!nb-bsDEdHrkH?n-Nl%i}`_oE-+llN0e2*InP8DStTajo_>M^}f^ z&ZsG9`4t5L8u&L9)aKEpSy8k@1`<(ww=Y)^L7&?dXzHi+FPZnyRkTE*k zR6Fn=Bd8~GwE@m_eDek`gWsj6&;!hIs=^i;t^}JkANLBwTrKn1cwrnyi+R|w{ppJ@ zSKHs;C8mj8NLV#>Rf{{5>nSl^_BzBF6;=}XO%sl^h9w4U7)=xDz>ttX?BmQ`P&~nJ zJB{C%R3z75M*24j@Q0#ib4I3K&7GJNw#gcKS_S+0Iy=m&)II7-7_aEEro?x4rT7H7 z;rj4`RFVD*(~jOuD#SO9BBMO7ix>G*dvQ#_`1u94M~ItDWsh8y)va#X5?iwD0(R2F zu*Q&EIq;b#pRBPc>hlyXXhf+Do!)@cW)Em3CBaE8$8*#$?htqM(@+tiA8>P| zU4mPavUKVevDD+dvGszUEP@d_dUDkhaqLhS) zxg5OA@yPyj9SVR|t{z=|FedKo-cPNiRoJxzi=H7d{2i%&OHJmz)^ZcS2a12p{tOhw z`?1y;4gj}79zJ$+?R;WNhX;480Ke4VQG1x#619-NZP%>lG&xDl&!xuOd@<_w{DJsg zr#?c0u~VB4|BESWU#C4oyd6$S`I80kI`J(3k>HDH!!`qg9XK!*oJ`J zi}+x0@s3cE(}U?xt8niV%x=n98ja%5-aP{$UkI(Kpy|1QFuz&VvGwUOekRyJ_Wg{@ zeThz0cgbKm^W%1+R#M=|z`*8g)VV~*;E~JFuwfu{={54p4gU>3nLoHivk<_t>=?EQ z_}=Ah;_3Sj(jG-$t12wu_mV8T!8Z1|wdJ-$8~0uTm{XN`{dK&CH)Y{CjTwAE+y=>s zL|?tnVm|Na8<}==@IFIn;V`U%hPIvw-k1qjLfY4PXJegF(V5yBMCeaJ30yT#;jJVF z_c9=8@C)lnm6T_QUVZWE_s%HpS{*-c0 zC#HZtGZZSQ4xj|(b@aKsFeW+fF!>lNWdezH+&};qC zR&a)vg)B5VwN?FL$5}#EOipshGEXzK9oQ+5321S#Q?dKKFDV{?=MS;@PP+90{0Axe z8$V%-m@S6%B3lKQ%YV>tvdR6ZNb7Nqsuc{sl-2WooGS=BPeK_+&j%X%(we|=%mAtJ zN@T~blLb*G1eds7&W*nUZ6?^t9B0bAvJC1hPt2Lt1W+nRFI<6rbw|R5y(_+Pr`mty zFqE3ItG{(NLHHL6NDCQ-k0X?|CLGcY{by~-d( z-gr!8%o!BU8~2JzO=XH`+`=Kt>3%Z_9~OI`Ol$X`ibnGf~> zAiv~RzXOZ8q}cGAq~rC;#rmm!U{aWwa9W2yr65j+&j9P5T`JEE4t;;VlHwdS_&52- z>#t9Ke;u%l*T~0!? z8zLZ(8g5d7Z3PAnECox4ZNPeV40(T+&rTyW)hPbPdV|)79DbK=3rHXVNkYjA9#}j( zDE=8X@TV5#oe=5WxKQf=^&kOUFCe`^?jCsJxC5s{+5;lsb|LN+7TOQod91)E#4l=U zX!K=x=Hh_>`P4#@gqqGLTNsx2Yi&UQ0|AI=7Ta9>1FQxsSvg67VvjrY{i{vSZ3Bep zA&5-ge(3vCVPkwO``LDobTb-xp- zl2e^RhW+}PWx(OD0JL!itr;pL_r9_?ZV1SIo%EeBZxzW~i*f?nWm25sl_2il`Sr1< z+n_5hkX&)KX3X+Kpo1u18w0t6L!dph4p4!A;o*yPK#wKh5fcqubRJJmPG*BTOfO(M zV&1Vi)*dOLcOyXG-bio-DZrqCSxvSmm5}56#ey?ng;0-yju4>PRT*HxcR>RST+e`o zfzC)U$yf{Y-OWt+tIjRZ;7z&}=KHNA6rRcs?jxR&GQJ8Hf=AHc>{` z_Zk#Tlc8N?UvjI|VsfdbaS8N?+gsTFSFR?bOsExwd6#yHXj0<3LPZ#0za_s4lo?Ep zAhdYq>#KJpBqSL5tF6f5-Q^BC6@LrDUzlIoy!jP@=?9n^kb(FHdhdVt>#5T+zYC>x zK*NMu1IhF85v~SUfdsfBJ8|S8;^lE{#K#54kP#NZWF_%-a)1f% z)AO5*gLU#W z6DiwP&I{MYb&MN5^)vis(m8s8?L@DJTvyPW<5Bs=#kZRI8;#2*_m9FZ$x34VBT*4= z9yJ*T-#GUuHyK2C5_QOHQBd^i` zh-kl)C}j2IR%ve$gj+sj?9z&!etRfrTYYXSq2cDgQwPn%Q0WsICCenxA_@ zWw{Rt9tN)#n9?DY30!~mZF5jhT1>`e(nVq^$vImC2wO1G{Eh9L((TSY({;?NzOveD z%NKKW5hN+G2P9cQvhSF!Pk$1HPh(!Adw`xRcsZiE60ojDdW@W%4NFV&HO$>ej6gv? zZE}6}1ikGvgvT(`m>p8x8iJifRvf|R@++s18P*kh`|Pybro(dK-YIbW+J^H;DP>m* z{qOLJi9|2Ds3d zYvCX*qb#uz&niGpVRre8DZ!icJQg=V?P5P>Qu}a)%S=(3brOVc=k|vwaFZO)xp(;6 z(o5#fxzW3H^eb(dV<;N3U?Pi(9M1!@~lWVY;+q5<+;8A+6U>X$OCN< z+YjK%2tG?o&4krRNdRfrm~$q^_@m77e|oD2hO zmyfl(v%0sOBG3RPJs(cj^W+KmTqNp;;0Z1A3ec# zUc>&hYrGL6u@@TxE|zz0epM^cGlahV4F#-CU&vCIsWM>idoUWp^0qnvE0Lg0AWZ)( zJcnWSE0lEsYMe3g%$2hkwWHFi(MEscg9YAy=_QnGaS;!liMYA)7EZLI9m>2D`56OX z9{oeYI#etc|Hv}VEa>jyfFsJSGhR_^<)tlKp<^EZ{5Z2ofK?~-JTI?AjB)X@#?UV9I7*Q3FiS{VQDngAvD?iMQW)2{eMBUjD%F0|7pq``R_@u;}h_zwb~>? z7VDnT8r2rsJG-HKQ<6SpLyt0wTEraUwuzD52jYX)iGpc5fz+eP5;{P7S*eZ5NB@*^ z+q;=XG7^W$3&^GVGsFJSIRvTJA%!|;^ey_TS-U!43wCN868Xd)kjhUo80qu#|Ht}%?xy{+cmoBuBBGG{$C;5$vf_MZ@+z+(6c@}s@ zC`I77g53^y4>^VS*0VYUmpS;%u zBxl%~Nu@f9u)C7ln&KPrvO1kS0@+Z^iW6s{b~he(YV=J5^1@`-NLH>%+=9*W27cE_ zGbb#=Fu)EaPV(*+kxUEq{53ijYQ5yTFDr%!f8lzrxeI@fu`wR z9H2fP8N}>~eRBf>3j>?eP=cQ?#spr{Agpyl8fWTaj7CRpq$l^}QbSK9difklKvd7G zbge558Vpse45DX(RBrHMO96WK{QJOWS2MFxwEC+;#>fC4BG{mW&^9hB7h$p-35d)w z4^JPUXUG!_RhTpSTvFkx{|@`!a)EQPB#Z{EHXA2uC()^SAlqW?a1QoW9F$MK)C#f{ z{|wO`$?VN-)V{L~kqa^tT3$U$DHCs^dUsj+_o`l1wX|kX!2@UHIuMe9B%hgR*_8WG z19jBpWVP97vJU?(hxH!`BaS>D5{TOKc@!$QXGZ|8oAR>E#_bPJ{i~p0-+TBU>a3f! zP%*&BHEVC;c-55=Esz&(0!fR(%m_2+$pTOwJ6WcRt$fx%uCE7CC_-pU@&xhRNVyeN z+wV%|b_J9YK|y4%+^nM#P7~kUeR4fjJHRFrpdk*JHwsY;i3xW5v$8y`n1ahLCZV)L zW5LWZ-cr4h$z=Qy_wjxIgj54eo0s;isLK!73R}=AQS#BMnk^gt6%GD>UEJtR&kk{{ zgVI{1N>iyDGC(f2taAg%_ijMCVLb#w9g<7z7EJo>r^*8PP-b_+1=_@1*UO3KhJ3Jx zuo2~X7O87+d1*j+|0U@k_Bf&PrA(tjvCV>Pqp}Bs zt^AO=<{2zgU^Y#!owMw6juUA!Hz=(Ut0NE9$|D8v=+#HUDxi>#=5-<5KIP%y$OjMS z=<`~U)6gY=2LQptT7e1BLzS&vw}zmxq*|z-^$^>O6A8=)Qv=OJJ7Jf2kaRmts4}n_ z2$Ut-?F}?{(dYt+zo@WgNbG?>F+cF#U)=Wm)&8ezHxqTMAK3it`)7yS7gUw3UnbwG z8Q>Pz+}?-q(^(#NXc)Fiw(0!K!I8&5oy+{IO=T}jYQZSyT>2hjPrWlPn30Xg3AA`T zVRO5T3M+V=2TWH#9kh5;1Sv$^_|uPC{LY)3^3R*4m3P+Pui`LzwtA5pkS*{0s(eQa zXATBe(d^Pa%w=u%-4LhOLCTm5jkUd1!rCmLXp7w6!6ejaJA`|4?)!+DJ~kaiy;g(5 zp`Xq^M05@>SAh_T9sQE(F;XdP_vtAkaYPeucpUamBO@`?+7UN(Y~q@or(>FUo9<6B z9b!VhuPhqq#X+CslwbAY_Fy7k+ST={p}aQjhnm-_wC1`C!DkXim72rGy^V|bw?kFc z<}75+JLT=nPd&H*aXzQ+PDBMb72w1CvT#f|woJo)YwV~5FJAq`Tb1)R_)7}JVqbEo zhjOlfg8H@Y?^jIug=GHmiyZs(7gYT#k@Tyy`BxFvpP3g^N3c^U=|H14G#O~8v9n0WMV>jn&NhJ?WlmcRYs2T)WouZR^a z1fU&emdooVGWS}Xym96&piN0sP_!6p!2DS;;k=jEj7lG{EoXrL48!ijp888>PiP=fSRBw(?B#8d&bvh89gYyzr*A94S zbOHDIp_45)+aBx$BD>$~uT47yx1$gblr*`NI8^p6;cz(}?W2;H-ZNh*|(okK~$ zXB&t43Si`>8Y+{{SE4h_{Z^UYa&&j;ls_l4dAP@6q1vz=&*TaU9q+v zFK0x*i1ym1oESnfDn zVdAZus1Z$S>sTc{LhdEgcNoUhw+EA^fBY6j&bQ;tM-4ymnkcb$mgl(n8;Q#{guOEQ zVQ)KqCnDyWj`M4JGWZ^&;j@@9IYT9)0d5zpM$%%kxYj|_M6bYoSMcf?OEduZXvLq|W<-#bjyu zDD94fuQOULjdV(F%e(T^oO|}DV4w6+(Z|8GoNLmB4ID6mIt3M{65@qsg#_Jiw@c|i zgJP~;!!RVPRbXr6Tzx$zH}XSn$oMl;h=W!taM!=}>Rp-ROCyqS@#DpulNw}~tjET? zH|(JwJA}++CTTQiJB5$wTd6c*p6_)g?r!EjKRkuWKaKQi37LKcy;T#z`D;P;9g+d= ziVSb{2Cd6l%EV@7wk<_1waANez^>u;dwZV1t_&J}3Cr!Bp&uHA$tQDN9H;PLQrj*I z%e<^W&%9h-ng-F*!nmb>jL=mFZ71#KnE-D4Hwq8pcs0*dl##g+pC;f$Y3{I3e1^t2 zB3f;6e#S~Iq2|`0vQzk{%U=Fq5UXy9n#R!Gs~OtdyI=~~cvxaH3m!^aAK{D0RIS2t z&M1UqB>Qpd*&PSq$XSXD2fMdmm#mOF)+CjfqH_`H%4F7~cWI1n_yE22<*P@lJ#Z;*qDkDcj=d-nNBD(^x%hjT?G9Q9gs6dyOjd3sW$s;I zc?K;@PcPh2_k_BAO&=+D!>4WVXXJ_X^-_}~9Znts&(|qlzTvS8< zGU!^sc)oq{`R%a^*P?1Lq^{T)qf(oSVr|3q_EN3yJ1jCby3A%8TKiOe25!YuqUKf! z^vsX6F6c)3#>3Aez3jkRoIV(#Y=lq-Tg~}Il+-mJM!08l3VX1WJf{|Cb7V8K#aNB( zEGK>`aH2qaj>MR1u)okr5bgKrMbmPfEr-m?Nrh%x_5G=!8DfXW%rAHM2ZPxv6UpUO zg)IN#7}1V)pZ&>}&6OnwiPb5d12wiDMa?|RI;SvFyPq!cMh7-qCP|rbkJHzayzI(Z zYYu0!1EkD^+m!gTAEx?C7CiK)({E(x20HpTUB5k%zG3fSo`rF7MtO-=I`52#Wlh~V z2~IK{6TuA$y%RoCElfpXXu{cq zCKLa8$!2rE4x;e~!+LlvSgBMEB5Scl3w>zyA)J^HwU7p_EL#*aYkx3mT_Hzm!X=Z6 zj=s^p!4ScLg_n~4_pcrwGTzvL^5n>%l? z6$$|gGJsAzqacdkexK;jr9l1VHwtLq39x<`E8Vzm9h$KY;*joPWWVoJs{Y3x&H9T2 zlQwuuNx`!}L@`=agDc4s0#B;EF{jh1&Yh7=#82BwRgl3#$r?}ty%=yaU}p)1dV68u zpzTM+M|ku>5P@}5%~F|BdSlMy%O-jZ@LaW&n={qxCY1f^#=OgeyHk($fk#;BOT>hX z`3tJOC?ZAXgiC|Ama=)zIrA)cjzx~ul?F!d$q7M6V+MEjr{vU>f~l}^a&=p~bzdV` z#}}hQqwW3sbsfUSYkzs)s?_qzDQ5v{df7wW!P;O5Ocm@57zo+5FM{;{ryh{MFkTcv6gv?|A zGE&3);R{%MnyFz(9|k#vCH#jk`H+7lCj0d^qdjs_)6X{O_YolL0lO|Kth)3~l4$)e zF{ttVcE@<*tL$|`{kkchl{z=n?L*l*C)n8hs^N3>{Ke>HASvxqaG%Hy_crKaVmLca z`ALK)u-G3DKZj&bukHm3f7Ut?){}(9U~^ISwJf};MZ>3lIB|{HF>Ogev87Bme$@he z15PftR084Xc-Xk2%P+-(TJ2^f?p_7J|aF1;bEeV?quF zK81}^mOg~K4IPv#!@SrPnPc!^T5gIg385A)nx1@sCCvQzQ6^NFGSfoM)n6QY0{#8m zdKLx${ii)XPMnN5R`>p*U1qKA@xgEPEO9mywz((+^w+|Njv-^0?eI6x?mifD@gP_` z(D_2ii8|Ld2EZ9-)J2s2FmVn`6A{Ej<%K$sEJt~US%^ntA|@vq$*4MG4n5`$hnEq+ z*xtI0@tW#iW|LYnkM?8MmFSl=xWKMSLuXR#k0&?YT2)03mO!W3p+L6GOp{Rh-{PW7 z?EoMsn9TB=ev1n4+L36S>1z6@=K)<|`o7>C2uZdCuhYfhcOLiF&Zaw@OXw_{u_Cuv zrCpdaju3wWyPj1gr)9g_Tl4TjfY55R2%Pqn?C(WN{PjPns2~$eIAhu9m5= z$TR(X%(jV>>p~`RtzhTdy~aR$m#{R0$LU`_4Cz<0^&^}QmK`kZW?FZ zAm}$wUVCWc*B=@uzdz9voTSj@6GD(NL9@##F(Q|(Ov8YSme$suVsS--WS*>5J3>49 z)^JuYg;DQ39?Yvr^Q1pW841J*gVy&`iX2GeYQ1uyvvznskx(f1qM5h0%hI2?Y#2#n zk9||bok`)d`ktpdY=APB@Zl%=Pg$`W^ko_oKp+0Vh`oQq$ia?-{?)(}rtC)E1J?rg zlpP&scL2ka+pWpTsW$806P4Cn@i{`sut%e}`}0{5Mak5fw*!Vvm594vP5d49DrnrC z*di)_i0Qh-bh466-9Ip4fgQ>>KQtz>&eJMVz@bV&T}NcTK5h(wElw`gNV|V zv03X6cWALy5~?-~z4b&>3;~#lMTkT$XQ-Rgv62B;>GH&WIfX%jn8k zg)DSV>U3-ACUHR^*#k76bcSNOsdJBdqBsmj>Jvp1>y{C6p$GTT^OOW(;EHAvJVz99 zCn2%_?*Qb=*e6+B-zzyocea|QwZ3U6m!P3ZUiv1hmUaTkZgd%cLg|P% z7xX!iZ{;b=aF$RJA77ekMYRtYg^@Ia!)SLZFOxkY&_qm6rmoU>TWwwF~dH630RvJS;M@&053uc`Yyt79?u+q|?T*a45XozWZxTUL2Ot9&cb`F{Nx?S}jgAAPf>;Y6!N* zmK589_JPjiZ-LvugBanu5)v3OIJF|zv>}s}y0oJto_~FFVctM}1RAXppQT%Vm~o&N zo92Z}uQq-gGka7q+ z7=yq;K&IN~3WEi7;-IHj{}-M3xkf0!`>NQQ+7({zH&kgIHs(XnVL1$qZ8li=o=y1>JsOtajf4w89%rwzHlE5t~=!AVH~ z&KRo5pb_^pYXiv2!wvMYrCuKab>W|-)_tI9W^kWAGaq#0Tb;9wX(iv4c20+bg1b=A zkn#y+aiX(l56oLhGM|``w%Q9Qz(vc6=;(Pz;NDCIy(M=g16vUURzhp*gRW?uA70U+^f#!`hCIs*=;IY5o0{MtQ9*wX8B;bxsLHFSo zL(ID2n-Aw}ss6GMf&JO@r0?JfxAPaw%qS>jzq%06=eW>y)2|jVO`sDb zl#6=~Dq-&Jl(|97dQ&w#nVh&L{~KM)0p2pO&$T(-E$}#KeB4J9dOG-cdX+x;j5V5x{2DtychCK zYr`z92=?@EMncIJ;|Nqh z!9f-_bQTHOgo4s&aQyc+>p3NM&cGvaiacW zzQ@yTK3ydVjxZKf`_h6FvqRn&V2Kb{0A)X8y=BE4>5kFS;$$7)WWe)bAcS6_hE7)Q zbpYJIpl?2#%=&ONTTic9u={+HM=z(JoXCWNqn|*S*_wdBPp+Pmr)u`v*IvMiOg)Le z+>PKqiDLu&z@BnJWG{q`sN+O*zGFxaM1z~RoLC!%J__QB({h6@Zo}$-C$t2`G*SUK z0nQ{aC#Tju+w3-1s6k^~7V2!aR_C4*XO8At~ZL4r!ol9PaniX@RNNKnaH$(c9zIn?{Bd-S+{ zU%%VkHSRxdmBKmSxA)p>%{AxTd#ChlVx@834Yf8=khE)+wYsh=#BXJoAHX{}b3CKL z*q;*(Z;;Oe4{@GMJrREL!quJoE&JYeMFM5JSWg@`&Q$Bd>2*H&!RBKP90hrOt|HW|FIk1KrGBL^rcl^8vP%^gGM6rVZ6_{SVr+DE8S_fy-xTPeL7H zo~McZ_s!zBJt2XFU#&XL*ZG%Eg-q8#!DED_jV=pLPoZZ^)za4Y^IU`vzjZf=9C1?I z$(kEJ@okQmjSswvqWCq%oWW0FbQgG43ZV|b`Gycs8L6<{zb8rl0~?b|0{QFxIo9RW z%{qaJqJu#MN!Bb+GPtWdqv~rxCETl?h~;_mtB`A+D}Q`#6s#WeaAx|2c}FZ0dz|&y z_mYjJ)oJz*_UXL5oG>%DuMWfihJIK?dRdUjQEHX(3}50;Z3K1PqzmK(C{z_Qr~TW( zM(7z2>UDe}{Sy$L4<~CTZlkWBY5cSv8)C(*&1vl`9sASY%lQwo=Wo0|J{q&JNGr|6 zCsA}XS|AfWt|r*F!GexyxotClAOv_*qZG@5h(3eZ!US?Bibx+dI^#!i%THv7a z=-iv+<9+1%<3h-?1^=kDcvJUmNArB4qOsnP+@w4)UTmWvY-x2;mIUDbeDZ%g0%^^| zh03OHLPO<~Zmo$KZ3+r7KkHP`@Q4+Rh=}dS0QH6Vs(*AtNAwq_Hhn3`o2~D+x{M(8 zY<0X|qMoGiWq*b&P0s0CcE7``V}6Fa^G3hl?X&65sw*2RfAU21087B81g$P3#rgE_ zCT}^nI?t)G7qr@FBUaXvhMrPO+E?Q<%7GRY5;{GTv1+^{HA^Q{q2wm3kReT0lnhbh zRaXsUb#ANOnx!8l=h6{j-F%_TSI{KweP7<#bgO0g5A#dpMY;w>!^6||VJ%v^i>(h~ z_d%)&FUn_%7aYk=ZIPYot?e1}JI=fD24Qu8xEDUpCH4`bmXULes}r5`g{(DNCcnf@ z^-L&~l4xsuYmq(LuAgdM`R0B$ttw@e_{#r;eIV9-WJW?9e zJn_`DCQY+NFNG7oNu`-Kx|DJj54qS#>#_^m(v!_wx@Evbw9~P*f(em*{d#MFq{sA0 znn6+Ccqv&~arqli=$xRSZFZ(z%BWm1~W*m{Z zF{!jM{z<9AQK5lA->DfG?r}Nr--i22x?&`eE;P`Xl>T|3F!1s9AcJFftOuc3{+bSg#;dDy%UOtDP1%bpB^8Kz}zb|3BgP z;;;X;XpN}K>Iy0TQ8+{F_kk@T=zsEOBmM!$3|TT|p(c^jYy|h~zIciqbxsG!65G3o z)c9iP41BP79R08UPpSi<0Y>chdtL3nGxyoJ6PfE-XshqLq5T?3eUw*S-t@-Tph{2g zLsC-iu9cnnxw^Uwo7ETgPBdFKJJ=7n+K;{KvMPPoWjLQ}Ro5rdS3Ny1PI=Wx-tMW? z?)*t@7mE&0e)Er_f~*mrX4xpw++^^cUc-gUD!RJ!#a40Z=57L%7vC7xp!0=UE^oJP z740nMELNPd8{UTa<3;@zUtd#u6Aoqm2LyUV zHmS&0{k6U7_3XKM)%PhD4dfVo-P&c~KUbiAXrvC3*8pu~WSUohWWMTM&x`W$qr>fo zQ@Yfd6CP(zJ+~U3PvP1+8oZXs0*5=l%~;i+PEzmDc4-=V7CS?6Q!J(XZU6P;dW%Sc z(A6iLMBay#>&v-yq3Lk;*Y3iRvf~qx$m8J^ZaeX+@lAH~n%N_JHmlFh%5Pp(5#5(0 zhvige(d8irOgsG6x!~jNfpmNFVlab&>nXdQIlN_=KEo zlNIjyeWpK{V_9Co8AL$keEQt*+q~;hGiHvRrE|-=hQLx2cgC&EW{sY)d`G)b0vw3y z=v?4pV5snfMp7Ep)p;j%xK}TvLQ_w22l1XHt&$Lu4i*s`W2vc#h}!O>2VMK!m2$Xi zlIA)pvM00Vrv?^gLQSh&9`J;grq8j(<}m+Pv-s-ouV$zfHSt@XFq97s91qS>)fy_E zk?tzE8%hhF5Z}RJmU9?RjyS&wLXjG47HZFZTHj#(U=91=w{R7!>DM!719L}GyaNu8 zlxthGvZs_QZ3Vpj0J&-p6Zn`^|DBTOB>Ek3TLA9;ELkPrYi;Yi(cwI{>?6OkPoRO+c*>r8o- zV@Z1J4NZB%dQaLMY$}@Tbwm7k&l(bV(W9;%Y9@X?N#J@-j8RADo`t~qC}-i-Eatw_ zE8QEYh=kbLx`SL15q|lBnWlbo1(kfP84U+kc9qN1(puu1wd%ExEU;6Rdt~3bY9`O# zVjdBgED(#hbmdVHhWyH4rpdp@{KZ6Pc{!`UyWE4b2_va{7PNX2Uxg(V#D1S4vvuoI zDe24jC(G7ub?W@9%=Nm$DV^(=T(DWxe$wN~VS`EdVG|0nRPwEA-8EmHchPnFK0)a7 zYW>z0@U%95R_Q?4f0+G$LfGRi|4D>BSM=rCf{t26hjdJ@1m(OAV{!C1Ynn$mjrERB zmpt<;iOV*xC`nT*xaXuv)iv^7f1HY=tXf&{V;ju3jCr;5FV)Aj;ysD zBP}npvoE#(Wzx|LYMI1umQiS;wNq`ZpI@SY>3=CIxBB<<6Q1j-l+e|JOSJau1vwnV z6F4568(Dcuo*yM+_>I0NkxK0QL-acP^O-IIx((0MxB>*P%EBWgzF*6nmYb9<))va{ z6Y*dZ0Jpyt8syzn+X3gV(uorh983{2iFw0lE!z1MXP=cE{gQSiG-);KBjNdR6PM|{ zYO-pHa(F;|>hnIGv=vFZu1#S--8t(Z)b3;rS7;-arb+Pa51XrR+3=6fizzV~Y0nQ_h-rPwsmJQ1t(D%q@9y@)`v@QJj*7+NqgM*w z)^KSYUpXn#Gp=pXrY*^(T8Gugv|VZ#bR7;-5$y4NUlcNF z-1p>q@2vNAQI{e1A;ZSF^~zdjA9;MSQLNs?qJGp|t>RKj=g>2enZ1GkQGej}iwST( zG!)1zu)l$~!-|YRE3na%1@Psn<7ioixPwcqjX7Lbew_~NEs}8HT%j?y@@xD35x=On zgdVBC9OllW>v;9=h9V+fI&OW7$J4JcK^JfNg0o;EmwlGe>xy$Bod7H4= zEX-kWI6S+{ma_y0uQ$`LrR@zOhB4=vae3@p&@zJ5VfA*cv_TmD)a@VV%x;PJOG)%>%_f5_0bLW)1$nNa zHL}0&$sTdiR!=-VIOE*XWnfgg3~mC=9U7aQv`aT@Zv68Eg%WPgXu|p9aa-3h9tNZq+)WE6!ncDQo4f4-O;tzYDklA3z>&{hhiXZ_;GC*b3YLwL%# zAd`!2|N9T2HSuVrMUOeMv$NxpmXB}Acj_&E=gaYdFQpfgKU_@SyLa!0{v-ISbASI8 z_B8C>x%1f1pFeTp5F8*BicIGAa&ovTg~IIP<@L0!$g{7zGEBsAJbsLK*RD6y8(CQ5 za17PCOzyMsk9F<7{dQDTv=rCWPOC+q+1aOFOyg3E*En=kWSDm?Wrx+bJ?!`-H*1c4 z`{YDz`#<4P#&94|_Xlr|`|j?3d3kx3T_5)=Dk|DLIK-ZReUk@={WLW-wYRs2;{*zm zA`eztIB-edU~?{hUd_NDd>01?cFIKze964hQx%cpGM9r5BkleD@z_)oiz{7YahelY zmz{cgdOniKS|R%xYbNsE{rgpr-_vaev=TH^Pi;P+?UkCEIy*b7nqoi|G;7kx&(G)4 z&U&$LQz=Imc@ECv!ooK1w(o6iHl1Z#Z{NPnjh84`n9uF}7#x?Cb;RD$k$1-qDcp46 zUr%iiPoF74KT|9C1n1zWv%+XCVHmMacT$vfdqt)68vDrWlswuW zcR(knyDs6}G*)@sY8IjBt6wr4zTMtjlX3OjZ%j;m5&pXTg1i)pBK@p3%B#aZj~_>4 zPt27!e=f)2cm7UJPBjsyH{+-#MKd!q!y?DM!otF2=&;!Fv6*Rx<=Yapvz4o(6xGvB zY6Y%+mCD18oRrK=VNo6mCFYId0TNio2P7nP4;y_{_1<-|2CFGME9)|@E1sU88LGuF zTI>8e{{;VNWtt_hyG$O#r}^VP-WHdcs~h)gsakcGDgE)s&4#s0C^Cl_Ltwm{o140F z)4nr5zrU-^y>5>F%ceTH*YcN}?Sn26>d8|B3u1 zy7kTlTQLfS?EzyE2L8T!wHltjCfmx$k6Y!`=|}r2q4}ykm0FS-7ID_S zmmr1i|JC!%l;%L&C7SrI(|0kA$2`;kGdU5^+wt+>h5V(vx%K~!jM$+%=o}z(RKJThQs1oC>BgC z68-ScpOtu+chxtmz2f5Hb}NR&V{l^dH#jBDr@0cI$2;sJ6X# z-^k8xZp%q=b7xExe>S!clKK&N#NxS_m{^&Ik8ww_kM#Q8a%V1Ij=r;+#n5*9zkfPe zGu6=eW6;s*ueX?0Uz|Ud+GgM&AtA9%%swMf{4NYD*mM*5^2jLglGa6^v?@QfCK z6#)W5ZlUv7PExzqqTKF0ey1A%Q8h+6qRivghT)MBmGkGH!ZwnGM3>q|q3|DBoYO55 zPFXnWuO{J~aBA2NKkWN{hFa;)Cf?Y;7x2h~9%CPVRS1@N*bm#&lx;;u!{9tUnMv(FXxp!WLZpa^5D;Avat49uyGN1DdGgy`L+nso z{5dywQTyR=lGAMln>5#~S%XnqZ)t6h03+*VI_`0H5@wTRay>U`|BYSrnKI3s?2FlY^%*yLx6jXbJnP_#mi@A}d)IGBi<-iuPKN)JrD+K$CnBk01* z>tE^St@GJKq5=Y{3sV)YY@%kP-IYB#tHi9ky~g_+!gk9C?nSaTEV7RV61-CR{3P}> z)AGbOyDXuwc{1Mq5Hd*}Mo2Y5OR$dOKSt06sG>nuAP zcUE2{N}(2*Ne#v)dgxNc z03Z@}xBvW;VG37%X~DuQk=kli)=jwOE4Z zY2l9_Z;=m`&>>F+#H1Q2xAnJW%VHhIdW$>75WJCc0nf2PhzAIXtYz$EPKe_~>WK>v z=folsX|M&(`Q*VWDqz#AO0p#0{1GQv2R!PDK{))=D|UOugW%xc5`VrVz`KAHSBlj8 zLU}GSV=Od`cysRcIJp3RLuvbX5u4t2}6J)oB# z4_TPkyyYb=oa7NaEm>+Pe_T}Z!x*5aF)C3!$mtC!m#W)~eQ@DB`AN-{FHv=4kSwQv z7O|^ndu1T6Q>oPHkqXJ$XUCaEX-!KTd8`XWeKH+oq;`A z)@|FyvuDqqYW7bSQ^S{nF~B$U&}qEz;MIMZh5ri2GBw6+DI`W(?AlfnC{5dd1dDYr zH8rKV@(2|*n!q-+0Yc_2@tBHkJf%a2pL+bsr<@!2X`I0oVx*A>?X^=5EL?Sa<>p84 zE>ko`HoJk5R)!yFUZfz$w5ozKsqbT8yB@CSaf3j zfFuu3ulG=tz+Fu&DeE6!k5!K|W9}Cd!@5nIBH^D6WaUsJX&dVy7R9(R+bT+8Vb;oh z>)}(h_KprN>hSn@{?PJQn5G=7E{~w#-~`15an#M(xj7&Aa|l|rhBZJq4Gj(AFYHz^ zGrJnR_w79RATu*F!?fXIA zD3~2CldBNOS{C29Y$v!G_oe$^SXtg35ZEtkY%D#pBLPA2&q1WOieI;F8sx8r= zN;2~5DeW9ijf%iL5~=!)gbm1Jc80>O~r=0>4b-TZB3{1i#dND5c*3WU=@PZjumyWwQl&@387 zfDJe4l*4ScQMw3;rYN*ov~avpV9>8wtzd3y(7Tfm0HzcaRS3*Q&?y8Rc_w;&@I!sQ zI*#&i{+fk++*=c)f-c<~VFoh6Ss!s9f%^tyWW0BI?)#ajS8!nS=FNT8_f>q_?C;;Z z=MDA%V9fv9;MV1zW4TU0iUyUy>@2nJJS`%AF*r}0_%kRd$Ta15Urzikn%Vi}Hnae- zcoj*@hiH~Tny`ub-@RM3zRNCzPRodh@PMx&7{S_u=`WK0g?8a6e$3%v(PeVy5e5AX zsl5FBRg52D6w#GZL}Ggdt*BywgspfjpA;ZM+h zipzH?#Qgc^uVCT=Qp{1}=-yh&=iiNtjJP;C#eA}H594*`sSf{b5>AuHOJ5;Rn!p)N z=%JoV=dsu$6+;t;3Kn!}q)FR)=NYPTFWRU2Qoe$Cto}$%i|eRbq%Q$p5=>b2(9=_R z&Wec=?n=j`ZtiGq<@U9ICoK(dw#qF2j_hMgfBl|H_;E{yX*JGRn@}N(TCOtF%rTb$N_km*_dLsM@ zRA3c%PzCu`k8H1+?=&^jDI+Uea%MtLPF7ag+BywEQ0DOqMW$;pkAD01?HR_XIp6sl zecC-Rg9OOpRTW_CR%>}_uNI@sHP%xtjcikAap=&YKndrZ@uAivtgIMRW>g|MTfKsN z>-Y3xA+A`nrVgtp4fm;oLhHOSgC;PMGX`7=^S)6`w4inO8N&bIbL#3bM{(RLk=R*(bU|`MfLXf?s>jyJGx(&xiNJ#l+hZ=7jv$CyXi7NWdwfC7^j^vFbkfD92fq} zFTePrnVbW$L@-N>^Mp>iX@hcAD|7c>fBp3dq|HWD8ej`a>w-NBL1)k>g6`p_T` zl#Mr-)@(nsFhAGP*%=ur|D>d$K@P zi&p1fR2;O8x%l~&X}-Mbi2}i!Dd!#ko|K**ZfcJ0K;Up!g}KQiu*4)j1Wjr_{k4`O zKB=oBR1X-Rlm%UOkHoctOtWTXjHK+NeF5L&?_9WyJ_oQO@TgU!jSY?f8>lI8buE7)tcU}F?$Eyl zQI1Asq_aHLrR8d4ni4hu(8>b>+hkYb`+Ph~F9-w1NH3Pr-$g}5LDEY4@PESmRJF!+vvr0}04Dfxo!(^AQ*{!Z z+8JO}!dw#y3w%g5R{uU!*4uaQmL6P%u1C0`_ZXEhxtu+3Zm+DXC**Wj*Tjb)Y(DTt z`;KfL@?Brlb^awM)~IgIP@d}nX$kx%vNI1Aq4m2V$;$dlNAI z`Vhs<_iw0nf7AZhA3uHwgx1&As#S%Zjz`FR0NcX%^2be1ZtzyvilWl>k^a~T8p(g) za3$p@LRUAeU3(#~BKX}!@31ia6h3uzb%G|5=Q2JnDDIrvJ3c-RfK-P+@b&Wg@yUXT znN1a{si5<;^%V~$or^3WR)U3*d8%3!x#lw$${#gizL7JA$HwHVk1nBP&~JsLxUJ>} z+;y4#S%fFVYOQMJYP|LK?b~h)NJ*$C>l~kR7uHafcSB;;W{!*g8;i340OvaXIe3TP&M0 z0sNOpv)p$-Bkg-vSCn?PWe=Dm5trFXXHejC6SQztby?c@U^6c$EwptSMOnGIaaba> za8NWG_nwgg(M|4sagLh=iOYwd0$s*gtC?%l7irp%La0Fz+(h6!gxYehuRci~unJ7v z>WzH5+E`IQ1Nr8r;HCNp20$vS`R!Hu!^6X)G1W*h<*sIns#kNj*CciraMO6LZ455(u{K(8WIqNPd3uK`g?H z7LIP42hfxLGv=)l?=Vq7v-JHoF=#W92XMJ^lv!(j0x%+3Z_|BA1!>p5{qYZss1jUT zSa=-4QjJ%yNxz~_3MueZB^I6Fg*U&h-hSkaf`Y=fqn6?OHa!3fSCdgVdHruMz)O3p z;3F1)U^&M6p(7uv61PR=W>uf;Yp9uV@gBIR-xOflY0>Qpsa&Crf2Fe@~ zWKfDx#Q`c{SmonG`U0Sj5QC*6bw~t27MZ#pU$QLp3C7TK9`h42{k)@R$Z?o-;wT-8 zvr-SiBch?!CLfG5+b%mdfBdN6$D>vSgb>z>GliQ9vhwtiM%!Mx_Q~mI&co^|xa5>iy@2h8VP^jmi3V8sK4FDWU zs)6kWF%I-8XD}(FjaQNaa6^aJjm0MU10oN_Tn!bGeBmsv3`bKLfaOEy=NL5q`7pu} zAWQaQT^dRs+a_w+{ugO6kOAnQ_ImNtVxbLzgssMtfw>gO0Lnyk%JLP=ru>ucE4+E@ z7BpxPB!S1QzZr9aPeG2V2g|B&yMZFLjDE>2Nhz^$j7*5qY2#)cCr{w%zCMKre ze*2C7rAro3{ywyB1Og(%AmN+aZWoIHdfBg+uZ^x#)nofnI&tZ?h;!;8X@OWhjR^yB z8BztAA|6CP$zSx@IhcvXgXC z)!-MfLh3-exZizGYS`E1q_|X8R$}WUMin&Y7W&M2Ew{6`H^!n1JZ$8#Yu7GN)~ZA) zCf{-F)-t*b_fK~D5X89b+}tZ@+%V`o&OhI6w!D759%-f=aQeen`Wv~)?E*_1C}y7m zm>GsQ2Bpr8<~|m2a~Z#hRdq;o4OnCPzGe!)H;7B@r#?lN#kzIt@C-$Nd3x4kVmEHt z5)CSi&fot{$s3mVa_89neC{&&jR`My2uUZp)Ss^pY73@>^6C_0;sfNqy}6A-xtB1{ zfG&qc*RXvjVhCRJ{ll$cfPdhV(RM99ayKZRYTl%m&%HUMd3Nu9-+yG%9}J}X_M^(k zz%_15mXbMpv6DU*Qo9Wx1CX9*`SRt{?LLZlX4k37Hcvko0RSWDLJ;f&EQ}*}43vR^ zDw!o`pQ2y6j0OX2`-0|6yJT=H&++3tMC~M!#uj>#*sf~l%t7`a2SH%<7J=Kf!qFSbz4u-vqm>F0WL&Fk% zl2-D?+ci8TOPBK|WD%{Ngt`Uj@eR#SeM{#d^ZIoC~FTRY99mWL!p z=$Rz4AT+w8#k%t8=`m6~KsgijDwSgi#gq|T6e{iR-McO`1HzxvW_l-Dt=lo(`KBCHv#qt7E{Pz7Bb z^xxB@NMLP&*E)~yvgoPeBCr<-suJ|3YVyUC8bo;7#iRE=dQ1N)t#F#$rZ5+R+m_8V91L171X+w+w^;+W68wHRt+JB3v0Jc#_7H zOAj2z`;Be);oXii3=-bJ(i_&TdydyThheM*%}Lk#;#YB?O91f+>aMT`WDe<%AW7zs zwc(6HOCS%C|NquTi%Mu*4-6|p62P5Y#`RFWbqOpJYvpcW{UG`1681WOVcyXhyql%1 z?ITX&SO5=OOL#JLwt+z#C|;V3*O5sEmLmNIQH=n()9ptLQ#m5eq0FeFCnDTm}Q+IKLB*H_sOr4^CO`P(HLVB_^49|Yk~sokI!lZvN}(F~f2jB203JQT*ISgZ@=4o*wxPJS zRkD2CheH~-gJmk910&oOe$3AhY{sH7C}wP^RSz(X=n>%sLpSQfsp|4))uUg(%-8m4k z&bwzIi%NJHl3n2xuVg}7B2=nCi?ny!EhKFL=Mt;yJf;Saq|oX#;+{X%$+BPLg%k-} zoTM9xrl$`yoxskiLh>WPH&u{$NDGdZIu22aL{@E8+KZe40E9nLb~#!mC)g`k4&M_$h?ER*O_YeqE=) zMF-l@8O#=}OdE*>hv6s$flZuNLnKb)y7Xx^fY3Iy#h9qaSsZ8h+* zH8|_|8OWJFWLzPxrEWEgu$9|nbNJ!Y^eZqaU+%pd@ z7z~mji9G;plq!5h*2(K|+_DKNMlB9~`^h2)4JJtJAxjsR&k0;0C}3E&g%ES^?r(H_ zesufx?W&Lfh?j&A7JWF03lFRx8mwCg+(Z;VUQn{eBj3W4G5LLP z2`OQ`{0^?xlKpIyi_LI#A_CdiSOh9w9c)6t{Gki#7 z(ur-Mqr~qt_+tV3urYLXFB8&GqPz;?)yXkGD0Jy$A$kUu`wVY0mPmS z5woZHMZydDD;+Yrsj?hA?*@_*)SjuWuUuDZg+w)otQs`4-~|X?qgU9)Q~s1Mp`F#r zwv0fgYHDgbBryaUVKVN#eqKF)s88 zRRDjCLs#(iGLM^g_o_rb#NjK?z$?T;ZYKCJT;=a9l~0Nqk#bPuI3B#ahz{cqfQNLu zpBD*V0D*-RigJ%%DKbWkE<;Qe>ynfv^y{&ZtPP8ft@w4>G7|*aiWMtp=*uwOMr4FU z7@+m*GXfMycL{?60sFWT>lhT;Pk_RpY7f|Se}dTw&(CP@UM)Q-Eqw*Xo!;i$Ote^N zM#8HAOTfK7LyhDCltP>zBvHspH#t*&NFg#*P!Uc&+;Ra8;j^D@MSWO2r1aJ$=_O%J z5r@`!e#ZPxU>{)`5&ema3)(nwtibeJ4Pyw7bJ-6@!UVnx9?g{ShGYt5cZx8>dNFDQ zdk&)Ar*b;{^hm@R;eazswDn z@)1{E(QB(Z*aV2QyA6qi&=1G}^j}%E_$wL+RTLLeLUdl*DZzDN-#!V}s<^y-AZB8w z%~O&G4W`Mbr>|8iZbu0_OSzND%Tox#ydh2&e(l27@4v z>Rs0Zm0AbApiv9|N z*C|+PR#K}_1LPHc33rDJuaAHlS=aqXfUBS(Ox4WnSBv!>?ktZ(MwA%SmSMSbAsS?( z7Mi`qn2s`;fsl;M#B)-=T*8wSL*T9+h(>U}_|DIcRBSLm2euBc)D2)mo6n#44&K)q zt5Efl-D5Z~$U#HoouC&~87Mr~hX!qert8=mv6(2_`dURx+`39y z`bk-!gcKlLFT)QDrdu^-*AENg{2+%bLEi#}e-;^F41qvA3?}YXR zcmmvYX>TGr0j59zA$-?^THJ&U9A|(J2EU9~D@v;2Zy*GmcOJkpb!c-JMp*BLn2bMH zL)=j_bY$O?zLIb`@XZj$j9|%We6dRK991$m&6m_3-8?O%sl}90?Zj>vB2Ah%^fH`O zj#CfM;GbAaLh=xN$Z7cUbRaAT%-;>q%hRp86tED)MkkdWBuragYNA0T?cZqKIrCx& zKkQCFjs=Qo{6=9bMs!Hs2f}T-jQY`V$So=bp<4)#i3a-rrL8!lxnO7UWnkr$df?a9 zkeA!g!PeCidcn@YVH>s?9K=G$IyBPQw~6E6!GjGMdQLxY?<_vzGMfc_u7uqWVW1c9 zT-fp+lC05@BS*F%>yzJ@_|y~=FcT`a5C_b5hm%7{s1C?o0Iib|H0y196hqQHjyjA+ zU%+p)xQ1Ej=6KlO2-CnT3=r7Pv5>4AcMq9+D}}WE>BwV+6-8z zKC!#bdYg7AgS;U;#Kw(W)Vfk`kkIj;KAq{y!Ybq4z55huQgNp?LJh~ZJ#}-V5cv&* z&)n>4(&gfdpJY99O$t#eZ=006j!X73SiD&15BJ}Lw?7U`jVf>`(S7>1z!ioq7Q6=z zC;)IPDvhU!<)w*ht3fQTr{Hu;b-|=j6HR*H-dhy~Q zRT)gwpkops8&Y@ssD-}mU=w%Y!42!zhavIpW*JeuVU5qj6qmN0K8?4oMhPVPE$LxE z@Hxj$l}UJp?5=NvelK>i>sDA>Yr%aG81wl@G1Frwrk3o8VCc36>);hIt8;2o_WTbguG zS<)Ygau?4F6iw`daY6dQru|h2Q{d>Vs6bxvAZbOPJ|!&SS*rsQ(O=7HT&3$fnA|oT zp5VU(Zt>x5$7Q=XSfFZs>+CetMk4lZW3x`m1Sho(%)lwkoI9-V25o3BFN3h&dgtdv zUM1>;F()UdMr(m}53LeLvg5QvfC7aF=kx2&0m8WhnYAGy-pS3)^+u;FZFst;iELH? z?`hjQ?&9K7=eR6UE4{?a0J~y`@-4ue3WNB5^8Ci^3PG1yJCGal_%`RucYHtUK%gfcH8ieu7I=#c-EI4* zWNQujgmi3)GMG66-~gnShMc>>om)Aa?)m~R!SyASlL&<5D{Y5bf}i)tO!b!ye-s@e z(bU^oP=K^c>m8^rv>z47X4@0dbsS@f4OfGrnIP>$>@IOo!Z$3IDIDi`2L52kr(Nr> z>brX9SpK4Pt~iDuAQap@fiifsQQ>F-nNw_b~91Vgo}6S?jn4fV1Zr^Zw;4 zR*>RC#Me2fu*}vNQo1f1h|?Hd{GNtc60ks4-v$PpTO$G&WQv<$P89-=5u6kQe5UWx8!VkmPk=)NL6pSni;Ii9qZ88Tk}ckXN&*+s7R?lcWDg5`E0T)Y zi4!Nnc^qZmhchA7oxN2z6fBBc+53D~WlK%GN!BuSyYVKG784>?53(5qLz=v~_D5nsYf7Fbi-0#H* zLh~fO9N=kuQY%suJl@`%&DmDc#GC=k52SQeY`6j`)D<%k8kn7=lN*UL5{;5q7sPBi zGk%p-%=+N1V-{ulR6Yq7^BwcZ`D2X2pofc$^9oXl4LL}Yy`9r3-zr6sCGT3x^jG?e;$|<Z&l3v`|J$dRNCCL0jJhZ276>rO_e$0A_& z<2<3d5aFNzb}NxG3|xy0tqdsX>+5Sk9%t#|;~1itYC-kJZk)Y9I;m|t*2k#E(1_`b zXrFL@5Z&+VlEn|dxaAJMItra0tg4^-k9_!()LBG_F$6j^DV{`eo7^nTpoC;ZiJu>6 z1G^w3uyJGk6>N5cD>WAQi*Rr8>wa~!UfgG}mIyZ}bWgv2Jr7Z;4xinQjw2FwA%aBV z&<_QxNQQDzgMIR|cu(kFBpobze$f>>sKl^AI6pA{#gOQcAj!BGI54Ex)stXL1uVXw zaAy@gi_{Ep@w_1ZlhjZw{d&;Pi<*&CLr}SnL;+ zFXx)z+HaL6AYc5}`1LkFWowz>lFlgOx~vW@!=v{dck?{`{%F_t#H?R#nodmItq$HP zeC8{qYtDf!G|F*sB4{MrNUXKfMWl<3iP#c>l2w2MHn6f@wdSN8WNZxH;hBZea4toj z0V0S(^U+PV!S2}TbqJnF#6$ww2?8>h2pOk9ovAR`#^JvRF2k@BoMqD>-0Bnt1+fpd zqTCzBZV$QP_jj4<*bFT@2Iv{MrNF!ld^gI~sijp*+>SDW%iaw>Tddwke0m3PO$^z` z0Y`8p*tjI@?Ias2PD%gD9}cGx*$_xt`aH+$*h!(;`ULe_*Kt6w{PCf}SASBx3K&hR zy%aRei~9PPk~N5>pFmTvJZiGC_mTYFJ(vvc;}kv8v!l%;(uu6Rd@WocV}rRvCD?Hzevq9*r)a-NN&FHsO4OOBMSeA9aChtIrPjz6)o>52(Yq%b^Zel?-2qQXT5X!?X({Nj^tH1(t7f%hu` zHm;dh1*yfw#ada{o-HY+>~Lie>ceg*U=g{Ny%k8j#Ljf}^NVAUu#P@n_2DIWjN+mq zX#i`-%f-Po+H(*flEXvdlpQ1V744=?wkxCKbnc!3S|Xiun!}h0ye#eK`FD(fb0-23 z$?r#xPI>Exg@vK{yR%((8Ksj!%C=kL{nI2gzPUDh{`}$n`%~D4MlAd9H{STL9`$Mi zN)}oReBg)IufILh9~c-AL~Y+L$3&T8XF!iq;zw+0k!a=zjSETPG8%NUO>s5mVJol! z>9;}YSHTI8;2OHHkBjRhOn4&b&6aR4+2P8FSj+4mKd#{sH1tG|9QlCdPr6*3;;>3? znOHEymtIibDlnQOrR-*5c`_q=3`R(p!s?$QycQVonyn40cPz7(=n<$kgc(O>TefFu z$|>;dX|SOX?VE6&oSNp23vDrPkn#EXcTw6|7%AWYZi|e>P6?Tu>$g>rmaO5iMtia~ zXw?S?LYrgKN$;$dq}x25`xsm!naawalpT5udet%+=0F&{WoWR_#v&_~mY0`*Rh1e? zlJz0tka_;pDc~|cPU4Wjxz~6$x0e<3KaV3@{oq6sWW6w=ccY9dFYjkP6#FVv$V~On z{a4#Z254LWc19VdPo{QS;q?d+PmVTt6lKV<`NPMlB_GV_Z_vOAuX`|4!S1A-F?Wxf zVx|>*EUGWgT&Okf)VJQCvfRNaO<7qvGqphuWG-r`isq(wVCTq6&(zIdi|8qjKpJhx0l>|I+=Ee{(N!+&nQ0Y8Cug`K~#~7&ks< zuNqlO5o7owhT}PQI*!&#``EPgZVU?(OV*#6FllHC$k}RD+r3vR{|)X9Vm?Ke{!RZUKiRDIYxh-RY-N z7`z=~I-cJ9U`Mdn_5%|uLQEQ;rE)%*dH=38KjPhCiL-B8nsc1z0!;)iKOc8yio5V- zyytzegoDP)$o!<%Lgu|(N2wvTS7H{js3xB}?fV|5?oqt_x@Kb3(Jp%3(A$_lY6BhY zYmNvG)zpd@<}Yw^?S6aZ)fX47DbI$+slsjhkN(Kz?D-TD+?akL;f<}Sn)1b)CITk0 zjq+O2`UN{v(o?I&3M^JMXRFah8{Qs@ZFmuLYLdyIIDg6E3ohqfC|(xy82T`_W}qR) zQ+I$@I_ejby5t9@rdQKBuLZ8ybbw~2T^6x2uqHRvOwUv4OUN9FE!NNZC)t^YlIrKD zlXLP!N^cp(>%QzfAZ8PUN>hi82mJI%Q5!+ zvo|KS1D7`1TB&^gm@qw?w>)P%Ys51b71c=PjloAPRMkJ8j5B>#GW;WcFI6$m{FU5e z_Q&iLgEjQ%w9IpL5zt)Yp%9oSp%GD{=EXU-<&c)XXr`>N+0Q*H)*aeuhKViVu`R1P zsY_czI2WEdkJ`N2kUr3$n0Wg^cPP93qr5Pm$o)Eeo<=-WnuCYkY4Z&8*wv@!6>CGT zh9^Z%7Qe2fh9)-c&8)&sLS2s5s=G;}bty-04EI#2T;e_Zh%?wzEXL?udiC4vm8yv; z{+z}$IYZcZku>#XjmGJFx*SD4V`p0$bM3?KOv@{I)=Ts57gQ@+^JjE@v_-6Hst5M$ zs0^_jT;Ivm$wUb#G$#mlhq@?Rj$)K2+u5$yNty-Srl$d-8OB%E3FK21Yu#I@x zTbJol>CfrNByH0#e1f}}Iz86KZq4)ksRFxrd?c%G8vZui(l1Zojii+BSV(P5;WrVR zBic4k($icT9&w5~GKo)Ex<+vB;Wm}NmJ_se&#Yjx|A7CUf*8#{p{QIDZiFrwAqDOJ zr&*sn59{*L=G=D!%l7mY;0EwW zy0I|zImY!&_xOHUd8&f2?Ypkj+5+8^!M{uHNXn@wNtU`XlW~1~qHV>nv&vo9FA1{g zy(KNLl`e>8w)l7b2nj}lQJFhCz!o^wJC&Le9ud>9Qm5Lv(lgsm*IRmx`0$9t06o=w zXv$wAMX&l&2h`t$#p`b8*EaHCuQVF+s6^b|e|RL?P3gGJp79KupuAgQcjRWJu7AI{ z@yH+_3CYbBCrrxKN79@Wp1yQ$j6QX1H;>lya3mgCrJ`MoYMJAB%++Rp!#hHYsYsjnR`Efim=-&(^O+k?Wt823tHi=%y%Tl{@?6r z-$YKCa?>92yDs^uS@h?$eb2=?$Ei$OIEjltUEK`x#H&OSom-FPJh7A3Q(QPB>Z;8_ zg`TQ@*X5SrhY4}O9~s&*_t1Ts`?LgIJ%n5)(m7)^VycE$2bw!qb$i`o-zz96-?!Q2 zS?HsEF9yS^UtMsRE9~(s>(lT=Js(7~U@XcSlR z45^iC3B7H}O3eBq-0@S6tM7B4KVNY}tzh%jTZ*aXV!>t`n~4Pnob8@-85Of$VGJ0Sk;6(@s9z`y!1 zKrjAHj^Ri6-}!^>TmK*O-a9I)Y}*$uwUyHbIJS+b2pAAl1X`kGZ5!#5Bw4abk}R1b z+qBY3E|O%FoHIo(jATl(#1hFA1wv7jMFI8A4W93vch9}|z4484&p)4I^cYB6wfA0o zt-0o$zp(234T^nVCx(RWIm7Y5&$EIkVgIpN%OHr@PMhgJ5U!R;zGi(0v5EDiErm6Czq|dFds{oR@)v?H`}4)+Z$o{i4!@i6V;-8#oD^8bQ+(# zuSGWa{W)}5X=&?fsQ>>nq_DVk2RfcXWFXHshwZ zyUpO!%7Y`NI_PIV=5^=f+DKZz4BL9zGSH=ljdrCK>28}n@}w;$Im z-lm#Kak%=<`FU(rsF65)UP5TeXI6WNCeeDki}N0Y4(IjuDUpu*y$r1L$oc)*Pd0<) zsa|0xqL!3bhNAc@?k#w4$jNFy^Z26u(nhsG)+Q;zW5I@tQoR{2Eq`)FY$P1-s~l|Q zQ}9b+5+^U9AsHF?BX^lZ`?~KoS`BMOHCRw^z@DnW$iHPoiMXf8FWx24QZxSi zb}ck{g=m;7DuU}w7Sm-_Q(M^j;3@cGy)kTwq3RA4MJPpXee-{{S4b#cx(F50j7R2s z#DzBPymL6s%nN8FsS2`EMEATj^_bhbAy;9xb-CZMUyaN-qTBJFf@FvWoAXKoRJlR0viN;9!(A@0}Rth(R zs2aDO%cNO771j0|Av3?|8KG}6~wY$DTL+I#g+lM~mpeQz<^2D)_jIM%=%b-`^#+AW4>O@_b>jY zTh1TS*5|Z8LgvvdGLvj1CN>fB?zzu2Vcsl7}_oz45Llj^?W z@6~73Q#FU4$MljNNeS$*?QnWk>B!K1OYcIy9pB|M-(lUbKCa!cHryaw5%@uUz&p@u zj@XXZ$$ng%=S=q6uz8+(`gHQzd|Rlq0C_ug({cW-o|o-V(sjD)e6FTNyPU+%u$oVO zleJ?6E5yHo{7Kw}%BMqhfac-XbzV9fIlXvaR*pTv@3MBq-KSwnG1BAl@*e`CX){3~ zDUyyXenLcUuC<{$v!5&O*EOT za(=ugk(_Ar7R%J49tbU8lkIFVGAq3+B8J;?GAd3AMrMn+`4<*g`;iv}RVPio)F$ex zEEcNwws7sHzC#O{6CkFk?Hmx^z(SVDJNQNs@>5-GbS!fEdA(Kkx$~?B&2A@GYh#s? zqLlU%Tt`Nc^hNzd=7GZ0!6{)p=GD17SSY_X>ZD3GgK6SLJEameF4jRv zn({^pGO6ulyhB4Wb2V`c0Z7O1%lsjltWcif$|#j|d^j>3vzli}H2CFrEaX(u>zuEm zr+BwEC~PO?xv%TChf;1@b|tn|l5&ZLjnWvZZh!;^&Dw5PC1+l4PLExpw6KUC4^v7#_3o8MpWs36RWG45#?@o?FY zZ8*VxX@13Jgy;Ej;Y&){Yngerk*}~B@4kIIOg?sf>2(a!8)?zapCNRYSXNfJb{)>- z&CW3i9ylfAl42uf2eou+_c1Ylk{^|e5|)%?yw;05MD4cS607wU9jo?wFDxQ5U&s(awL=$w`AcRlINV9(=a}M&Np^&Y1G=en=cHhbNS*r@U{6l_= zog^$TixU5k%(V=LR`rX-Vb|qCr1WZ5Sn@>a3rcpe#mhNnFy$|NBS1AD$;<9RYbo2Z zc#lbluTib_u=xB|cCmb4OT+hRJTpK&A-G}w{CPF|V4NW>+k0X5Ii|dTnS@QssQdB9 z%Q{7W90;TJHm&&8I#Hj;5{~qR2jhF#f@f??mc6kVm)+{A>jeaM56aVD|M2ftP)I1) zC>+jvRQe^p%VR}l%D25JDL9rlrchfkvwf^iAGekGMN2a@x=8;P@({vsW8Q}RY)|Hq&Iqr#&B(-KhWFC+p1l5?_^LwVT*%i%k|hNNb+nWY_Y^BI)8>vf zSJ-tOE@W;fL&j-bc}hIW(AvWdV^IxJaDhVeb(?vofwV}LEmta2ns-hkk+oV*z=$Dk(cW(N;tnx2Gkw zTJ;31Z2b=-*`ay;jyX_@eXNP%7o(bz#N0m^?Qak@@aL4jj@8b2c(q}DTbuonP!66E z)-%W2S-COAPCw8&xcFoG+TiwB!HA2ibI&~@t*>M(5WhI`>C;?jZuy$-tnNm$X|wHk zmT9l?{k3aui8xIsn#e&sp|Qop({aH3PFND)fmd;|BE+m_h%Zsi}j7WpFZmriu z(m@WFUeSJ=oZnZlLs+NJEcr43XTZ2JsXpIcmA=u^#6LyjJR(4eE=dCMsUJx`ZpZ@QF%^} z-p=+Pw)kFH`c58XchQ2s3=FDub>->eU&5sM_8Vhs)0cQS)OIA@v39UX@p;=wcIWxQ zOPci=3phJQ%hi*e%;uF=mGK3FMBCarGY;BfkU+}bWVE-epi;C=Uhw28jQp-S)0NnK z>GPKdV@q$%shEwln$H9g5tyW=J4ITAW1szi$j!;o+c8};yeFduSNCF0&T^tgcVtBY zKIw|HaU!wq@>YeZd;H`TGh5qe^b{fc&H4(HP zY-7#M%*hpGtyiW~@aG7*^-fLynl_i!T) z6Q`~{O0%=K2b~C_PDw|Y2!T5dkpQDC0G@2RUp)baN=VpV**i8id4Rf7M_mZH00{z3 zE&Xpl1jz1I@-0t5ADjUsM2(j(U*1yM(<24;iJ*(g4hTRYtbc?a;o>4H4HmV5-*Lz} z&|^h|sgz|`5`ZsFHoyr-3_hSWQ)xxu2GBk>+hpH$?p;OO)UOZ?z{z9BQ5ncd>61-U zfK?(!Wk4-J7@feLO$9fG(6Id|&%NK#xqc9BBUUIte!r#k_!(Fd=K_2T@NO&!xH3Rd z0?PjWxx1(U{_ohXCZYuV<^CUtUKw8b2`&~s1O@B|@E^D}!$403hM_sY?M>ftbLUbR zid@f?FQCr=>_-KG(?I{E13)BVokRv34d7VaY6dmy9;A1II>X`O?A&AM*pP`xBoL1m zX;zgp;2Q;8^Z~>~2&5#4ND=soJD=KBv!03L1d=qsy=&lQ|90+T%Piv600@6Ra_K{d zv_KOR3he)u2j8Du4~#P#Rz7ed&k8!G(6B%s#3~y?PcdjqvcOvGA);~x(Lpx|F@|=% za$EKx+{Ab9egp6Ak8lwyfS~hXdfLJ~8LoCphlyYglo3}@z5c(DI@k@_0`R^8-W@T( zG;uvWwGS}y8#i|393bNV8T>MUlEn!#H==X*&C5Oj+v8dvdKn_1bNzYB37jc3VCt_S zJT{<>BbT}zCuEOYhjq7QwE7qDFY|$(fEYUU^z^_gkzF7%w{f69$p&VcE#TdVUvHtH z>5)B6S6-|@$sZv54*cuD2}Xjb(u3 z7l8UiFw=5?8#{~0m=U)6e0D0h8|d%ov5yPdz>Nj$@?!#IKfpB*5*#p2yEP(M3P8+6 zB@GS&lw0YsYyz`*VD5JmYTtJ{g9`8k`@x>Pi1pq13w(Swz|w*zh(kze;K$bw7Nsy7 zQGwt8HNt9zWt;(U|4FX>H6KA_j;K!%RV%>!{V?>?!1O|fX}6z+nz{i>0=>6SkFfv! z@Au#VL=azZfTanP`~Uq&}-dle!K_T#)uGQa`HU*6N1&%Je*|x3lbYRzB3$3z-tUqyCGh)(g&SC z!3DvKj{~%dZ7|;8B8a%Oguqg7ng0IjI>7J&O#P}5Z*qi^%EcnZ3+6q*QAgyopc3W- z68_NeDwP%W)pK(Mp>efLf~e^wa1L$2zvq_H4GCp58d0I8_-|vx?T59&vjvI0$kStX z4J{%N&yR;-=$ZgdVL(fPWo73K$U5ShjEFA~%WOoq1jN>zP#6AvoFBuRgHsmt5%Q|^ z!Lrw|DyP8n30(B%K`xGnbaxhS`tGy%G<9~W0=?`b3^8mV4_!~2N5Y~+XiPwJ=>v8e z;xh|JWDg?4EE}>*DB%YQg&Xi%5J@>a%yEPe57)^8%D(~7{0fnP0nfTK-gG;I05v5Fy22iamSColNvF&T zfLT5ku|*1uijUg04&|o%0Sht#yt}}i4~{tqIrn!|g#GuyJqNr+5X@kBcxN$FKINTrtBnuFmfc4`}tOjBV@H$!`bquvj^4jk7@x!0>`3<3!RGCPLE(m63&+8KuPu{I0MWu#gZB z=#FO(QK-8^yY2n!>lck{E5O+PEF@CNQZ7#g_LUF)`QtHHAn7xT4puPsZ@`@3 z9joK22BphFcX|*)9)iT=#K~i*UH9jy5Fkjup8H!C;))Gph7eYXzO>}$KVN#_n&ERE zbQ|z0%VG9@+6gXantiT17t~7FJ#am&CF4SU{^h4*+WDKJ zx0G5r9vu5=jq#qcT69;xivDeYRQ8tT1lJx^gyPN+M3|i>FHSGcnZpP3O{(hQV(eB* z{4tj``yeAnnZGF90kDv0GIe-oslZSFUX-at>0DPzH-`c zMoARjI86ZOU}@9Pfae+Ps1$G$k0MRSlJ|j%Vs}SKGY#kIu2^PfD&ricTL0;PL2lG-jqQTJ#s|BX(DNy;Sl+&daAY&uz_Op)0~XG@@&(y1~#J_=?d?O zs?nwdzk~+An1RKyLgK7oH1AmLvEpz&pZu}rusH*xf&}f{tk?D_dJHdu<%kmt>WuLB zO+77H>3pB$qr8J)BA}d}RaRNLGX&>^_VS{>1&Z|2|kJyix>_44caaHWM;*R*Uc$2NQF6D*`W8N3rrVS;L z8-tgsAxD5N{-MF*TnceqOJz}zO0iCV(ubD(qOr^H67r0tvDgs zM+``<{gjWlw2t54B&J@bJpjL8oXKL*I8ent563JFWjwiD7Z=H!T<>{Uw*r$`2?Sox<{yHQ2XLhGcIiKGBskFp@o|KiD ziN?Bf%M}D#sTnpmj5n5d`BIZ*m1`6pjZ9dbW9MeijTI-|4ZeYqp*)meIy*vLlxaTVFrbGlRL1nK2g>As4>s15Ci9yTvS~<5}?R z1+-gAt|!sEzdc5sHr$?GP}Kb1W?Y?AL7Q?)Zt&8{7BanK_v^2ZkK$XO z%g+_SYbDZrbt+Y)Dho6W$t1aacenaNyudfB$T*?fSXe@CL~sAm13n81X1uw-y?m2r zn){qeZ4I6U4)V{bFLLx|_h%U@NKBY$U^dz^mHgyUsE)sOD`wdlOP?g(*7ZG#hI?3o zT#IMz?&ceqnGS|j3PfWQ@&+nNHsgO?wfUm6Rmy0ct36gr?b(i}AhU4#%Us42LqA=- zes5ogV^Yy*ExpO!G&nSdSLdSsY9kGnEs;h)&0MP~Y92Er!uw}+U!UKut~su`JX5IM zVDzNf?-tw$>^VD?Pu8!k-nawH^2c8a9ZAUW7M7mLqo$fYaI|eJbJVVXbd;QWRj+7& zQsHu^0b}btEwbA!N>ejB!c~v`b~M(s*l(58ude4Xrv14#X2YgeJppeWIPQ&U@*VM? zQAnK_oK))XuDf4L`&ErQju#VqW$U9)%7x`IKcm*Mo2Fr>0siT$5PN@tuVtuvJ~H(v z@~%1m#roU!b_?NesDP9IsKxYgKb_k4;pdX_zGb#8 zr9Pn%DeXB^@w6*GcjQNpg2{T>ww!y9_Zl=<#YsF}>djT=ppJ#zbK87ac2KJBqWskx z{K6&?yfMN~%kE=}uM>Yhg%Np6lE^hMzHo@6J4C#_lJf9R2VzZnNsd-Rz4>F_+?x#t z*hU+@uSy>AZ;WD1krE^-Qry;zD%4Es(*ksqD}|=?sMh1RVX1L@9X?GaQ*Wf+c&_^U zN7Ke{b#Kzov191i89?zL+v?@1lD$sWvcs`Ps^@$EuTSJL?wHCFMXziV*s;?nfOh7FMkr*?UikGeyNyW!3bK z3+W{eQ~OG8>j-|@ELStSmhI-pW8pbQPyb-uw<)zCv9-@uo}Y!Kfwqcr``V~?K>i_v zXYFK$m~uknH@Xt?A-A#nWNDj?G(8^g!v`w;)I6|D3vV-nD`#+HSYB@#nIjHc_2VSD z>ySX|Kg!sHYBBxK7(~BZLytp0Eq%YE+8r8SXJ{Ck_S+=C*uUkXZ>bqRh zE(V#w$F(X&rVB10zZeS%*URi#8Yqo(FpA4BFQ*QC4k=26IeoO-%5}*gWb`1rW~ei} zcG;syM%gBYfpw*>3vxj=m)_7|6XSnN@ldVh!ijem_~t&x_dZuw(e7Mj*B&VREo-28 zvgXKqCUnDc!I=TJ`_+(thYwVRYzPKBh83y&fWo<(nOTCbIAU8~D#~mH$uF z>+gJ)nWNn7m#p}ekSr1P;`q*9=&bTFJfD3nz55dUu4C8uRca;}U!PeNngG4l>wGa> z@=bHF46p1sm;{`je7JlD(!1|p!uEHFU&x<-149U|@~>%Qgqg(YPx<0_vIOL^sH|N# zzBA1~6dC7UYR9+sNqhS%6vf->Ei4o!4n-eCp(rK0c^Zmx0=B%LqLwUelK+w2g;6zWw(NkFxy~MT!rIEQCYj}@hO+$u0z`7)s)p-4MWq-)j;>tQUyOfIh1c2 zEOCnRz+-6m)7|Fr|1!lgXja*nVgBqdso^Kbu!dM?Q0{^AOU2#sh>8;T-l7>ade;dV z;j^6IE9Y2rN^ev%1)K#x+Zjs=UA^IA$|HylH?|u6L%4$asG8ThLyyu|VRK~vVLw5Z zaBdWeerPwrA=rms%lO(GxncMA6tZm=R{#~}=6xisP&01KA((jI#JWywh`Vi%ct3yE zkm)CHul;FA(UA2?F0;UT!?Wp5=)xyO?9Wc?AwtC3myM6L*x|S~DxLp>zJeB+Uf-)nF;+vjWH&`XT6T~7|Edf7Pa|pTb)_YxBlpgrjcO>y+Vca?)Dci$jL6Adhh8sNIY#pK=ibqP z0wN!>JIL#`TXS>gKXjVlZzHM3;}#-M?G_hMOuw|8-S8u*k;*g`pWCR+0y`gyJ^eD! zZFoxWv_t}a%*e{LzH=wQ?-MgIELkD{q^;guyQ!ZxY+C&V21nqXk^gt6~hIR7Z!6I#BuYH%laT7<(uyKpIV)UEnz?H$S^rN-Dvte&usadaL_hU0y zsi$ODq^Fgmqj?1nj%R#qiPHClmcuXX4m*aQDrq{M(wDV5RBtLIz7z{q#7=3Y$67Xu zypguNLz;EvpD&U(s=aILHzJ5G!wyVl}6oNF=gadTnHCGk@@Q&>kb1F^aU22qDA>DnC-~Q~r5iR?iU9YlidWlo= zm9N#2heNt7gd|;`2d`{mi_^*+F7$-zE6`tC7?BLzD*ZngDgKt)U1NLISwQ=D@iv`1 zt#1?y%p}z5V!b5DXnTZtR0cCS2d5VnH^qWrJLvs>$&80L^eS_Wmm|&ov1L~H!pha>PA`k`DZ+} zJ}x@ch$fJzvJ_d09FagzqDfJL_#IPSx^!qv^`gseut)FJ0<}oV?T#d=^1h>YpFu}8 z1x)j#g1Fr8=BAfmwRLr4VJ8zn(4zpIS9y7P@UL1g3kYl_@3|*FyZ;=F9O};Q5rC+R z|2U?3SN@ASxa(ks+P~Wo3HW2T3HR?ho%`Q(b@oh%7pMaLC9ppMw@>d#vbr2qyREHl z!EOXZlRaR>rNe#%_3bAl3Z~s~0Dp;5r)I8TJ3LsByTCD71Ad@z#>j7eeXDa4`FA@D z|5S!QbK6up3(O+uGm`jt7tAY$a)eA=bsfgTflS1y66{KXrWQJ>q9 zu3mse58j`t=)ex@c&9L#rRwOIn3#*~s1h;c!SBT{R`jY3c}|9s>|BC>K^2T6-`FGq zCWhCO4WHD3cZfZ1tm@BasDR9!>aBe-7{tQB}-^kap*AGna=^?)efd zOmtBT3kxur`RcJP={{_oZ4qbee#z}HDmH3oj(>sg?^-vVPjVY@5PHV-2Skn?E`ZL$ z#eyHm!(z($dEuh;hRDpYH@CljzF9ogw2w{uz}`(xWOA3Z9hP-jUL#McmNUa{;WtgH zhMbc%o9|t?M0(_qdmdHtho>dHzmn0PAEQbv{|(Ebq^}P&Ee~)0>FZ!mp>j}?=bDhz z%wJs2?<@T~3q9l#e>|3L(M%p_?+;{T7{;->&AeD&SxoY8O{EO-6I-S(m6weOan5Ay z8SAYZoIzukeP~6pYjZDnTc24S?p&&Kh|XuBp|K&J4~&-Tm!HUrJy5YE#3zo`IVPpZ z)lQAS5j>u_=}$-$THPx?LNu-dO60Fm6a)W&9=`N9@R$$JVEAO$y1EO z+PiwnQZwdT_h4_9Tlr#x_u%0gT1|-Om(WpnS`KmnFWT@q&LW}&o9xedw*S~slKE&6>nOyZakh|j>E_){TzEuVzjFT6=O{w2< zzK-W~9#3O&w!?F|bXm)Z`fvXsE45m9Sk^Yx4o9s;V@H;`W1g?xI!Rk=s<0lrf_5I) z_s(u-4Wwk>7alzR$SRfD4BN$2yi!-!EzSy6Y?6x?!ETW9TzXa`l^!bzycQ|a42V#U zXq${*oK0!eBhT71^&`mTN`u2a!szvdQZ$^BHz@x*)F9zean*beYM~5ShX*s-pOUe>V-oWbhD9XP**ZFf!+aw|U8~=2MR79YlBnpTkAd@` z$2H->m>5*$6l^W^2yo|a`A+(CPj=%QA|1J_-IAp~;N>b`Qv}tIqtb+s4rAK&T8GiI zTqBZ%k_j^rkEKm%;*ss?uN7lcO7!M(M`3p&j!&iYkNx6OYiWg~XvbrGYV9tGUEr)?3Xm9#k*(lj1fOb~=(jiHbn}g*(=Kgf%=(jO zCmAzS$jH`Attyo*ni%95UT@+TuxRYtcAXrau^F-CZwobIDiU|J1s(iNBe% zpFg8_h#X+slV@RJ*-_ACYsPP})n)HIjB{_FizBwj)3JJi=Ys>!-MWR=>;;zie0>C~)ZZ}!yZp~GazVRj7+i+IL1 zVYQcs+SL)?Pbd2`>EDzIFt*qLZ~7QtW#RV9?7Iq!MH#S$-U8WFy7XpW_u=E_Y-|@(}D^D z+)&qW{vm`5aSBOne5%53(J?90N(x-D?L{X`f1Rn_91Ne~*JCw3>r(i);5zqN)79Bg z*O8X~bB9m`D#+r?$^PdJ-KkvKLcg~_ZP3B3ZSVD~NE&x-)_j_)? z%=ThVZnjU3Wfww?0fIe`NKFn0?3|@(NZv z={X=TwfNcIW_B5(>yUfCTbW+Cs&hXz-0GW|S-Eb|aj7a|m=tZTp(jT}zoA@6u>CLo z57S)?la`z3KbWs4)eJQ-KFklLD>uredAGH5_LGd_riv6ABl3&o4|2Lc zj;dc09DB=<&PYy{ju?n-r>Wz`DT9y_#V~Gwl z?k<;fe#=1W za2E6B$n@s2pCCJXsIKRS%q^}x_bf`0C4)`!T9T_-d#+SA>%ck7!NC;`z-tP;Se12| zFE}(-wa&dm+7`DlyUf+&Hpz;{Hm7=t%Ft>r7l+W#Fqf;1DLlmGnAO!=Qk#D!A{)bC z7qi9Gln{ywQ&~Rm9|!v(y{oh?D^Gv&@aeUCjg*GEQ_*f{{q5Cn)oapg8>1@<1Pi>c zL^~4B{`AP6j&@mUWfR_ZX&|^cJ>4Ww=9$TcTKxr1T{14q2c}iAXe~?gWyeju?d^^c zsg18YcSRo)WVZZhbvT!cf}}i*hT@LSn<=d+rey4~u;ELS#FT`<@Y<{R`lZrgC#oEQ z!^BHiebd*6b&*+Tg&x$XsdQh?L45!lz$3gi0K%B}uTuW!Q%=Oe z7C7Lbu$93bzp|b=WS+hrJhnW4jg}s?x}Xdge~K>^73bAUmh(Uba3pg z)$fwWgbx!!PsQxaa2RN}+ocxN1}$1gOAw(zFcq6o>tzBXl%SqP(9d#dz_f@;t@p_| zHw{X(Ge>zAcvO>PMwYl^l5wW^DsyElqy!YQ zSzUImm)}y2ziR`55-hYm;a92nxyKH#I*Vk1u-Y>rLPzn2Z@Mks?))1n|2dLh{cNIB z{(-(y!hOP)=-KNJMY#Abz4vf=bMj`{y&I}FHpgR4 z+Vd#Vg&dcDQV?jf?AR28Tu50}RTcShORYAutD4YOPAJmwh4+diR9z$BW))&+SNrgO zHG$hWir2AmOT&ay#yZpdhX9>cBufA8j3o1=>TDfPX4h`(l=Iar(bQ+swk0j|x#Ol` z8VI|mLfGO%OAM!tog?}h$O)qtqHR;haiq=EIxJ@<5wj5Od)eo;!Q zetAAHH0(tme)c;l_6T3koLDpRdVjuV<|+SbK2bhWS@@HGiCM*oPm=t?BF#{yz(bNYrg5c0o&O=3O~&mtg|Do{OHLVijJh|B`j8{( z4;L0?zkU>uVr(*2sSOgULF=7y0;DfK?#Z?GJ!f%7gZM5F%B*n9+R0Q1n^9h=vt^!C zDjUbp`Zprc=!IkYdcNW<)Lj!@*!k~xzUARetV-Oxh$B*r$W>~#l^rz@x_INI?FSyv zM*I@Hta~!*yIl4th<Rj>;Zu!|FYm%T);J5H2Sa5t%osCB zT?vk;kleq+tA=bMkD_AZcHRZ4+c&+S z|E1!AtuHT;!dZUXTrCB&ot)(R`pp+olYi01$fK<)G?v8ts(mXgAp$C+wt0SP3uKrs z3pKQ<_j=<5NUNut6&UzYYBPs($59!icCT!XSHSc69@vy;QksG>^Zjhjus_W%ka*OI z@*|ev#*z+qX8b$^Q=0+;N&xgEcqvlT+|kyVCmh2e--*s0XKd4y_X&GlZ_Rtdnc3 z-N&roZVsxd&%|8inGvQXC%tBN%Ya$qR31?_mRURBI_#@1znx#Rz)#F^Nh&4}i6;+Q zC%;v%u34wMdQ(5~4sLILJYZ#F6w6t>lz5DAI^kwm<0W(t#QCE7sR%JpR86YYoi|+e z)1PT%mU|_W@X%dtcZ=hRC9l_t@cT2D>et>$x#>0({*5SY8P|~@C#g_9w|%&QM(0H< zDyAp7s?dzQUToEZ95FtY zm7Hir5|p+mWEDJqlD5Q1&48jn+Tv4Cl+{#C_i|QQ**Q~ac~#i=7IevTNo+YGCHy9) z1r1BaJ;l%ASLxzjX+wvqb!Ut1!7Ezw2{EuSSteH+TC{f#_v*TLDTP$m1S53z- zZ{J%&owHl?q^^_mdK!&&$g>+MBXMkG+kbS zK54Y{UI7g#XS<|kN7q2d(5C_2-me|7;Xzm6|Citavf2OBv;Vf!tw!0#f-5H@{+Ii| zCfkFjcW1Hx7HQON(AcfY)agCDzGDMb*FqL`?-KGNt_NysYa_h^BXfo+b7N3Cbr7US zaFP>7=lx#P!VZc3df?hJk*VA)2ImYRjc{1tx;M_D(i(Tazz#R&K0i=6qS>sir#IVj zKHNwV+VC5iNb!3_0!Z9q4x^sWh+tGWGvq}o)VmKK{bwWrBu?Lv1nBzfBmqjnwu4gK zOSCIPSB*H-y_Y}s2p|rJqXz=|qmh|@m#>-I+pev>on$8f(GKoDDO6%ON9-lchn>r| zXMhq&zvSMHWP6Yhw3Rk<8@~Dt)fBf24VWjp4YYE5FP63}G=YAg{x`VYT?%UK5HRO$ z{2k!{oOL)@t)O9j_=3t0s4Jm6SG7c7tyAI~o2l%kS6KE0^oQ^M8vNEC)I`YH%rI2b!7uUf`kS1!e%RZT z@ZCxAe-deHx=eYaygZHUd%3-qanc7-KVx?Wr~toZ<|9`r=?l>cDZU5*(^nTQU`S%5|8)#6cW=4n&0{#6I z=LxBX)!zJVq}Eo391TkMg5MAZ@Fm zG;*wd{OfAGoRxz+*w1^d&tVNzfz zDxB!32u;}I$aHNwHB-}l3@MLiC&0xkaiFsdtZYT6#%JLIlFlPz>Bt4<$83Y`s{KZg z{B+)J&T)07d^TGn{fXkJ%(7pS9P5S~pK3}xvTqL*CRl#@dR6LM4yOw}Ezo7X)!V$c zh>P+if7xntM629-kafOx<&DUSaCmroV~}wwd|)Q$ZIybEdnW~+(z#8FHWj+UGd;_} z^}_r<+T(L1Ub#0b)Mf6*NCpI8q0_Qg>TLuO^#fWafslh z6DaHLd$t@}FoE1HrCaVDQrXmwP5PnLsTQ2b*8<lev+5zvaeYuU(VzhT30=)3H zVywu?LNbTDQlB;$88#*RZu4kCF>l2!CBtd|Q3CDUH7{3gOb}Q|Qu+t@QIN=j3J|}8 z!K5d7Zp43kAGri>T*fMHh=Hy^Z#i)4tzWgMW56UHfm!zDhIs zjoGWmX?w%7R7pijIRu~fsQsv%c4Qr1j1c0iQpUQDWZZP9De_NVM*1w!ZdTPPf6oyb zmDSPFbb~F+jBy;}JB6_xdR4Ywr(Eoru}%wRVnURd>$e&)p9kD#P5>}+fUw(RN6xhm z*gIeA6y$10lzsa&ymhpOLh_OP7WcN-J&I?VhtSyY$(NY$i}<1AeN!I;tBSN5RbN~a zBs6?;AkQXO848DT#v!di(xQUn_#c4EmHz|ExwL~){9hCZ2HuI}rzBw?dNbAE^D+Vs zS{ln_DN@=1>!WX4e`?2Ug|^bS$m&+OeMRW}*qHdM<CWPqTQsC$#oIc#z8DLLXlsC$gE|)J{x9#qPG4)v z65x+!5bU#Br=8v%MPuRo?DBW$0a-x4PddN7IzM7HqNy~Innm~~mVQgUBP70m$TC<> zAMcX*KFpi8rQTDfXh9}d|Hu4XkS*IsOh8K^C8mheBWKLx%{%LpTWyS?S0e_Mgr7F` zjzBaiD_LxlPD+GBj55G}4-jYD5IE&D3$}BOx{`#N-kph-6DxSRd{(c{XTBN=&OAzu zZ(ZKn!jOHW69~I9;V;r72(Iz$!em(oGLcXOm`4CLzxkg{cmJytR6}C{Wh(gF`aie; z@UEK!v>XX5*=z`NAm9zM36!+?kmiib+F3cHr4M|(&Y$%19m_Wlg!*zX1_?+zgWilf z_4~=ceO-S09UJ+6I^+KogXTLzzfrSH5T5Q;pGEp-%3PV|$32di_>lug8J~lJ2Xe!2 zGtgK01uII|e>ci}fzrCGDVWFZqr5+(`J0h5{Vn}o&st2h&9uJJx_`p~9<`Ln@0OPtG6|_q#VwdrYZihTU;ydL2RfguFu&0?Bc<<1m?d(fayB&Yk;E z`7%h@y^<~J@9AjSmDlB;f-^n9boP8YPV?bwaM!?^|2KKSVEp8hxbb#H1tuOTDxp#x z19W|0zFv*AIm4ztUE9v83|JPhq-d4^p}O=IZ(Kmn_(%%}`b){;4*N>J;aXcmK9C|xULS>06=u3s}5XFl>mdhZDF zqcXp597CI2boy8?7q};9l-sz|>54 zS-E^}H0&$-Sp};qVUr5oI$^U9s%`&Csxu&?2hgyxZ1G0z){;i%dh%HBD~!cc1qlhF9436Rk@SHozl7lIj7for z)7ie;V`<*oqJ2!Ly;4#>9&msB?@u?3XsVL)pVCM_pz^UCe?e(*(0PT8mM>T)pCB!O z8%*Dmm7(0JsG5 zlMe5WO^Vp8y&EQHr}mu}9d8s%wCnRLW%Nr9V!QIsHhL_ioTPt^uY6*J)z%{*92F#w z2@WwGn7q>Nq)@j6{2OE7VzkbT4Yc(K;viw)a-)7*Ng1iTDvsryxQf6_h-L7=rV91+RG~cj1 zGvv73xgg829K4Wdi=OTOI|88~n^}9Yub6t6j{uUg4iaA^0cg=$n{3d3vQD;HKXx+A z*K9({q1+-{aUFq5l#{&mi(x7WZN`BLKsTVRt(KX(RX#hcBa!?27qQfvoeEX^OvcS=sA+Ti<3~};zUPs-5EgD+n0;y%ao0e-acs-#k&K=jXSnWPu>dk^LR!9mJ6X^s2|ApRY+t0 z@TqE3Bw`3Pe2Fx2d3%iB@0i*G%m;i&P3+*mG~5?WpyA#w$6V$1NI!DiBnYoSaSz+H z92+ThDLgtFjgyOQMpn@{}viaY3-t!%crl|ve=IWQE@da^LxF!GU0I3LO>HUScr2K_y znCB)5;`T`s;h}})OnB!Jlj=NJC8mT!DdqdD+a0_A3m!_A5-qd`lRO|rlJTBG0JCqs zK~n@ad_sP)zbwnz)|lq(@_*f8_eEp(;TsjPeT~sm8w#>9k?DMWGJ~tb1-8u<4^86F z(&SSWj(AHI{beH1?VPPFjf$0S##@?3HXGNOwIg&g%tR*5d6l-@ z)k|oIN(_>X+9uYB1L*9$x?*Z<_jJR=19jK91`#FbVja24l|ee6$YbQ{|8$(;gUxvL zN<&RL%Uh`79ut_E7tnqi>Ej4*%8#Ddyc#5vqK52z2<8g~+2M`%H!Fot(&1>O%Pu6x zur{acvB(CzlQDTcD~7#h?T;-8fQOWGeZ3tLj6H^bVY>B3VJtd==jgsi7$f0ty!{nqUFPW!^wxDAlC=&l|&1?#fTdpbP;HX z3N^t=V^`H%!dtXbXoq0Ax&Jp@YfSkExK z?efnOaB^`1sQrRMB`t4Q7%zYC-}e9dPqQg*3xas9U)qL=~-7 zlKHZ$K)qy(TCU0D)j>w?H6VJXGY6Hd$5Fh~!MPE?xqp;e1wNfMjrk=WAASWCBn2W4 z6henRwmUXRF_DuXZ#=zLE}t}|uU`DYP-SVKP+$y;b^M zTeD63rflRjpp+EI#Hgj+ZrMG`4hsx`s+a?NlxYUxwUS&gi(7VjQ~>_Lf`=h$wEMC?CCMrW}z%A zX6f*8gQEc^fdT9g1TzQ)hOD(vCB4SAv8S`ZOY}!U6%3|22GMo@|7B22!3Egc|L{iR z->J3#!UyWdl|ecnDFqIdnk6|=C=`hF4(<90X3i{AW>&%D_kKl#h;^Tg^lSr;cEm6p zQ0dMb38x^I7*ylV>_*!H#uq9&*N^^tkvtGAX$6i`pnzs6BSLz%@PmQ&-$`#au1kDS z0&KgWx>^Rf5xD?$F=Ci53(TBZOD@)z8geE9SQIt*8|vytvFu^Z8@~xt@MBiGvpN8u z=R;E8-ZpSN0Wvk8&xV}Vd7}3RJv|9L(>`x)EYlowv$K_fPFN=}JwBT8N11vQP(aKA zWUqT!TZ+6qL@$Se;{6~0_d*nq0Gay!fn77|oVtBE-nQc}5Ua2H<^as-6f{u_TO*{X z*?_1QG+xiE1FCWvfVKXZ0}B-R!Hqit!(jw4hC{=_lL6|xJKL&&kpJ7C0><4T

o$;dF zC2?o!=k0?6HV#$G*xHuP{`Rn& z9}5%`i=tS^$Mj?zQhk!Nq$>?|%&MC$GNip5(_*4%4^PM~_G~}r-7Iygd*;TZ!g@J5 zC|4o?KLMG=bjxcoxRT^xA<+f znqR@Ogn-pTdtzdq`$%Q`y=vr^iq9=Dj52e|)k@Rv`d2W&^JgmSkWb4%jLm4AyE|X( ziOA;h;*ZSiKQALJSRWawSSw|4fwq!Cw~5uD&8Rj;x$1=9rGmo)8jXKPGt9L4tvrhM zBV)Fn$iId{J>pv>}n*iD_^4aCe7G7T=`9_}ExY!NNFa zI&3Z4nzT_eK*o=0|ke_WbvHJbvG{MS9?i*s? z9e`L3FAUhAkx~c%7j^9G+}A*1=WHT3fo@aUn<%sZXR*R~Qd?X5M8J9VO!MIU`3d63 zAPrA8ulRVA{9qmPl`B^k198GphGYw`drk5&T-_%xLA;I+Idndi+z`JXg7!A2(I;I! zL=6lj3sD0CbsK;#|6Z+MFu_eHel=vHs?6VOhw3`pel!XQYPx>ucIHEe3>uy(5R8JoPc2f`xNt?>=z3tOpor40)E0I`!~jnbW;JAeD=^%R z1gPll;lVLxOVWdF>`3Xa(Zcm2Wlzx1X*po>luR9czXnY+)QZ{!Eju5yIt;L5Hk3$m-0D23g?i#-Q6OSy1Tld1CW0}t zZv1SZxFw(iJidr$E*oaF0E;21oE!R246QO=s3zapvElo7lqv_B%owvKJQGSeI#`3j{h%vuVNMZ(*TJ>| z9aQ6YTZx3q?G*w4hTg*ggot6oON;BENMtTKAzz3tQaWmdqfL86Tr{C$@@u^TbhZ+% zNCX8X2q*MXN;sJ$o_rBBssyml4UynrL#iaKY~->myV6V>PC#V8zbFvqADoPgjFile zUmk1Q;CBHQS3r}+VXXJ%``c?kM3hL*n+Sxth-1AKtTVEhh{qbN-$)!GMs}zH^%^pO z2zZ|;V9m7|P^MgjA|8@4VkEFf*y($U3p_QjJ^YUShDzJdTHqTop{QE%8jYY`D9f5# zPI_9X?*yzKhfLUDodB8kAH&Z@SJF^fup`MzhxgXT#0~(b0k=W{#S&y5f;OOV8W|@$ zx|twQWm8E~1O&)MJPItaNB%1R{Px~QTZXLWdmHGlG?+sj?e}Ifr8?6>2i++G2$0qa zoGVD1qOUr-w6#?mieNu9w3uSaK7i!AyDTUg5mLQzULdt00azR{lK{f&&jiC#W#W(u z{0-J)-_AD~u5=)a9q0se%U;gczyyd1nEs(35Uj`Ie1Qb57JUq*bYrE`hy(D+t3(7& z!_J3Ah9*u+!1zf7vPI$|t$Oti>TCj^lNE(Pu`!+i9`9~H3Ast*C5dRQ0LUn^Cs-bG zZfSsg->9R+aRxReUO*kOteI$OJvABG!tZP7>FGte9cN-Xi^W#PB^>hP$rF^ZUPd@a z;|K5`e#Z&Zq$WY2_M3K}3VZiXRQewl3aU~(iw|VM!>9HaA_&)3dW04y#c)Dv@W!{6{q9?SMkjWGy!>ej#zdc<@}l>96xHCSC^hA@M#W zieiYqi3ng4wv_^e$RS`n1L2P-9Xq!ntBYVmRKyuhq0_(5oEP z1j<40+qcC+Jh}ja5o-?khSp-i5Ih%#2BM_ph9@#LU6^x~ba8P3+*^_6T6mTfj#3pi zK>%)1*rCE!tJ5EQUHbiy0DbbmLiwyDh_$4C7F-AjoB-$9T4?#Jg^7(c?8U69os>PN zt^pfq*s@xR7$S|Mmo$J4uU5GJ>!lL`#959A(Y>ul+QrRqdIQivRI3wNoW4*NDxw1f zscbyGm{b4Dvp)jvYmx|bUXnn3d~w=@IJDB;fhso!8+kH!5kUNjd11ISkVYPf^;nbR zcuymA;7FGX_8~^}qZK73M~ZY8Y^YCiDCf^Vd*&PyczkW4j#zPll$2Bzo((-$SY>;g zWt-YoSCwK9&6*<>){0T!6T_o zj7&@@!@Ut)Gu}LCQq}Vu(ur}UmzS5O&Z9YE;kpM_d*7N!J6Fz{Fhx4$q1peLw|w*i zgr^Ku#F}s+7-tAfUx{}OX$BEz3;@)uSft2OV7w=vU+;T;qaDn`^tH~Pf6UZ1#X5nd zN`kzKw!JtG;V)yUF7pnsW*GpqUv?tZ91a&?s}rAJ>^B)0O~wENVINqrVuik;A+a`O zJ#>JMb6;6!9taKEN4NG(1{B2JS< zK_cdpcz`75Qr-1mh){@dJvTR(%Wg;$06OcDBNXDK05llqi*V~a3!)a25m_5>J%NJ@ z-Igt{fTkJ47ndIgX?sTpF)J}bSEV27WSk+PU8itpl79VO$6s&G_xx6&KY2v*>{&gv z9Xof%V45jkwyZWdTB^D@IkLP6W>`Sbh`&38;#nEyEd;?gLSqeW$$H|+LRLPG6#KU) zri(s)6eFGWwU@~o{ul~aG3^*C33O~Z! zGPHRT4H6?5eUPI@UkClK`3}ZKIRu`e91oJtVuXoN4V(|6(T`NE5Sn}eCoq_y#z19V znPLXe9NpYrQ%|4c*Nb%wH!tJ>gD!^m8Wvy`P61+(vGkJY#mPd)4uHTnO;lsku=E`N3%_Cr_bken*8r9HwqJ#I(Oq63{@3QpFWKq^YhBc z&Jtt*BzO}C^g}*Jn#=$Rza&CyoTx;Xju6_})#aOP^FqW`8^j9K6K{^fu-+SPN2Fzl z&v7f`j3W31Fph;j;w>5OSz(V(;d?sym|66Ma|A(m$k#MyNDbaqMCJjh?Yt#YQV+>lW zy~GQ9ypcj-EM?cFo0PiO&f4^k`F=3nLPOP(l3EA#GZe7+Ynq`bxsUH6*(Q7aZPpMa zQL!D;SRNkhgu>g0JFM)%6}inuvXb9ntTo53)ukPNF0+&67AAl}>%)rAIIu|4R#k`m zAszNZ!KbdD!P%mMrgJU&i)##0p1=ji2w4!sTcDR<*RVyn51AwC5lQ+IFvi@b@5FKP z&=&G7ZUwq4ooI>=N=g=;>9bEKsT!=;VI+v;8xn*npzvd8C?R1|4?M1W548g`1yxw% z8{PZacEcPN7Q@Sv*~QQUM{B~VXvvgDb?5UIn*e~RTefa}WV5rkxA*EbX7c8H6_-vO z=v`N}p8SoBH1LGJJB&@j&~u7~-IjL*de5YtG6QS#aad0F!a~rOiCA$R(qT%hT5i12Uf>M}%4N}j+CH3sdYd+DJeAsiH>k9I8?AT`)&>O3pD^L0vd z%M#hJcaA&mxrmE&!T3o0lM3FQNhC8O*+8PFp@ zq=4JuN%l|?*L-pWk_eKKkr4<#jNQ0>=GqTY;#f)!SWuXzG07hRltbhyffi`SUZg)KGzS7|H$! z$ckeFDNp1!#M>R(H%?<4Pg}BEd5>gqy|QD$EV#XTdDYgLhGqr4lUa`+eJ z>_uehksA6>Ge!yc7X-0B#M2DRsH=&O)HOI?DpC-KJCFtWHJG9f_@+4GfhYmbD{?UgDcOik8d3<-X~tC&oj0^d=>hgk>2=uQ5IzF3L-=}% zk(LjVTW!ZdZcw1=Q?$$n$ciQ@Ja7`DPY*U5ql<{5jSFTUPC0&jS@0ibQY$BQG0lR< zQf7_513&mXH!~{np0L0G6ca2a7c$*h|5r6?REJ&}yFvu?qcO|~$WggN&O^QKu(Y6PCYL~3;%R- zp9e|SJe2-H$W z$t?{i_^vOaMC`M0 z^9?;pY>TUFmq#|$7a)q@JIkyW?BdzN*BHXs6Q|tO2F}0Lk)p|I__9W*?895S2`n95 zPtktG*J%70e185AlKPqx`E9_B(yw<&0C8mjjzUaZxY5i5GQ(4vkNlOKl!zQB$K`w? zP_0n1_x#Xj(aCr6E8&H;&`}B+t)H|^_8h-*2UMzn%&oaAI60G#rv8(YH80|5w^vCC zq@=m5ulwA8JaB(?_2G(gg-v}Xenx?TZlhj-0Rf8YjX_;<4`u3tWV#i3Uiyhe=^M+& zMx469_h{WF7S>xVH-?94M;8v9UUhTDi(y0dI&Z5_>R)jbUAqYlGlWsX{hCNP8qG74aV8!|L$Z z;X&2|l~);u^C8rM1EA8NnNjj|6c$K;a+Df#>?{I1kh79hk+cViX%XS7Az7{#Dk}4R zgd)Sz(o$*^mmoz-?zR7h(#VOqmeWE5BAtuTh|~y;SP|8Tnx^03PAP7kvbp2qqo#sByArX9nI07 z)Q+!3*%HC^S=k|e5m^uOk*1LAhHq4V-E9y-#bh|6$y=I3IsmBLjI=dyAw`06Jkn^_ z`kn?5vIt$SJ3T1#ls~)1M3NQ!g?drcfSo~1Ey=atgdd6QvTb0+Eqg79Hm5a;rmni{TT z=jkqgyEjT!2>zle;608Fzmt8BQviCl56ULsxqiAeS)Sb7FRMcHYVhyc+{%>NL_G|TloC+-NgmXg+y455vmI@u_R25`?QS8a3jl~YG04u`$a_AvYP%1 zKs&&#O^|;Ok}IK(f~jhRR=&5M>w>QADXZ?s2(>n#%XWQVYARAi%(t#~JNhpa=5Q;w z{56MdOU^VxJVC}8ZC?iCJDiX87&k-LBprg^!!&aq$R&=k7~zE}T|!+2v6*~IIYD)Q zUm<%j*uXUa5P}UpdSM^NgUssC<*TBXu8t@Gf;KY2)WJXk0wU2jLMBTns>F*_lF?bw zdko%~`ZG@Cuw^3DLqh0@VCZTE0qd6lOY~&cToXWI{4%=Dn_rSL$|V;x2?sdzpY$z3 zJ%^dC#!e%4{j=9$21^!sX0!W?U{Y^Th$+5D+0rUmO#NQv=vJ+TtjmuH2l{0BR<)z< z&+DZ32OI0#@c;1G&e4(>uc9zKWtE|8m*ApfC+B=-pfo4Okt4FS;OJ*bR!~U*2WRf# z1;Gic~C zH600W!P^BR3Wyvnh#@Eor~QuiIdd#-2W1ho&eG4H#qpyV!&_d^VWRkN{0>9+g&kxy zTx(X`)Fe>alfys~Zm@XGvs!Y(ioXsoi5=(zOTXYAGmG3z&7O9jA=TAMj|5-6UgW6y zm^c)@;jAC214)gYpllM@Vvrj2`O4GK&^E2MxWLw=cfj85oZ~G}L}HVCJg@&X0lz*D zx9T+53d2eg#Y;zqZq=;jLK&Vwp<+$$9>^}pzxBvUsm5AVr6VNOA08~YX-!bzb!-w- z^v*pi4#~md5p@3nm>d)Gc;$5(lK&wj^7607&$|hFocL#zRGazu6Av06ENgEto^5}9 ze>z-9xd%EeH_e3j(r zf*gIPsrv=-)4HK7+_iyyCV5SQ^Y_Ig_%e(JuCEf|)9~$=sS(qPGX9b#^66mowPYYq z7^;KBAhnle@)x<;0TImHdPDPV?7iYEZ}ghX%E_Cp<4|7ReY(juM6r4LEW2uo1|8c} zj7CqR`(yXIuNJ(0?U8!($KpMi`p$k8=XU+28~j9CG4z`Qm9;%eM`cLm{@#;gty7!U zIc8@`Z5yp=d?J~4x%#-!*d;n#`>bPY(o%Sk-)Zbc>y5}n=T@~gGjGq#`&D6@uj-lU64BGWt?|J?Er04Tc*SOWjapfbH}4Oj22&}dPjV(_D{20q@SJo zIyLY^ZQ76ZRUU%`tFgYSho*`0-Mo}@NlCKq>YJlD582iB42!%Jadnh6EZ5HOv08yR z{|(8H4Q>Mq4}aH5z$nCitYbX2sIVygbIGh6K9gUiw531havCa-o<^fDc zu;+!UOc7J1+x#xZfY#=nIXFCa)kHr9jVg(8$FQQDjgIg5cvWCQ>ATSpTm#bcOxq>N zH-dq!hsmg`@wRuw1nnAwBg1zpJ!6c0z4-6K)vdB)ZU*H&_&}$ePr; ze>}xW`$rVO@>t~QH8}7EXvxmH(_DpG#f6wC z{#%p#r%KkRG?-?;xS{lNMS<+ep^IlynEJj|T6srEGRDOo3WPf8NPM^L6W_P-acvZEm^Ax-Mx`al%ABAy6jojvWQTF{ZIeorCYRQ78pvsQJaQujx zX6@EGCzMl{zC(L(Z(Lk_{Y5%@x(`d%-uF+1$R!l$E>eJ@NJ!qK5Xvyl z7}P0T{7Y;JCpQVE&^X1Pg<|dTE4S6xhuT~iZ5$s^X4cGSrM25ZsS=}AhObq4U4lqL zt?O>(cmrQ%3ZADj`XujF#20tr>!$~s^b$9m_x$|In)jdSfgb^;K^?NJ3-k2gi>U@T z)t?(}tn8iAEWLRv5$8R@OynR2sfhs$-3xbh_Hx zueO<;5}2Bu?(H2JHA)$5wNqc?`Jyy~Zz#fL&=N^jnh5PkRE=VM{`_U>q%P6--NVDl z$OBA}E@Fh%l8fxX1t8qa%hpSz&FPi;mxS?)V;<@@_F(}zY@C+WuzgYU{@`_oVV_!u zjyyW!YN{}|S*HbKTat{DS*z_Co}jTGD|o+%GFs5q#;PgHH(iLa?~tM%?F=-V-c`$% zjLIXvEp@Z{c+q~Dpgpm&d4?tN4>U5$t{-;Me`MV2wkFN$LIR1$>;kgJ1LZ#bjf+rI zJ$qroC-<8$t5IAuvp&|$1m;GZ>)}Uv?C*TuI&qAV;amL61>QG4+u2(Z9TulWm3E!4 zLXd&8#tP>}@e8vnuE?moKc(BeZ_#$dlBjf|*^RFjZSTqRQ0n&O;mJ^|Fprv$ z`=V!-c%$T;Nflp&@`5OBk|mzRj(;(HmoK1^#@`LL`Sw70gArFdd4)3ox?j+GK;pL& zk|?Y3_ft2|O|feI3xWwI9FVwCjG5}BS5rf83Ut;@+trAD8N>G#F2}Jm>MAvsw=J<~ zbxhuIFluI)emE=MnLHnn5ZPc+Ew9_sRFX)l9v`=(@igBh2q*$5KVHx7^k zpA3pQAd1!UJ4jI0i49!*wMZ9xDK4OUiD>GPqoW#B;!)2FA$I3n-Lk3bK+3kDOY)7SdQ-w)-wz)?d`0qA!4_@cFvE(Trec_3lvBfVLKM}U#w z z0Otdr5$_eCIO{L`tL!*Y&3)iCY=WFSk_v}hpl0%V%@L^Z#Y0&DcC-q~x+R5>!NP}> zPOgBz7dy5^YCElHDR-3U@l7a)AXX35bww0GGhGGRhq0F~_+@IQB70RpS@m-N0pAqM ztCVdzN%if@>})OIVX#rKSyBN_W|01CoxD_E5(1_U)Y5{$*x~mzaTct3e1u|d0T0=Ud!N)vVZQ}_ATV?vhrmIl zAdU~CK_1l;prZEH_qmWE6t5BX3Ch-pT)IDUuXutB@<~EDv?@ zpen@#b#V-DMC0Tg&Kh720Xm!&70*#E_xFRah8b|rA9{j_JOF5&hy?i<7$y|=fKrWk z7>H2;-~bA0!n`_q_wYf_ClEOB2rq<60npQjv8s(gl~AM9K)u<$-F4CZ>)|}g7)wm# z*?W?LQw-+oT`;x-6i)(_a4D&e^0NXeh@<%DQ&6`@6&rs3#Tb3KXQF~@iz^6Q5;VyUzS z_!C1j09hC@L74;gtws~~2K$~vl{fH>I zV2~;yi9*o5n9;j@qB;CD%iUv@~ z<%R|Y|Mi~>*Ec@Hm{|p5NMt1i#qJnoFz=1MgvKW!ycj@94C+C_;3@&wpN{~AGz9^4 zFdX6$pt2kR0qZtE$i+hQe5RhGEC>jFtpVt(Vt?m>3SKGay>cStsf!4kO7g<-U|dE; zeRS#$6o^ne9lD^VsAdoA2Gc=w%26itNadlLAkm*^r512eyaU>y1jy^1UoTPr1Pt&H z4-7F)5o#h+omHY5y$nxo^ww#a^h%4n(sb~^8iNA46r4{LnmetZWUK)CgNXewVUnSX z58aT?{U__sUWaHYK&nS})(G$O6vL`)ILHaE^vEr{_&a-Rkfg_)i|k+ z?$iGQ`gS669)!wNZGF8oL@*`5j6t~)&Z-Zu7d0D@(E_deuNw#r13kHicC6^j`E2OR z)lkY#0dtyLKo=+U(EuhvwbNkjnxSG#C(A4C>=K09yfQQu(AY`Vnh+cpC4C@GAI_5DvXG2-x)i zGKJ!rs=9g@Ar5B?F#&J|bp#=`%z>Q(qBi!Gd6Yqc?t*va13Mw~JNZT^1|CEkT8Q8) zFidOriPlO`97hFNh{{yd&DM}VbwY4P#!;{nGHr{)pn4O`k&94@iyc9uSu7XQBcd~p zCqKLd-G62asPH1-(>bf+4`{*wFTEAakKX!*;?byt{L1v#&+jN3LNE$&v_eQo=lOyO zuq`mOyNxcc#z9U3LhooPO!FBydmH4VXxJhrR0Nj=8e}?bWRoFVCG5)#bD>^~Dx|2d zh)~F|JAo=7Lbb*c_U8l%avPMPgsCG}RRC`yBO&P!R^$jl3LcV>UVvTD9mu`Ah)IOr zL#Of_HBUW4wL-NN?Q|e+1jFqlf(P;yVEn>rPN3_aMo)mQ8`0Un@d)-$k$)aLA0B231UXK9PFr86&g^3OQ z*~-aR=yBZp69$eL74k=cmxDqDwAsu6_kbWQC|PxbxXrAcp8*vx9K!s7(f{q;BK7alUDG_Q& zqlpa(vKb&wkcI<24ji3Oxek`lIMjTH|A{!GfDX>IAo~vOE;2~cL0JN!8^B)?F%aya z9{^)289aMn6um}EEQDk{^nA?^qR2+HcN5^z`798Kp&6(QaST;_F>Nq(kkvZ^5ujKJ zNCjxf+5v$Ug9Ts~vd@LWCmEZjviKWPq<^qO`5d)~U=sru3=0fB2FV}*M<2X+7yv|x z0XG?BqzH9$1`JGSWZnUo^fZ{z=(zz7=Gci7K~Sha5Oad?NNCX`VP(NZ2bs_&SCjgIK8~Jcb$vfchx_ei0!9_xBb=yBDmiju1G3FyQ*pfFpb^ z-+iq5B1JOjgr;D~7ur$B*g|+26kP2G~-0*`>xc>6IfDHKH z9q$*ph=`P6vfq1!^fkbiR-&44mU3m35)wZDFHZfRzQ+G@2fmVVfrL^e?#^j4cI}>B zge8TbpU6KBJ{r&&pr9s{+lh!gY`Fx8mSRvF1hTNuMnh>^A>g0x8o@3JHBunEEubdE zCL$urdj@n|h>`(oeIE4)#1bgijVoT%<<01JD?P_2nZkeLP%Y*+RN6-njh( ztQmuz)L`MY808DJfd3~flOVtlZ^KV$ViFNqzU2B#0|_wnr=6IE@;ZA03S5vz4!(!V zKPk)~SlGcHJJqG-x_|&6{^SnF?T@hXxx0ri)}zH0_J?qfaIK_a{ve#ev;PB}0U9I7BoV<`Xl)%GyI~h(dXFF;1gq z!Su+I2lAG~jQ?C(q>b4}aL(k*OSs9NrOt|tgDt@{I9zgGKWQTci!N%OQ4$KcfIWJq z%|Z-tP29yH5gXg}npCC$@AK!Wa~$_>P)BZmfPE)%)7QwmFkzVpSS6yG37gatL2-$U z#dyOx!Ljk68U|4h9@#l(l8p4fZ+~w4^wEznm0(`;etY+%<}B&&dHI;ufG+cpI33{$ zi4CDSPx$-~&Efx{`Eiy`a(%8#{+ej}m^b-;u>sOcp;ufPM7>!iv85DyZ+1f za!*#NaI(f$omQQf8}l!@IGR*N=-b`fIvby$9q7?HJx#+awW=xdkk)qln|q_vvI=!i zsY)#-RAsl+bXtI3hD50EvxwnzkAhSwHi@R$ud7)8vf1E|Lz~X^2QY0-Z)aAju`l{^ z9fH#s^c3zrt4eN7!qH51NuKic-E^`G%T3c_z-$gnS16~t>kISn3Tcg_Dgb;8?v-ZS zjtIQ<;8JE!vou|*(a*eoeR5`fNb$o(U6ZQHs5h-i-y90l-8jC(Yb{@2RVHhue^@ggO8Vqhl>AEm zQO;biJe*_GwvywRM?$)DDW~R#3JIygPp>iCz3o)PqRQ|FRmI+F@{RXosOmdAD!}PF zKI*HH0mmlS9=|e$=a{xzQCK}EEN8&1()DS_#eV#DcAe@8|8x&=OyNYq{sginhH5Rh z#WstL!=uj87KIrxy!hrnnaJ8Uh1%=>Pz?(;wr4O-fi)oqv~x)7W_SW-P=_v zID}q!amCiT#46*Wh_ien&CJJ4GGclRBMGhY;Mu)aX(huXZ87Lbg^P=VB@0GL&FWKV zh`bPW!VNalnP#%Gq~Xa&(?XLP8sOSq$`-Nm)t^%(SyhQQVlH}O|237L6uQP4vtc%4 zNZnf`jiw{{6RdKTs937N%f2%C{J+vWk5AyF?ozv)zJif!Vt076aQSMAj+T1bIwgKHLaI+1}(sZ0l>j*OI%UxUN z=65OmHOy@=FhcxtdPkh4Ek##Gw%~H}MO~@FNdNDvhSdZ6BPjn#@RN;(*SEG9Fl6}7 z;<22ib(NgGjnVJVAZ?J6lj`Vi7I|kl(6}_$@#&Hc$0!qKsjRGzZNhBW$e-S1PF_?B zYsziuTp#8+&1YHpmfxnvy`eraKlzVN%(XOlZVX>zx}S20bd{!hOgFq9dBbmW2O2wV zL9TmeT~>YyH=iV2hUYnG95mA-H(nd-{MGcP-p)DNk*4A6<3E@gjV8GD#EacJGLu=Q_TL z`o&RZS=A?;TMk}t=3K5^eza-RcKz`NY5N5J<1GF(gUNjL!tk#1!XwN5r@swgnDJ{W zPp4~IYc&MB(lNH%o2$yHy4Ra`OJG*OH@mxdSnP#)7#e#DV&qiUHOX>3w$7>?O=cnI z%hPId+%DJKwElW`?u~Z3ROhqN(bx&@c7qzdfY<|AlRo5 zyWxlhbBC6HkLh0?&`cy*b=%<4k(KI5ANGhJvn>sv?=;GZ9T@LPb5v&m_tWa zt}W$ty4hg!ae9oeubgwfmenfNVlot?>2cl1)9>&`xJPaVxNkUw(AO3{WP1C{I@yi! zhX_YW!c2GSrpdUG=H}{&t@D0sv*|a|<%=zJ*KbBs?KY9RU0>2Dxtj5k9%rp!b&_H^ zVPN3+)U(sKR%ReFtCwzKXNTi-qIJa=1gvfX(`_hAX^26pHob0V zjf?BTZZW%FA>Ht?;933V7!d|X`DX4svf=uHd_HQHO;n=%*PxVX8u!%w860^n6e5ZG zqv1@-vo3q1x;09j=R~ZlA6e4SBx~kPWP>1xT_%u zH+B6E|K->Aa~UF{_V)y$D*U?4Q>eocKAcBk?@9QS(?jK9|DS>X+YgMivCR}!+y0n! zmC?plYX2&TJnB2SI88fk=K6j~q8dBl^UbP2^YE#6Khwf<7J?#_4%Qrohe$!af8`H} zqyP73TKkzO(ZAiO&0h4-`e-l8!6_i&l^M>FuD8Es5VL#)Ns`Yej%j&|;S32syBYk) zO6faTv``);Z^CPA$sJCV)>B>H*>aoG5hUEPC*|G4;T`j`=T;hBtZ<(_Xg-lY@LQ!$ z!q?UwW~^8LcgKF27nQf;6Zm3bz@w*I#=id>12-wds%5AJrE1mw?i|@@4PUQ)n5tDQ zrf0?fzLw)eK%-&j@jy}eYwkmdw8qPI2F}Y{;-F>G7Nk%yHVa}iM!GFNc~!;><9GN4 z_;C`k3C^ao^aYQdCP!ND_Z3*X$Ja?YKB;H>uODkIt5tPm7TxONYiEHUU7CZ!L( zdq3!r)*Zh*L8_h}(D!^zbw|}AbES7%a5yS$b$FqEvTy&jH+XV!`tcW98S^DWO?zcA&V&THIKjL)3pscwa^<6OJ4S)-e$DuggZ`t z-0A?6I&aEPo_zV(p%e1(g{)BrB9 zZ>^e{jQK}=xNNxD+>edEoJUcr9Fd6@W@G^~Vb}w5w`eEl{LSK(aEPI#PTJ788=f-0a*%bMzt7zGy-7RMZgLM`XmD1mOVXXRV|JuF5z-p7rHgzxL zof+z8wFol9dg>Hf)i6gxNug4uG=Y}X(9adK z!)hl|E?|y1zkgA(5*Qc$HbM&POWVVUk@|m1<&35_cbNp!_jkbW-|HLPqZu7j(uT$V z;l_@*cd)-V-Yi@nqA~*pmbZ5Tq4KqDfOq)hv9P`V*gKdt$}sX-Y%Dz!4}*By+s?PV z5wf+I*UnqLjO{+Mg^_d^0sGlI37K(>|8k1h6AjNIb+qS&Capp_wDZTpyd$){Ml5)> z5;6w{o{tP$ijZ?&;)`jWnBC8^(#aOr6zb(ZGkfTD${1ke9dua;Pe|;BB)~e&= zFiL}uT=FuVsIu2QdsWLyq*EG(Uv1llRLE)|{m$lKs6d9knS(^Ioqnu8TA1VNk2-Uh zXPlbrOY6~pne)xtk7puwUAag@`cRiy(A@IS+sIM}!bv=f-}FCUel15F(+O#Nw)1x( znA-A{Qd`L?6Hw_ZmKUvA*)ZiZ!)?uUbho7&O@95n#8&3 zt_wLUgr15)XMFc7wK+keiQIwwe@;>Jpy;K}Stxz~M@zZC-J)w`Q`8%k%5th+#e^tP zH>vsPXh^c(ot$d>_pADNkM_HObKSak?NTOVp_U7xcLNRspnph8MO8P>M!2a_Sz8Ur zq@6ZMLqp{k%>`rRac#X0X+HVnOUkQWByn4PiTV-)o&1{ir&Kx%Pi9^N;okkYEv5?< z&s&FhOx)E*T%|(lzSHl&ibGHvIMJCsI3gjr{~tI{I-L2v{roSoFeeT~&Nh#gO`12n z9*87}YeFSy7-}J`K&T{+;F~NTKZzRGtsAxzqZ;`cU`Y$8f%A9MB4Y@BAwp%9$l4FS zAw)wRrKR+l2KaCcf(n!+JpIXlQiWzw;XCk#eyyGfP2P5E0J9M?e(!(;h!aDL@&}d9jk*RuBPY z;KNmJ@uenvey~Uy3*DaZ@1c-Xqs9{>C9X$l3dZ?|Bx#hWck0Fs7iqIBZi47Q;Zx;-W)!Zuk(Yqv)?dv54az z2bCM36H=b}U6jichm#f+9Q+y6d{_p9Q7|@6dh)yM>VF!rM}gp-jJ#v94?n3RO%0Va z?cQ(7`@ejL#DH!d1GoYMgqDXU{&ahy-{;r+f875z=<-ZHIYQ$PaO+M}70~S6?=7PV z06(4uHQjo{tf|jOXoH~syN_T%e@h$)PrCo?Ft#y7tNj8wv{vdKe43p6USX7RS2ez~ z_-{4nueFmzHD$cpIg(a8Kci@HO7aZFZ=|q?b4?OQ&D2iH?FV4b6)1*l*;{qBP5vGn zFgNZU;5J>eW6+iDnq(j3-bkd)`PRHD@N-e--=T;{&sFCtJkzOZOB5w$^KF!G0L8+8sAUJobG$IfF9Bq%&;}8+TvNHKl6( z5OeW5(c5rgozTE^a?u83M`J6N@7fUtF9S2H))JN$h&V~ zY`fE2Yi%DTt6UZPUUK-#`)sRlSMtEWdG62iX_i^}HR;IRjjdb9JJ!GKNKhRd*bqBF zcL_=@G*>0(N4vJV> z%WRs*6$kylwR`?dtL|(o-fI&V1+YVK+c|(jz5H_xL=McjZNVV?tMYSzM=!!yRn2 z#d?04R%)N!PmUM%lPTe2EQU^L*T^|0%XC~OD}CkR2;(-H3k#(<`z;5PimxQ3cME;! zO(r}1R-pMUU?b4wZaxs1PDbr0QC32e=6nF;{9>UTTvKFato_y8-mFV;=!MWVHz3%e zkKtF#FNbGFW$utXlrvP+a|Gkq_@~+sCOhusQ~|6v0=j&64cmd&%AP626o*} zih5uay-j)M;$!G}PC{zj=eYEoo=IdebKF85FA&V!?=6M_(;&r|5RxfP)8w;OePQu4 z`_(7TOR9&@-mN@Nt`N1Szff9{nY<_JTh=3e`>%oHht$Fa)#halL;kdE%8*ZYTLIPRTc7z?B2A@cbbKZtISHZ73+Xc83T&2tu|?XI-9k6GSuZQ62e za{D|rMoG3~GccT(@iOxK`Gyd(N4A0Vm4h-hUFr;PoN8NlhttD>4rQ_tHEXVvIi;0n zC5=55Bg|I8NgC?^{z;7dRWlc15nZg^?3V)b&iw#}YzS!Y`tBWFnje?aHV-#wI^|25 z%jc8bOkI3VjWN+KG!f<3xMmu=Oh^A`Q5$NVc>kh8K9jU$gE|8|0m(r`WF|pjEN=cV z_TeS|#-&?E&Fb&N8&rN`9y4Jo|7K_}KETh!ukq>VqRk^2QRDp%mYAl{wU#$Q%S;J@ zt>ktrOtsmw-=+cu9ClL<;p@Vz7XMyy;7H+d`O;__5+@(Rj$QcX-uGz5|3&it^UmM< z>#$~kbMSVs(^jr-7{AU8$!vSxeSD|UG(DMveSNEwL5UkZW_Y_m>boS#gaHn#s7tye zcb8Vj&#!y=T7s!shXS>*hg?khFL9+1*1PpoIUOx!rqNdWC4+A_DlNXKK2%R+?{mJQ zc$_!2{i*DXa$CFfXIZT~*o56p+uUk1jo6UsEtRfWQB~%p2Bw{*s-9wY4L8l~*0*0H zsFvG>*oKN>`LZ%fEM@DPPMt52u@c-5cw*z`KC`7Kf1HAzVJJO_(TFae?CmjLJI;YO ztGLWRcz)^5%Bq>bIa>@*ooY=W=bUzf0h!Gsg$qWZLP_jy@F4r#w!?n>{kL+7Ie+ll zQd6GA1-5keK~fM2dH?=>QSR}RCt=OA&eh}Ee_F@Bx~84TzBx4iVr&kovQ;eKr-F_2 z4dVQ>yu+1o(F^_iG0Mq*u|bS5dx*-V{`#jG{(sg?=+A%ZWB=MOckAguPyWWR+2$LD zLj{=qD<;~z(!!ZxgX3Ca(MYD_J4#3LoL1O;YCn+{qk%ti=9?V1x9Eu@{L!`a2f1|U z{#TcXn*S-%{jc!V{o_8}{(1frR=q<HRz~n?7VhWsL<19l5-$+L7;SO+Fl>l# z4!L!D)42AaPO&pXOtSM&P9wr^f4d<`;Y90{tK%sl)WxIa#w$MCy>;B9qX2((L%H+W zOaA?VKqBBGP43yS>Y5!NFTHQrtbRUzmr_ji1 znf}t(F)-8DH+5N+h^%Vd(>+Hr-yPpGV>+oO?xvNiQlx1wYa!vbmGi^$^5ok@=gGi$ z?`)@f3F=K(U(2Dwc$mD(lT~GQ<4X*|OduWFX=bY3yvsZ~`(XN9#9!&nhffB^$!z&V z3g5c#;)ua|d8`&tsiY-f++u1&-Sj-}#e|smCS|x4k;a52Nd)6@CeWzt&V&eOu~4|6 zF}L?)c5kX{uHSj=^?dl|%*E8+ET!N#p~?$qEl4}!3QB)j;Kr@Cr4u>C@gY_WveU~x z%)wVoa^>Gq#}+kexYOzpo`UDon-pPB%GTyu)AS{IEH)nS;8kz!XyvrWxHwFgv@F<- zZfnT87$(tAHH=9gqu2}in69VdC#jX3;f?Q_?ZrjYmZd)`Ic$9AWfL~ejnQvryOZLO z{JZz?PrT}jH`q9}(s#I+u9r+O1Up$5@JYufR%Lsb=R_ahWEp;}hs|EKe$HO4%hwsA zL0k4WgZ;OTqfxI)-%CL~JJ0F0Ywx=Kk+o@D?E((YcQArGQ0>E)WBo(X*xO4crJVQn+RJ^zTEvS2b@l7A+uJ8WZ6Pr8Y5#F zK8m)P^F@dTzXWBY#*Op;@+_RltJx#lpX(BH$Bm3&<5~xur)x0_(_3X-#`hvlCF^{^ z4jtc-WBkZr+!=;RR;$&+a!=!Ej%^wj4;|l76{fLZlFz$|jjJ$fPO@+83m2X>*a&4! zjmWcNva8U{N!Mw%>x`jSxuU9V<;tfe*%YM^|95V6D(1ku)A1kxw?{ zABLNM5z$pY)FmgF;Cg{XjStgS(ld$5Sm;oxbe-znakccx#&TWf0%<;s zVqTRW^JMH%O%LmIUfD>~@I-P$xo*Fi3cJ~DjrPox^RYhTDewI_2(Mnu{l}q;j^Dpe zCw$)1-eADw^73w$u@63^@Y)^7lD_)rlV*lV)U@)-wL{|AQDdugceZ}!#Wjna!d0uR ztHZNh?WA4=Yo#vQ9z5i9@A^)dlD}8d+j)PivQ4)&G*E;;ptn@I?RXSTH|>Hf@1e)o zAXu`5v5RzIlj z@9O=PiB9y9!8aXcRs}^~MU@C|48PEOCB&Hi+o2i}b6Rv{);E*$&4DTnwXnc@~(QQe8n4z}pp^1)f zNm?`X1=wC*vRnEgPR;Kj(LFMv@}<2#Wx-=zxM)Y3bP*f7Q>LfJC@I;#Bp5%xm)f5P zH2msGxsdCn`(b6{HTMs8DbhumtcK274eaRbH)Chtn81=}=Vxad-HVSE;9Ua8lQk0} zap|$oZiO%E&bP_-i*9PCi?r&b`O1Z78OA%iZkJM(T~#Dc)5YYaMXuDQHLU4i6FBrf zzSKP*M-tl5Jr_rxleCy2;$fLjwW%iC&R62rEa!o;mYM7w`Fk6By}}X&QxR(aJi9V$ zcy{N7J$M^4&$N=$W84{}aZlyzETYMzgE~T=>2aHd21TayJa;O%oTy%T?H^@mcuE}66L2W7?xMZ7V0~4?w{dF zQ*RtVsQY#&hmMH+o78KgbSspVfzQZjXM6qd&e>8~nC_h(#`9c?w+!XfLSHY0g$VLr zpG+;+HPsWeP!H-9VKeJcbStY1->H@okLHip8={Cv+|sS-6^{|#dg|4}eVEhpB6`F64{;j6Ed@Uu*gApREA4`R$ujKst-9ziNl zm~c@K_wjR?pIT?hbyd5)WNiuy4vz8Jc|t5{-&PxH3(i-c6LPrCAnctiHN`cR>nHoF zdw!%ErRviL^|DIx%1sP=~gi0Rm^9Z5CM#o z)UMkB{9}-w68k^x&+Th5EW}ywUuj*)Nl} zzO+kYSL*n#WpP_tx1TkPxz_KPeskV*v87^)eVutUdTPz4UsN#D*++4LmC>y|Fz}xT zL<{P~x3f%nGCR_oPB*)z?e7JMLeT1quAbX3T_!7zk*0QYu9u{*bU$gKJ9PivVcz1s z*YVOB6lOz?glFw}9wI;v!t17&{pp+A2v)h7JYZmI9Qv z$Vf>aPATqR|Gz5Q{{P;V|1;kvB*XB8{i%Ti_Qu;g%Z`#L1@Tb8Z;?$0wrczmpl3%k z&jOYwtxpdfXNCX9LI&zyV3MVU(sLVRO-3}}2x8T#fDIpRGa$;~0-KdwyQ2X6rn*lD zlz;`N$B%KqMF?Rh_6%NJTFTu28^>S!TP~vi?uJ8`4mRp3P2hHjz|aU!Pa?eH-&+Vs zY!-kqAWSMy02{gx3>zpmsvmB?AU%IR(r^F}-;+I`^i4q_18EFnkjE0B!xI5Y&jenM zpx=Xx=8(SRnDg=jpKUu(?FO-}?`al5Xz764IGB9J7NPk;S8jha${-~eBsYw+0QFJj zxH|!M-Gkm+@TBIcpFe?aMKVD6k@y%=3X697aTJk^%-w*jCE-KyA#cc88Yo29wAO)C zhunFc%i>}9+(XX zCk_;L#BkK;M^u>WO(pqOpaU|5{C|j2>fkp47Js#5cXI{_?jVh8VBUpwiGuDE@-Ia~ zd4T(`9}*d}14cduj4D*c#5@uCY1nAZLXK8P&7>rS~bYU^EvY)4yiMe;S-)_OSp9!+U8nAW3In zrj_B>UAjym4U)mn=n?A)26FS#^>=fCXUniG7K0^09r>*xxd0#w-38P$n8V_&%652V zi$MEz23{Pfl=89l12-j2yWk^Zv=f>I;H`+4`5w5k5d0i>{~ItI7%+fYmc3RAYINX% z_Yo+mlQ1+WyCR~lA^|tVd%+@sKwuVs=~#msKqA$^IQ{T9gVZE))`MW(SalpIFW^_1 z4s;Ucecf$m;N);4y%|PEBXxZTxO{oH%@s_REAV)bT#DszDH{evlKeqgIswG!k^s2P z4BRrr>j3&3Gdp`z$^x`nKsG{)j7jk}l5FV7_<(eQFqBR8uTEcD0qMDjx3=HP4S{?#OG`wv1x-}ruFH6P#Qp^tLqrw<*Va)`4gmWdU8JTwc4F5IW|H&xJ0m+FZI*zb z6Nm`bS`Q<&zC;kagISN5Xe&UoG6rvwY3TYOY1=smjSV>iWmkZ<8AmiPi?G9uIn5tI?4Yfgk2Ga&i z@zjhAPNkY)({RdtR@fw1mD1^f{Ey&nPS6gAduw1jNq;Z3UeI1kcg+% z&~X7ejsda+KvL02hYZGNC-j*S*GH)gt_;P}BP28(*P4Z5K{=ZVqA3ythw*hSG8Bo2 zflg?`BV=6wl)g!z?;r6lvp1`)&iZYDx_KfY%L?)_^j_$;4)2?!v#InfcZ^CZo~S(`=bD+@g2uu#R=d>w!eDwhAXER zJc7|(-rCtGR=PSpUI;Y+xdTwx$9WrY{HZuG>4G0ypt42v<(Mxydy4*b2nk+hNgP znaygh>+jb>N?9Obgo7?F4)ofA`Cnw#SU^1jL247Eu7NX23`=H9_5k!uL4{ipLN?sU zLrq;E)PT-&+e$C!*u$b8a-U*zXMK1<6HiYq?5qr?2{OxBAoeD+l~4^DSjf+GiqaD# zR1visaSNe6%>>*$q^b|(_r~ZvHBd^0*&*vEv~(vVHw$>B$PEBkHmbmlk*TK!+0}dH zMBt=>A zTimyA-^}ys=N*}|VEl`H&e{${yqaU>u&gm)808{cfDI;F?+yHKL|%s3g1B!ck72dq z1{YX#SFpGtm9Sp6Xb4h>?+L8P(DBDlaA1Uk?1?=+i7;)D#pQrw6J$5?K*H%pE_e_$ zU>WDDD73vM0YqF_0zo%I-tM3Q z>+L6Qs|#zOtxONM7;d?c1_C$Xr4SSAmmFX)(PNM}X1F~r&s0)E!muC>I@BnLJI|!% zDMP$OYZZ)bMOcTyftF5Cu!yxggc4NmfTl`Uv5Kos1p#Z5u6UaM+3bILHY2Df{Ik3W zh;e5*bXmdR6|vBfUcl4@+%`67T>PS=2%j~?9bgIZ9`{DXGY}L;x~0JOg>iWoIFbw) zNaxxh#X^HJ>ndcCY2c!y>anvL3RB_(h#nvTx~Z;6a{ai{pPxZN89081qKG>Qf@_UW zqdSad1`*X2+xA@V@f-|ElMv8Of7mm`c|zy6`Sz5yg&;^~V3i?1<(>g+KPHUf0bN6T zOdA^2ND3RM*+??Mupx{=*n)Kf zd3S<_2dtGh3}+x@#KTg4g_~O$&2u2KgJh6q4i-WYis?YmOolr!yl@x7Fmgxa|YP7Q(eeu9f|rO=a|A7NSPt`l9DR;u~3$GlaYa83fO0e z+|IVO;H^y01jr7>O*Nmf=WpN27Fsd-g5dv}I_~=*EODK&k|bGRm(!?(oF>5rV(L0h zO$Ge()@?HW-g!A1VlD4D2zk=a8M|Q)ce)^AB?8MbZ1AEvI32;=q5TFtEa|+2j=U0t>=;~q zeANr+{h&gd4aPSRaqGy0ud+UUiFl&&#XqXB3-3tVV2a;^!Bd3hv zu|v|G^@fxQYZ=jD6cg`={|RG(zIJ8FUl4Q}KWFcjH6M=zl~5RVh_zm_HD82u1CXl$ z%IYE6YemkTkQ5}rl1AATJ$O}7Z<`Zh529%U90jRQLxPt85Ef&|kOW+RWuOdnaikg# z!VoC1nz+D%jif5wcQ=e+d6h@n3Wzlgs`c5P=QU!NEiXv3M9p zu+`}XP*D!3?jubd&`E6vwNgaj15MwEXuV&|FzAa{0o$te6)8p4|vKL ztZ$EiUgtG>N>Pwz_OgKaY2XB^Mex9?U?9rYL%dPDg;ju@-fqicxOC9}VSYgYC8T^N zFkj1tAdil|P&*5&R3!zBkVikK(L%9?Ar-BBh5`6VU% zEF?oHM>SMKP+6rYzK%FF~zc@@+0*BFG=`tcDh1LH_y4|lj4@hmAv3ywYLXi@`au@J(s<6t_97G=karV^MvW~jSUInaM8PI^57GJ|{bB-*I^pe~ z6$yENOdtz$N)JStDbQcIddL+FH;^~T zl!m{Cg=6BZYa*0DN|0w15s{M^EPyqzyG3T}ASlrZM#$(9fo2twc!D_|oC@gLZ@ZFc z%T5%`s-XyX3VHp;HxPtrK|4;!2u$V&3V?O51c4Q$Dp@RBnSG~Vz6}$~o$rNR*3=9C z7(D~|FtXW#Emjwt3JIKRY4xAagX-xz-xo3X`En^IC#O*pfXG_!eUE{xitu6i9Q>pv z7a8`z-o~&VRB(76{kZK0b3MUeZEX$wvvfm-296Mp_1u=Ecx4WAKMdCA0K>=d9(pGr zBRT!YK8$4GR=zAnZx_g{$n;+a6?D+5-_ZNr!Ony-S+6Q&jUdN)3$%?Xa|`TeLI6#C zJcD`vN*;P~{>9%0`!a~}@kLGh+TtMk?QuX2S=hiXsOZ9p+rBIFgU}qoOr?QHff7vS zY%*TF2vUYi7+I6YB0&!lz%h1 zxo%RtI)D7wF{v|$`+9nkKy3SLGI}rYal8e80&fgO7~RO~ThkC0L&g}_wx=rlB|N%I z&!5x;L#)5T&#wV^536)ox1g|5|K*|He>qJ&1W6So{d;S5Ny$kEufv4k+nl7@aeq5fs9HWaBGT2cUoe z0|zWMKtT#VtK7SP4-`6%nJQ)Y0|eC=f)pBQ^q_|V(hP*} z4uV{8_Px&sr9%(Uy+ne#prsD~CMp-kmwyq%g#R>@%rTkQ^N+ZFVB^7>gm8Cy!0cSX z^7f66qzXHPqc}7jpy}@XNzW5b=`S18q-BYq7x@!p#uSqiGpSyyb!4NL6~s)tSs56s z5AFyYAo8~oI#!TuNY`&r=4{xFshSNu;pTXz@;*{Dn549bl+d5p54W*B~)io9io~d@x92?o^+bqsz z_ZB2O5?yrMwk!2E-9Mj-5^@t?H4Saeeyua6*V=FH!=g6AIMe!v^~{CLT8o5|ws zZ>T!*KFrSUHiqVT#dI8LugG@mW>R6*Q`RA-fn-`Xe5cmDxHt(WDrw3qeWR7ul=wc! z$vDfwQntNg@!Z*JF2b5+$+*VWz=_q?Zs{DprMMqwEbG!;F~Vnhq%0yt%uSc)S2ham zSuoA(=J?*_ZsQ8yKZ7~OV$Ko(Ok6B|XtA!7N;aT70=R{ql45Ua^W~z9C)=og_p3dX1!$U6}=J`{VjM7Y3`))D+ z`2JDVZ&didz06e9;|}(zDs5y&YvxXS_KB><`_56YY4r!SH^7&YH71(v zuL{WRJe{eH&B#(sd)+ZoVrW*^Q_W*zeNAO;xpnB-1gZT+iQt{$v55#9#qEjBOQzix z2|~vN@oISjV5ZuhoD)E`RCkg3ZD9(@keiI6)SY#8zjUyGc%LZH9hCa^?ND^SqPS3G z;~E3Y$XG5Q@$P2i&hvl#pkIo8Srid%zW3_E$luwfBbFy=`Lt{=pH!JMB_l1}3p-r4 z94sV!4Cj(qW#f>o))TU`S@wE)QjN;eUYXY=x;vNe=%HMCYJCEKM-C66F zvlz6!UEl3xrsNbC-kF}DSShsH-9=2bGJYa8Y$w5dZ{tYKz}j%;Rz|doS;D|zBw5Qx z-#4TV?ZbkXcy^ax2-weKcNeCb66U^Yd$)(nx~LT9hYIkf#|ocj%ywE;tsJ`3QXRV$ zF+Tpn(FjnxH!Q;AA}_hE%~=7Hcn3Q4E? z=_I&+UR|j>&K49Dlp?F;XV&4Js!}AuglY3TUSVKmgum1;fqNxchnbn-$W^Q?Jd))N>T zU=SED^je=wy7AcpqyxgcTU#8BOlIaUX>M2I6CAAEmIB>3^dsb=U-PonTkMgUDK+th zG&|p#w`7_)M@+26@~wA0$6?uBiQ)>zs7bG@yj<1!lZPi{gF)sL)BHBcyF>Oh9plg5 zNsmt6OZUfJQc2ENs~ZsNRD4pAIJy`B;pYzRy24Q6?5z3fbVf)*bI$J38cn|CYcJff z9hoKoUEY1QjE$?m%C}PS#hNEcsf+XvDQs+P)ZtY#C^FEqZQ9Fw_3xB@Xw0!+{Zr}J zWemhHm9G^V-D!+>u)9MJ+C^(otu0SvWx$MNLv2NM_`P#44F_KitoKjtf7Y0YpBIen zdTSc?R1CAkrX&*^y^u^5Tt8JF&@Ny-noaY#rx~5Fq$DcZxLqvIlF}Z#y%web*I8T{ z{A3+`4v*Ehwl=Nw_SOz%Z_7D2FfqoyevQG#y{aHLpzwMyV#)lgccGy#;jppfM|Ok! z!Q|aayQ!s}tG4W+`NX?p40gez^<rr zV0Ou*5;VHAY?Ux=RvvRf^qqT*u0=}rWo_8F5hM2S5VB=v)+1XU%`WGu6`Ne{`GyJyrMUn z>i)U8F#IEI+z)YKE=yascv;+oXp=66-7{a~58GlJbW0y43ic1kT5dM$P$W3)R8QB( zs7|fT(eotFr9a|Wy&E-6)|PE=V^?E+zJk+^XHCMZzeEe5Y@WqmgP*7-JlF`*Qru>F zJX1s|+}Opyz%b)HD_c+)=kdjLQ87~Dtqyz4#}{_SGcyop9%}M^9}$(=o7-yU?Xna~ zW~9a6=auMq9cJzq8UZV{*k`GezZ@jbt5uKZGI~1C&OAYx4K>y$aL@TbtY-6%K4t^%dFoP z`d0IL*?A`LFN=N`?H;g>qHb(^p~|u3(GyVxf5kq8_TOcm#;4`B zt!j2Zjn|s_XqGs@G*Hl@*FuC3>}51dd4;-HA^U%L=>GfQM|RomU-O9V=A55yp4dleq_ zsh^9sZv^ZW^jymn@w~ID6RP$qX+e(T=jx5f>PQ3eS%Io){kR;hwVAK)jM+3wRNVC5 zrQHM#d-*%c9k`SjtDv9kA8Gt}T0bOZQ7xM!ojT8-`b<)8X793lY)}3#*MT!I06iW> zWRN*0G+!z_?IE3BvW34%zJNuxOVhm~Cwaa%`X=yw%P~{Xcsm{1y*F;&_N3YQQYt~;I{ zCu{40GB~%F+gC6!kb}k+ZIoa@T~V*tbO^d&()5M1QSTGf;p+u1m$XUq;DZ9`ORm$2 zVQOnPcSG%u>6Es1@Fgkfm+ZQ#Ne4xFMR*5R`JQMKRJz6EI%|%Bd;+g~u0xz%&ewT4 zFlJ@+I4!el?GX(cTM#iOjqy)!#`a7lB^pa#w1j1x8e05ZOA_c#2B$2ODkk^{MK@v+urY@8Gh;YQo`y% zH(_3^rvjWuut1*gV~6rDXztZj0%f^I*`Z0)MqsxyD^m8||OpYW~I1?x*+A1__e;MemIy|b~tbp-UAI003a8s>6=k;h8= zYP&Mc+-ud+(l-^JF?Tj zJbz=8@Tv$;)hXvvs_=DCMSQ^4p;X~Z(JLvZ;hpm+aTWdfUlhO}!Q+$s4;BDM#s3}5 zUn%`az?~}rkogb?6)%+viOf|eDj%uALU-*w(mn~X5<{-e%z_Qbm-}Qk1 z1c-mXBtrB*lQ?lvZ+m4~&h1>`V4J}}XTxI;vyJIzXy@*E=7siKvK_w6utnG>dLTy1Toi zW6`y~F}?S`pXbl}{{6lm-#QMr+XZXQIj?KfHO?{4u`_B>;#fR^t$0uKsSD0*$cDI| zOog5rcmKIH_swuV$yp#fwDwJtq`iNCc?l%-CQkG$7LX;iM3s`bJPuN9&64pjZNE?O z6ghn+nU&_HqWZTRKi(Q<_SJnFu>!a9!2ZhFB-+V2Guh}8@$nC<9jRy~#mu~92wuA- zbOoEszXZp+7<;_R@C%{4L`nTqg5qTdoSNyMzNYXuItdz8B_>JTd#~o47oVldm67t$ zjiRZIb@ovUkp=Us5x*zpyUzH?H^)ocFEqZrLGB+xyx`}=KBciuaS$97e94VBAz9_{ zY1~Pwe%j?f?X{keH%fcR#_}aNAx6BiM?9+xlg4Ap&)^~`m~W&i-_2+j8kMH&KW9B& zjfvb^xRIga06h6SZ$?|>ke~UWJyTml(>87hyMH(tvQ_L1H&1D#o!vGa|9HVjt=BaD zC~`Q|$!iMTP;5_rW3nY}2Fa+XdG4{5Qaqdo;^quvef=10M;2oZq4e{){gfS1y(0n`Ns-n1ypGM zEDvNPjJ}bQ#QT6K0@Yd@WQR#i5gE>#zN8k9#N%nM%QE{1^(5M!hho2lwmfP;$ z0)KF_dn`r8=PVw|6rwoH!2L2k#cM8W;76g?nV+9ZO$vteXI+Biq0?~U1S(hkIVH7J#0gcy3XnI{X^wz>5C%j0yXUDd*$r( z#)p|Nq@M9~<%!yEeec7%wd9%4ga53cOs;L-Z7eJccNS7nizWduHg%VllmiG?$e2yF>(BkAT~xGW|j#O#9Yr8V|>G zD)nNw+lg(sU%{u?9C!`Q28iM(*IN6S+U@1z7!^)FnvFS09umfNEq_2Q21dGHSRRm! z`bEnP@)g5pBfsWVW$<#k3GfX&+lJOkhmx9>7r~Cx~IYUgQ&F;tJznz@p z-9Ua1MC$#-kP3&4&tpSzd)@xC+wzuv{0NACk zPkar#W=Ood(~L-o`muSe?Uh7dLFrD>ykXky3Rih%_ zLCYkIJu9D+#bm%e(jCj$wdK?^F#B16D4|m6vQmxnEZx(!^09|8?+h2VbDDeDiek%z zG%#l7?HWay`{=`a+0sp+IBR8Huc$l??#|3>+}L;JB8B4^U_=dZE4YBAcE^;)o} z>>)RrsirApy>BMM$L?rL?2_ZpM6e$#0*>lA0Ki@@ZfhS7hoM&4)yi2QymTDT$ zjxQH9YWSM9)fla{?YQK;xi{aG9c;+9&}&$lV$nYb?cZ*Ki^aT^J?&`)_wIc2{%L4= z(Q=J7psBZ3Ud^8B)p%y>%j6P^g58al{pK5F15#`|gp&OD>;&7C{4Y^!QvCQ&c?jI@ z+jvt_E_1tdT?sv86r$+JNG!R$|NK@n&zmWx@~#+9LGq9BPqkRX5zbVK8bteDk~bPK)R6-L$-^8L%`}GF2fuZ&SH? zXxX^XYWQHs{n=``+><9yE{K@jwD!u&+p5&g8g~s5#@A`7m$+o|Y&rF*;x>z0e}Bcg z43quIPIU2=(_f;aKbJ;b*$`WSX@3Ug(y>9gkI^6?F9DjHpBY&3S= z?Z&;i7Hf7qMBMt}0gjp)#euLo>03LB?-(v%kdb^fLakA_R3(4g|Kxa3Pdi}FRBPw5 z<}4@H)Oz}rP}<4B#&knY^GI2!R?|=?1ralX_z|>rKnwK!yjpeKKuOxd)Y@$#$ee+OD)>RPYzq|esDyLVUN2& zP0lluvZz7wmUEuf=H}*g&$S$r*idOFC}Td2>0DdH&wgWLU!P9B_jcE2IsLiq=PqgrCIj z8*ofz|8yuuO3Kj{UxsjO9_ITDxp9C$e+jeR%xgo|*qwWF_-eRnB)Z~?QJ|DK@!q&T%cMut56 zs5r~TOhAXDI@DxAkWfHi;I)OJ$;QhPJ$CA)?zfS+3pB4Vi-QIQVS%GsXS4*>qJ)}( zh%kssZhNh&J+AUsH=e-Mj8S{nOEq%t4QXn3gA3tV`X_@kJ2z}D?-UFSs=61GI>f6* zot@L6c`K3Xiz+H9fBMnWQc7b z^Un6k!jM+`OdRd3x%9KAc-eO*M%hSbmdlGt@m+>pVk^sS&@SFO+jCb zxbK4%_%YGlP$fvmm`oJ+#Wzdo>6mNrHH(T7Zw#pg`$CPK>t?Ni$cZaw@X0WL%@V7I z_w+2U(xl2zD6L?<{K)p&=rn^{5v!<0^}FgXPSs2t2j7Xilz&bIpNL-R{pqYXu)@dE z+K@3o2WQ>QtwC|h#yv_NGtt=C_Ci~a23Q?2HK{i@yq`8IGOGmz-ije3@A7W$H{aNw z8sD2sZe><2_MU%Ks^_SYXua@4v7E`#V4$<#i0f+jm-oTb2V+( zE?M50I@MX9H8TBjza}deKQ_^4Ls~gROyn)(g5`4;mzK|oh9WnNP>0ys=u11& zrKW~e)>J586~~-pS>0a@*&l$lm^02RJYHYAjC1Oz9;$!fv%a+T&aam~CN#o!V?wjM zf^S~#{_3obr8@l1;t%h6C?&HP^jGVuEGCQhejXSe%(b&n{AeRcND0-CALlL|p`)3S z*IGT-F?YQ(GW=?XKH>ld@p^cScF;V0H0~M{tuBFZM z^q&Qa}3==#%Jr8T3XhQq?iJ>S?$4{#R!{FhB z`LK+^x@y|woIzS#nr==R5m>Yz`Tq%? z=eL@>%1L_iD=TWZyW9+~jLA{YadcE~yS%aGICo4oqRagKmT4pSv@(GeHJ+8v+LHlM zWE*v){BQodiCnUKmuoesRQO!31Oyy~6_kr`(>Pe?>2{$6yEAkTyh{8t(yom(c&TWa z&06+|xofdU^V4kW{;`q%vp=AO6LN3iS6-ouLQAbjy@b1T1_{j<>yz?QO+mB>O^*+) z8rrV^DGAGX&G%WC6y-6$>f;+Y_xDkiqLx^~#l|lTzZrk5cfF5O-mDS90i24AdTAZS4QI&Fyfd}^pvVmFjv2ypVn zy6;_*tKevM9qFw02b1#k;DqI~-2Z;2%|NTQR$!+^urKt>- z%D}|r@tW5=N1i;N+K;y$+8y6{G2}-&9^ti-wY`3$KeGuHw-A#s5+K_3(T#{vCEuMe zt#2(}#+cVR&*WTHxO7eK=y*zX3_GqCi&lGAubRi!)YuHWXG`gX!4APHm$Rf&m#;X< zNl_r&!`Fb_rLGVwInn4085K7Rj^NOA2h7nI^XqJLE$@ga_N+!TxAKpNc)mHZ8;Tsy zDXGWA778Q1okHAOO}&mgl%L{V2d|z!2xmt*mRNJgn($6- zq`6`%dBz6~&5La5{lD7qrl_G^J&qnYF!6Xp!1&2eI?3M8+VQ1rdj|ts+=^e6_p-2K z*aK{?V^~{JhrK4ENBmi2LUp;08#+*6=Fn~r9cS?$aSxFBEOZoS%NClz+*W1JknRI%ZasrAC5EclSSb4R;t;YIK1Zu{3k)bNCR zd@}@-eMHswW@4!)eYG{+yQYXdsw$o~`RljjCJ}WNrbScphh{h^zRKc-@vfKXyH4j@QLV#MVH`+biMNMsGRe2%WLij z!l-4TmPoex`IOOP_h%P8NylRX)buA zTo1jVN)4tx@OxcnH8rmiDM`kNjH&wVjL6g0oID=Hl`9idp%gN%nOX%yNF3Igprei3 zSExi~B9Z;PV-xOQa&`UA%XdNa)TU=WX5x11MC#_H`*Q3ot~2$m{2J`^xZzH`KeNrq zBEF-a9|OOCo#o!y&&c0a=6E9ZhPW!6UTRWg=rploS*ScynUB}_R~0@%}55HD>c z&;A_#e)8jsHOpMc-}h$5+v|pKkvIRo=ujzS5O)0MrSRnc{eKM1&eyVP`AyF(nCv?} zMb_3-T1IMeS>_mLE@pji3=UKS7c@^;Je>KJBMZGd@uJ z9nP*>r!4q!CJ@H(RpQ7*{?J*gdS#ZI{@h(dD*bnRyWAQYHo+M> z9{2Y5Rgh0X`7!Q+DU5%=Ef&sm1@7lRue-Mu36Q({`}+U?0b$Vp*A`?so$A4bIG8SF z{ioxWk1iAp&Wt*Y>%$P!IbC~uduHYAx4`r`>+s~?U2J^+*(1mdy6f`q9E{qp3~~Fc zK}TukBbe0n_a%w@-S}6KS%B~k*;&uYShMAo6&MgJd2;&dKi?3@40@TH0sTTyRX!hk zrn;v`5owDENc~4EpUk)`5`<8mzjUd#uaBqO=B%d&RN=}OThF>aqyJZ1cfad=n`wYj zR`wfy5wYm}e0nY}uDEMpXa2q#jzO9E6)Ievps>~7PqvQcRzjyFl!Rt{k^1*Ct75ee z8gpQXo*^Ijc4_D+x&uAL#8?0O9p%V(^ubU%nf^;H>5~7o6n%zw>{P{teNMeY0v zTCcx^g`Iut=XVLZq&|fPBVSMjjU*~C8?QD8x;##GK((vSG&C*ta{P7CbnwlDP?7l_ z%6@UR4GkrrCCAdn25o&0S)}#t?bhc6Wh>BIK{6gdCCEeb_nwsFlFm2dEexWOQ9U?6 z5)+A_DRA})vLXvG7|k~;Yt3!g$;qh`>c`6n>ELYz9OOE`p@v?FWngeH0;*C~`2*qZ z8pa>Oh^g~LMN_nS9wzJ(@5E09SU zhTvL$&*}%z%ySNEd=&)X+JQ&OI^Vtm>nilJF}P!o9bnq;K&b;8Gy>u{Iy+BIPd|n( zc^FiA4_4|rnu3`p zbKIaAtOW+Q4i=bE#Ky*+#>eM$KXQaIi=Vu_Bnl48v~FGL!XB)ytoRic^1?Enx_RrC z2MlYxMorxYQCI}G2?^}QZIDKonFlS=fh)|gm$$Jqv$J9_!58^j@dP3qXg7l@?v0I& z9P9t=K_Xl}*J@fEM!&YTi(-92{-H0>0P3B&%tr>^&~6}0e&NFHJ9nVVS= zwnATDgpd;lw7}}Y5X1K$KTb_dXovd31r_$7z2z&?k5b7?Mi6-$xeKwNIUN#Q=mLrd zivzO}KSSrr28?ChfPUkRItq&`*RLynOPV>@+q(~gN@1R38x(6l0#iZ8Z%qfC0F%&h zdmGy7hlrpZ=EoJ-3YxQ?)l=_{q?Q1066Q^76Pu zj<;Sw;l3EOK|JXm^f>pO+mhrZKA(_(H?wee)4MS2s80c|d`4?GYn+_X#50 zFtRTjjjrkI3xmnnb4JF9#svO)R$GONhoPd-^Y;Mm0KRt;hM~^X5Q;(pZAqVh7%l9d z5X}8`#UAf&Ee{4Gju{LkJJe@`9^_BMzDwQ$wSU)X^jnVu(f<`bWUpSu>r7NO!0J0f zJ%8%UG!l?E9cU+jse&J%Q^!Q2nHV}E2q<`bVGMW|Gyx%e4Zy$vncw&M34gzJeZ7Wo zdSc>jdATr5X3VvxggYim>o|KGdlfoubfHxRs?T9`FBvom%75!S)gntT6`~E?0q_$T z2DbOM#BOO52GnlCAmvFooUJ&*jyCv_C;6B{6T^Ra0XRQ$r@c}`;)P*~pfkMZbFHs7 zGR2hIPr?^u+Y=yxjdX3*BUTZSSZK&Wrbq*~P8)P=dzO}#=IhS-z_V2BO9SwBjITn|P`>>SS4dVrjuUY+8qB4Kj${GoSDu~@5H*12 zsy=@P2!fR#h_Th}QX8$11>pz1c!6t~-CaL{N=QMo_xaz1XP!6~oDU?yg_*sGG?Dp< zLl2-5-9-K#g&mVTHYJy>ZogN5>2iSyLz!T5{ zQ&sn%nb;AA0F!gSxo&^ZY=0J-^z@)51!y}$L9&UcsLWO*5$Qo*XWo7G`0zwsof6ti z^q|?O41012J#cM9ffw??FmFQ7?)2iK6!g=bCnPMir$eLzd8NHz&%RP*yKogzebAMb zNN%5R-)@>n+`A5ATC0(v&M+TR2Au1;ix<)V>vIb1*>6joFbvRwJ_YUPEwRE;;0M$M z72>Csms?;+`nk)OlVEnL>0oZ{P#|pC-&eYLZPh7ee*Orc#Q+9vatKzj@;N$UaJ>+S zG$QW5J_YqZhzf(VRS^vOXI(zMOd(T#K<+0F4Qc~m&!Ek@*T0&u0kT)TV~(DPz0WsM zdKN2W{68zQ588Ta5qq+R-P@?Afk~1VptlKaAB{Q#Trfi!I%le(+4U(XQ##eWR&8%@=-n4dS12f&VgEmN8vXCjAO|Bf3hNI2X*JN9 z{4niR2J)H4?(SgdKJx-wx!xwYrYs0;rGHOTJ$dcHKGv2e9_U}l5jP$z>!R&a$}{gts% z^)jm%arfTyDSz~>%+-2ce{YXTbn=T-WPQ;#HPSnvmOfPOsq^fa(A6HH$CVUt3AuMzwp)tZ3gX91ziQ9wg^SIO&-MDhRyZCO3Dh2#gSJ+&+4`a>lg>7c5 zvfoF~bJ*2#WY&!-{5`ei%7FAC<`w#@)@Op{R0X916_cM)AqEi`b>yIL$jZ(x9}&F3 z!vpt219*(PtII7Ghc(4FTznKVYg^~l&h(vau>CPyg6Dz@P*=KWQQVj;mepJ%z=wWw zncTHGq_!SSK*SCsgx5Hanw0cTY}n)zO%LwWq~#PkP7m)hk6fusVH0^hhSpt+Q81-sZ(XQ1)zvfHHAGEjU$nY%MXaxxyRdS7{*Brwvbw_BApFWeF^;%J5 zZo^mH`tBPgSn5xGi<_m@RjAcyCgAbmf^s1)tG|D`V0QhKouK~{9gYYiVrBfdaSm%Y z`ME=L$`8K!IN~I#`(QeQ+n=ZA;hXHP78KRRSSJJ=%_wCGvApSN*-8u4+7_y;Xw>_q z$DyCPQbl6s_+2BpG?v#OhXzdAT>&nRj?W4&UN`}F8=F(5e zW&R$!-OwIaZ6?r>V~%HodTc0@7#7Y34?fkMFPgK_?&%12`R$yUW5ICEz>!@X;hW#f z0U#bN)FMMuds}f8hM>$xdreP(bd_A#ki_49r>tXIUh~kNdN|>BHab7i>4EGRg%APg zBF2O;R-oEsyoLTqnX551wPy<^@k$(a#y`W0jl&oxwa?CGRnqaWd&tG!&|hU-Tqf7l z;0h3~PwJc`6ipXkR7mfWDLM+|7k{fB#Qr7Q7 zjP<^S!Z3$*7^dC{90uSd^;Y;!0Rzi=9QMcy-a-R`1e|5w{PmgSWU^2}l1GjG9GWt( zM*LR7GYUP~j>kN@i9b!fU}1jPZigpfyLkCW7O8TEt5}(kkg-xJ^%u>X>9Z@+)^F~W z22jN(g%eVv%+f1E&TnEAHV-%VYB4f5 zxZV@RxZvEc*nL19A>{Fa;uqOaMeu&{Xsgm|lUA{ARBnEb{wjJ9OS_8fkPVH3@$C&fXDEyr6inQ7hjc zA=hn)h^of=!fM(44L5wS`j%0b!NtAP+Wmmp$ra0Ub7Ni^ck~85uwo+|s?RS{4l^5- zJihn0*ugV~D&9=sOLSzDoM2%@Kxy%H-94>ZIUEX44^royIz5h!Ll3vzmJ60sQ)lcc zy<67vIR!Q=vRZwEJ@ME?yT*lxhwHR&rl{iZPEYf@Y+pE(qn@5fy=H6D@moi>_RTC4 z2mj2|8%Nx=hweKuLUk3_>W#>ihtnhbd;fT1d>2)z4u{jXaF4mz87s_E z?BYNuXB^wF!j^Fr4qhR z2x2BdjG{aFTwJlWH{mO0PMY02lXV+D^Mh7&K@7b|t@UcX>8)kssM+7O7ncRCLbwt- zg>GN@rm=FsJd*9d_u~gDom9DgDiZPs{&o~!`{zn+rv*%QmZyeWmj+3Qt984_^=pP{ zrlwWqGK*<`1pKjK|0=^Ub@dv#fbP%QVzu+<9~>A}4gU;bvS76wC3)V-^tiM%f=_#0 zn0K~;wf(9&-&oGY9G4s&oZs~H)<;y(UsfQLkF|r|Gf_4fDxG;pLZ_=~)g_aiN%q!=K#l@Ph5}0>;#9$-3JG`cT`%>tn0LoEuUzg z)pgmft*s8s`sMz*%4@A-D|EitaryvnmKgMC36YYo0(SSn)hAld`kIt~+9oGBOYO4E z&LH!(%4MzGtFUnQSPPPBL4*~<1>T@LNrY6-jH3KW<`ufS&XIA4&=gqAjed#e8%RX8 zm=Gyn^sKHbNU3>GH6V?78_CCiq;dKUu`&hdXCKu-jX!N5SGsD9&a;INoPrb-vHQ!q zP2tS7{<|V=TrVmOGE^$;u7z8SHEg+>Z%r;mh*0HA6>pN;ugP-W0M}VHIJu_b6rjOZ zAM;G3;@n%U{SlSvwzpuTzE181+8Clo`&Xt$R}&;0xr#p27K3-HH{xJV(H}nCv2jtH z?Q5v9{n_6zHL`NIts+(*zV}H}&3i}vy{Z2&3 zGG%}-e@#QX=xgcU<^6hP8UAwmz*wz}x<6g=De5xKufDoEozHv6rbif@P%p1s*I@=s z4u|=w&@lJ0jj2JxO~k@XykNf{2kFBA;CKQmB0R`7xMb(&CxR%sZuctvU6E4{dImX{ zMV#MzE%N?uO;k=YqrUX!a#fIE(uKmGDqf&sIC)qT%oo%frPA1a*R%)C+ou@4guKCj z1XS<_?pA2ElpMS8IWC@Naxj}-ki*tXh+-drCC*D#+07h84||1697q;BRbrcBbvyY6 z1~;55ub9)*t#2iF4?0Yd0l&Lp-6Bmyi4M8hn_l`9)wlc3#^SWMuu&~?rG?jsY$ES6 zg>ks7v}yWZ-$3V3<~|wIKKpD|DojV&i*;GlDViN4GQA%0;z^uf^UPYs zY<-N{LVr5(%VvGI^2Kk9xJc;VL?}G~R{>`*=rVO@nzRLXBEIAj+ARx_C$~t3=m2o3 zz)$o8rpuBnXI;a1uGVOn-52RUOZlh1lfyS79Qbz(6B(5?5-_k}0p@oQsjX-dtd= zH`wPF!lv^1q_ulIYC+U3Zsi2hOP?SiIsKC9>*1j)PE$$v44SAt%Het2a`DK*w)-tI zS#go$S;M*X!&=uPZ~GWWUOl4%j$#3R%;2Ya@{PgzE!SN;jFp6o3ux{qufMPZBT>J&WkEthA`V2RDn*vJAifj1#83)}fOcTlM8b~q=g%vqJojLb zjf6aF$V!DfaQ48t2+r6`rG2wcfUOBAUXmc&4LNq9<}Bdoya*XB+vS0c6F-|q0B06D z6Yy!=w&W``(D&7H_0B`qc^>yfg@~v)tL)^cv(Q2-NUrYw?jXBU7wWY>M+Be!q!7sJ zw1KM-v2=KnIQ?3wze0KUf!YN4~qcUY)GX8|>j_VD^Y$3cvfZjF0=C)3nMjmipt_#9K6| zx1MsfLdUzfTUfyQ6;fDw8>(FMfF}&7G;A`i6ahIFLKYVUCvufsI&hqTgrJQ&Z$l=x z7Kps=zG7aY>8J$CO#mt|Wn@Jb<06nle+Kw!!DkQPjJ?UgzySCz07nMEA_uuM;4vC@ zEzJZ8f2ez(3=SWz1xXPiMl`b=gBtz4je}*w8^=YYHi@zsrE*$rm#Ax{C*;eXWNyz- z{gKoa#kv$Ug4{Hs-&2^`ER4labxHV+@xM_-&P$A~FIy!lk2WttZm=b^0+Q8rka}C+ z+|&W=H(fRVGvs&xAn5BTcX7B(Ez|(;vGuH~mOm z`}gmkyuITge_Px&4y34(PoKX2(-E)ub89z`l$V#cE`UZ3_~i!6oH&ggmW%KgUf6Q^ zahgDq(AUcg|MhFgvch-WR(al6PM4^G7Qs9~G6^Z2T$8R>Nk~5W_*?{{%3X`-Go-BQ zI92uaj5UA`RoS!)*H6}X0p67G#Ct21Ny48JN&Qc)Pu9Q@W@3K6KKpfz5pZ$UE&-rc z*RO?Y1uzV_A-^FR{kjMMIkQl9n++wl&u>deuCKTIy5k* zz@4Q$k3W^6W~jb(1hlHZjkq{o|2pk64QW_ISQH@Kn$CP-i;kpC1_uUQXE&LdeId}C zm5PIp208Fla<={j<{<;^N6^Api9Hbn>8NK77W-!NYh2WgB?9-h!i5L zKIsq^!OH6u)dQ&!8bs7rTsNuxW*R@WB9K506j{*%>rQcB^529c{D<2}Pg~@EgiN|t zWvu)0!K`W=5YQO{y`Jf4NhsjU3cyjk2y}9g?}f}R59vi*UvXOa>iX(x4`A_&OeXev z|NcHO^a6uhr}+jSAhZUSy+GfS?=UGE7gQX;Txuy(ZleXjo&C*+)k z-M|OTZW_{^JsD9hr9e(J$PTj|ftOCIJysZrZ(x624;GRM$T{A}KlX~`EeD#=DcB&u zrJ8iy9s+%%rsL$9Q+U|1!1lU2hQ~Rq0As2Znp3FFr@nmoQU%;yPzw%t^!j32mTDf*^WlLFsw=D0f)8=1qXE(4FV?sSvel&E zKQgtnWa3T(q0isf1tw(W0s{j9c#ZUVLwJvyI}{|^W^etq&orR0-$8{CPymGU3V_1k zvWq~szG(g_rD0@*q#I|13}iX+BxyVVq&3`_otqO!WLUkJ1DM!OH`q=S;^TWUkIt`% za0K^K7S}d6Gj-ct`tQ}R{d;v6mw8nbFqxL^&qcwljM*jl!YX2xvMOP^IX8xD0+8R% z99;Gq(-FM$0FhMN||}zK8-) zDZ(!mOO`;(YiVn%2Ymk|X8tsyZE6a4*=VM*`ft>Cggr-(A4Mae+*A9%o%Guhuz>fS4%v8v|m!D6L z53f;CwZfSPv8XEL&fEwL1}iJH*+8pOkqH1HVlu$d%Y41>0ca*bN;QzD*aO)?vZb&} z@c`NYeODAP&~gBa@NHUJ@6VrgfY<7GU7hL{IT@Lxl+*=_@@*3EA4jm1zzw?^+x-)) zX5rr69?n7(AannDU4I3kUp<6=0BAOKIlB$Y1G5+c{Uymp)d4zTY}MA;83^F%#)wTj zz>Wk}fUI4r0%B!gXQsgcJE}eP+nc0x!$2&6T3Av>22$xFuYqCh!9mY#tgs6QpvGXR zaX|g(*x3Vt8*v@J0JkGWo)*z%Bt^Yy0Z~1oqTmm5?N?NRISqW{MI5mBs0qsAc)(a8 zCpKb}_x+!HmVB73qfPA-5WvX7atWwt)z?aaBdjHY(+l*_5;*dJw(HJ|Bm$`YXubav z0_KkZkKfm8xB_(eFqq&X41^^S#BYz_Wb9t@C1ZaFKUJn&Lg+I~>}cWS1*vl&ca{Pt z$}%)CJdA;{6PVxE?ht=BN_hn-yb4BdpIov9E9VfK=6KR8?U^qEVqmy9*yP%Gtd-jQd(4m z5VRi_FsT)ohF!1)NV%xAbY8_DAG8)>b;KMA{3Cm}oy1Je&gBf(R_WA!R^j&+x&u`o z@Z9NCkN>e7boC&ex=z9z&o={NVyM6r_W9rX=hH#D$gZ2OMK8_F=*`k?egkXu5(>%g z-gXx)3YeS5wgZSu7uBX-EW;T0`S0fxzCrL&5AdkRmd1xfORy?)yDy+(V-$|nVuc@N zkqhT8)W-~`|1D^%f~j=wCrIh3P|58*@u!umFBFp$W{h!m9ohpcg5`XJtJ zXdH~#1`x=WwOnnLkbNnc@^ju;<<(>tdRY?Nc>44C`^B7@qog#6(|{7#{qbbL=Z`UP-EQ> zjC~4-nMhW)l|aBh{Dr{()r*<0uvh8dNb#5;YTqc2(KrZzy`%@T^O5^Du`&R$P*YO# z7wZ1I-qrvo3C;ca^9gwPos=ejF$)Pr1Iw!Y^1wp{g{v^}AQ61_)?)8PsPe#pz%_!` zf*J`60o?on%FOkq=M#_wlYlgr?og4C)CJK?1HMKC&bK*Bttt1%54x*YuigPV$iGSceC6+REiUasSo=@U{!@QUW$71Y-(7iV$Y$F%;HKM@NSY@M;_w zjPe@rR>Y+PEDWVNHGo2Y^7o&EwQSE@gDM+8Krs)7!Dh__`=YuWK8_F||7O=YjM1Vf-sQp>eF;|Q%6u}?`AHbQo@2sU_2NefEvb2uD zpzZ$+CITtZIreY=7BNA9qXy4T&S!a3%WXTf*8CR~`4p9uL^?=4la|KoO1>r`CKfTg zz0mEIot;gvbPXt3BYDSNgTcO${CRX!n^jK$hz13bOdRr(^fw02zrMRW4`!-XkK^@y zvUDm&3Ghfrxa32^G6m$t;2r$o!U)6Zm|_js0R*fCCxii0JH50d4ZCJMly?_^@;SRR zxDc=pfP;c8>bD1^r*66FeM8DAbpLa`PoIRYARhMQw8(m9;feQ)37*F~JS4hN%;a(URK7gu+k2mJdMq z59h|#HMOQN5_=FY@^H5v+CLEm*B5sLJVg{IZUxbfDG+vF|tC7PlY_XoDOL*vyMi(v$!Lu@TF) zyE3W-6<_rMG>rG}cMQq@D>u}i#Lv#*0nz5EQ>V@$*REbY^&{YHSq9870VD~gsvvTV+%^c&z9sLV14Kk%0gZZp zhW?L|2wfq@!#st+7PcoKUw7tHKmZ}s-5u==TOe`?OM?M{#~^?U7}7w9)qsgN!deZ>=?`kDmB!B_y{GzG8*|AlgLer5!sHrqBW_)ycT9;@i=hUGCYyX7XE zQR)I6q{Q&<1-MxWmjtSRbB$+;t^I_X- zC-6%&0@ZO5o}EChc-sNIn;zT`%ZNKS`p2a~BbI7s9!{F+&ElVxT63>y-Tfk%zWm`+iI?xVb2PG?z ztts1SRsR(r791TZCCjFz>7GYJwfLiug8h)zm*E=PE1#HkCUHG0o<6PI5eHgyw>h7g~#Fi z>E!zgd9gC=u?`?{r*E>d2O3s>tv(5^U+6Eh3p;$b^~D(GGTEfkvBQ5r=1K!VtcS`suF4y>-~VAIQMa`S6|0v5 zP(mLdog6t#CFB&H@8Zzy{)iED8g0VWN%}qzNmrjm8F)3RQLnEY^6<_;_Yn(&hNxzyBCKfiQ zWR^>E-WUX=A0td1U*MpR6J{~`KE{t_@`d5loC3jH*wjbmq!QT%?W_!!_gqr_{@55I z;IoZ*xn%z40?Md4y*)j$NDmhlZKj`>p*#BxH;$&eEzYrsW0bc~rNILqo)G4_ru5{v z>3DB8%rYtGzPT4eA(4De;Xr9h%0TF+avjnqx;|7{bwLB!wbX2# zX-TQ1x@_J#waV8_Ouu;_Exla0$&)ouQ7RqZ}GZ@!12x46$4wA@ABj74_$XLc-LQdmfdSPZhrVL+-p@5K5_OmTj-EvVWYa*wLSsv`xO;@vaHvPyp24={32w zJ#&5-yPLxP;08Ya$9yB7M!i#2b$Z_w;=hmh_|7b9O`eaGMV^_bm4K_No(6@aAYFEr z;~`TkULQCwcCqpLmaPXb>8?Lefq_3Enlwu)U_$3 zrZmLMZnQG|Zip}=N^E3almexVjR-!n!Y~+O%bJe82z80GaX3nUB%7WhZfR35eOIe9 zS)CwZyWhU0JijXHs`_3CHC?hjPVE0%Pci8-%S@SyBb?}CEef37txUe5;rch>lc+UaY9aTK38br=ZnmrC z0bU5JMkMr645nIvVNC>(t zo2}@$>C4PNTDr%5Cz44n-GsU9$lJGQ$8Kp%7$;X$X~s8AkZ`I{({}GTxp=dvw9<0b z%9aa<2gTHrl;hy2ujPE>ufKlAiFb(u@|sY2@*CPTHnz671pJTW6cw)nHTgHKN_U8W z2LB)h(B7-|a{zMj-bRScB{vhiAIKS{gHrtYwGB3#F; zCYiLANf{~NV{06pJ};t%z9-K)Z-YAcYJOrms5Ow!TW88J)%iG@@&tRiq#LCy_NSP$ z=dOl7iv%Gj-Hb`aGWShQH#4j-!Q01%a??vMSzeVUmfC9YQ;VYCfAy)F8Sp4Vg|v)Y0;wMloGE5T+`{Ax&4}5`mTbCx>eg zlW7B?x;Rqb@${--0JhFW7*9FhB79q!q@Q5RWPs9f+qXc}eDt?zPRDPwLi(eL&`R@- z;Y18uIYWF|I5$OvMi+Xe@cDNbJQNd^W6!;|O)#-u++Ec@>XwYn4!Us0C)>7zv4?G` zv8h~81&xv`JGi&2`LGKr4VyQ{b@KV0W;xs{B5Px@E2pCCg)gE;N{G_3u%(;$^Vl+R z*O0bD>dW@_%X`BmZ`+pztO(R7D9WUre*fpOp0js62cB_C0ozYn?)$ZW?wlGacL_gp zkpxL^<{I=eu0TVHd5U7uoz(@6h=XHm`xw5u@&?IYtkJW<7W25z3uT;etPR}{iWX6Y z77SY>h>VXa84Sl)B!;5!*RoR2b}ThuMzdz~I2Thl4Oq6X~5NJ;97y z`_>Gl&nR^#{aN;~E>{;~bh92G$yK`W6#Ckxb-xA6i0MRC&itO0jxM$jeVP`&S>sWj zm#W}-YB0BcKNj}+lTbqa`%B1}2}sZEK`FB=5Vu=eTOZ8(&~-w>2p?YJ{*=c5QP5yk zziV1MHy&iM8FtxD-C085J@LH72O0s%wNYqG91&!tIz-8tng&Kb1=kxRE;_2}MNRDDx>ZJsVJ!^MwiAq2ix($?!&Y*X znOjD(Kb{#zow{jqmKSuSs^ps0_*q$g%Y*AoyQ_M$V=;OT&62}ExpDV)OV*~gO9S3a z2Mp>MtR`)@43{)IAK|Sz%WSP(op(XXpEeyBn$^}KUjxFt0Ec#hd1%oL{AmHk53XHx$#7!?AJSY z8hR;gV{c7LJ!PqFVDn^Q!=I@~IqFW9a%%^VN|y?;w_}2Xw>u{)Z?9s8YcYDQ6XJoC z#pxf=@P^HNHQuIS>rQ~K=(wTl6Cvk@PM>+F^*=v**=yqRudBJM3YqO22B zGcqUXgI3ut8<=gi-8HKfdp>%bdGuItxOelKJ7!D+ycjONprF)DoAk%Lg(+-vIjul@ z^vB0z(cT9KI0uJNw5^aw97%4J4JSGd=;b*JqsahtX$WVF>kJ&@H$X=$bSJ+6v4I{T z6I-NH<1rsU-xTbB?3s&|1sVnO_T8;t+df3nOI3ET>I#r@B-wa(540Z~{i)QdOLc=y zNPV7tZr#>EH}rPxwC$*<_y0rKdk13qhVR3Vw$hRck&;kR_8ukKE6R>SR7gTrXiyT0 zWEY_!D=AV|vMHg63L(4fz24)Z`h0)y_Z|NFjK|}?uW^p!IL`C5H2&c4UprvDGfmkA zE^4kg=YDR~zSjGVcN=z=Pn(M4Zv?#3;N@ZHL}eHuyMTpPM2aH$4NP5UJ zEu}LzSqAnM{EHRXM{;0`V>8b?RRtgI|Torydo>xmzQ7Hmiu=Als|{WhJQo&e}k z(J2z2k6ADTacXRAtcVa^d#BV)GDO_*ED5KPm@o*OT8Cu08K^!VPtw9Sc`M`t5M}5A zDTejw0OVkSX3lt;s@drwb#nZWW9y}wEJaD^z89%!X**EV1q2}qCXvSP@->z|#tT4k#lhf%CmSW7;O4;4tD0S~!YPXQra4+#Q>$vrvt zMhK3h0KLQy>(6fsIYuarSjmHO3e1#!i%D=3K=80NF{I`Y@O>b^g<85zlYc~nL1eP4 zwn(IqH6u&{DwVE-VYsL67F2LV z(u(I}O6gH$-+-x)D7?9j{b~L2<1Joh!>)hp)q?d}+67RaU~Yd=N6GuAfh~S1GgaH-7y_pU(MGosPnZwFud6Xzd}^%E0;B@ymo;}vgQ88BELn06bAj(m zIFk}OT})ECGP>{Yu7+Y)AD|Yv5YSI25F&!mg3=)aCYj6EZMx|NegrezES_6bs_E*| zqi99t#Q+9xw1go)ikvoc({c1gAg53rqmgE=3VEw6 z@USkVgm~JVs;a;W9A{F20vel!Su7~kw3xlb^zB>$h##Qo1T;owas?L&%_QEwd$*&i zcoPmz#>|)XPyOi4>}eF2nBQDViCE(9vOn-T;-+Y$#z;AWMk&_fm6l@QPXRzRD9UMI zfLh-4<(Pu-$r@{Pm6KN%z@=~bYLj~s+BS$>BEf05{``5jO%%FkCA=j{;EyC5h5=8k z%JrCA;vs=PF@b8R-yt(Gt9aQ0prKevX$t0Et;TxeA;zG}$^ZSsJNe8j6fm5UNDeVH z7W1CNQ2E)~*Qbux@%0ZmUSZA;^7tf}NE|G027Ys*cDJ(fb#0LZ$Z7R8-4CXU7swoL zzz0kOD+%R|J%PH5w)ZcszXs#BGcq5fEd}&VMK63d^6fk-?)yM+u~rxY>T@WqWmhN6 ztR+nDQq)rYz+ab>cip3YUjoN$H7fhp)6vP|IEp$CvSH??(2I5BzDY^kxTA#92YTLu zTLF&MZ@z+!9244`14ro;&%sF0(DPi9o*lo6DQH7Ls;e+BmHai#UGnwcybti5)wfsc zrKP2D7i57}ieBiYhf^UtC|B%y045K$x*#yF!cG9FlEna`s?@)Rld^HK76v3$BP}Zd z5*;dP$Bx0S1$A{HC=V1m`>h1Dqp$zC5)fijES`W(V!gJ%|U3g(Q z*K2TE{9q{I%-#Vy^!@iHF(%D#@^HYmO1hlG$Xk?+p?KLe-;)|VyDz>CHtSlwm5>-a zftz7WP%5_uuR|m<0N^vv{a6H`j*d(BW2d*RIm`|XjT!iU!)%w0K$YQ*EbO@!F2H){ z;?s)*9Kp*h#`MZ)kko*5(=ak@cB*Nv8dLXBL`zgeC=4Z1EY?FlY?J=4GHjaI1q6Zs zj`zd%*iDby0&m7tRzYU*3vn>1H^kgEIdzhmP=*=WnYr6Lgasfx@s%Ea~+ktclH@e4Yrk%nMSRW z+R%Ri%5T9Zr%@qinB^3Jx;k6TEe8!358VRB4-@7e{Eewa$m`iOm5{y6SQeE1U&jEY zXPB^u;!ULoGG4y_JyGbVqYeSWL@WZnhsELz%*zdRR z-{E~x-M$pO)MoQ_a&mGg?0So{3edg-fKq{#kSQwrVb3@XU-7_S#laJk5^J0uh5<{C zOz`V)s;BCbnPHzTa9ALH-ogwxeg3C$`|f2*T!<8)Cur&FIxT6Qs+*WL0j#(X3I~j} z+Jqr?n;n0Ddut^`%J)`o;!=PN1<$k-HrK2y=mw5hptOfOte^~DvvH!>6cWY?c-}Jb z?wz0FH%QM6Z@?HWcZ-!VfA++LYvbadEMY3_@J8}E7bl(=Wc=Rqm6Yy+dtkGD9aIck ziMRsupk?;&r{G+0={@&^W`U8JKpEsmQ}h=*m&t7u0GNK{;r`15yyuHq06|9*3w1qz}Mc^CJO@#|IGopKQVfklKnqE^bj9f`4I!_0N~m}3I<`tMZBaO zDGww+0dxw5`vd4$jdL!79Jm7pq6!xhNYFdDf&Z-DE^j>2E^m<7FpWgcfDe-?ZLp{R zU310l=!Bwp8Jq~}U}F^mSU){EOjzCu^R)loK1>U5?^PBqxCf}b8H^H<8)1TV6to~U z|1G62HYM;1jtDlT{-xz+NTP@eVH$(P<%<74yO|Q6{YW%eBeE$8+T5fK0XQ`>K=Hp1 z$&nA`BkBcL_8iDi&g`TmCi;DV%l+>KOUVnytKhU@M)Ec^c))2RyBNFU?|Sn+z*6o$ zlI3JeB%;KHgIuCsu>v2i{`cG_JUDThWb_{>RHO$9)^GyTv`1eTb7n7GO19ue& ze!BXxx;pBdadw5o{?_u+%i@_G_ahA;2a=x8Yy}2=#v#jteJcL<#{vtm-p=yqdI2w1 z05`YB?Pz?0Zm(e|*0LtY%@viV$|@=!WR!MGIJs+$9qD8V=&42V#_Ky1zhz9ATKgaIT_led0_a2uFAe=jeC5WG^8QCh zhXoZE@`<#quG8bGFDs#>y;*3IYTwuRy#=$~GcnKg<#(I_ZV>Ppblm!%On_wZ^700w znnTy{*fDOj|4C05w{7u++CmR79Ui2R1OrrDu=c6Xf#6oPw6~8oWZhv`eNG8R3zeSK zS7tao0fl%r^xCM=omskkxdEjodyi4ptA}Wrgl{mJ0R7929S_D`&_|Rx++fIBdbiP| zQY$KNV|$T5i4-{Cz}A;?5J_e@Pg0im1a#Jh3#k!bvOZ7YU)V-fCw0zg-m z)p>BYqUV|^Xw~#A{icr(>%nU@-`+T`y@dDiXA+eGz>t(SO7YpmUDgrhp$40rR1Db* zU>Ds)36k5w*@2f9mC9gOtG_R~Hw$teJZsV~^w&T*wPfi^I`7XG3JNQ5Ap}l+yoAZ~ zYvkpNolY*etAL=&51ka&z^tF|BSLUac@{vg5?G80=n@cnyY}SZqOabM)WBF|%<2va zHT7;`T&)%YtGZEV$Z)tet-?WjVf|I8Hq+wJ3xrjR)K|GcVyFT;ilFfPkNl--QBv>m zkBNn)^O{%$QJagpF+e^s3JJl4%;jHtyhwOkUS>vZ4zYp|!*{D>IP`ZK$%djN9z}wV zDa+fDv7og}ZG$l~KynSu(->|R!l@n1B3G3|~HyUaRx3*t?JI6RJWH$nIx#BCr{ ztM6n=i>%K%1q&b0-5C8*hh$; zh@&DMxo~6jr^tQnfAMilz50>rRGD3{)9?LEAyyP3+5mJ6jRr039x;c&Aa)&&Spq3a zM*17KDIo++q`wlpSDMp6i__2f`?)^g(*gd)huA)ld>T=9=eJCL5ZRsHCqjw@bViutCN!?a-k^>lqjn5%$uHU*Lki z^(RmlA~Qvh$BU8MmQ#p|p|r`zK;T0BF9x(l!d2Oc&7%%Kwh;XW3cshJ3vdEpwF-PP zQQxRUiUdIXZj0R$?oNOz20azBp6%Txacp^IZSCmGH!V3TZ?7yR5^souWN|LVgfSZz zDHVw^A%R040o~M=gfF8osOvUuGM~wWZdgQAG!@Cat{bzodn8UOQR)=$F*Ym^Br-yh zFX9<$O#u#?_BX%HtEs^|C;f>r}X;23~I#IaRXxJdhDt0IGqu7Ao+gBpU(5pqp;RfC%g-B;n*t!9x<~kAOl1`99YBBuFTN zY650EkIV~R2aN+Ik=vs&5r1Ru*bownRe7XC2)cu{M3RPXz&E@A&pgNF|J|5f4v{?R zTSY;*;?m^=$;u<_0+&%%DB`Fr2ls_@rzkoc`pi2k6@%~3K_TTb+N+3|As*~r))dhj z!4{K_R>0(X2Jk331HTKgG&>Q6VwUkUM4`=Hudf5dBFQ=GX^Bb++Qnn9?H?dQqL^WR zF_4qnLe}~uD#eB%CNhWM91ew`!JCaQQte0Hr-#@tk@;p*5cF5bV{n14lUx{0WJD}5 zQG7kB7GhwzYiFzD6yG6tGt#3G2vZ>e0|I?Kb17ciYicwOMI0{Pr%+Bz~&=cQ$ z$+Nl63IctynSfveMD5H;6ooL4Rv<#*e9RcjpwRtn9p}*#0SXs8M+p0H8SDcmD>W<3 z1k4h_e{qT@P;z1PDK>BeQAy3HGN(8itrdWEE(Uwt% z5A-1g9gwRBiZL)am=E!O)$_mcQnVoK&D*w$unkVH*^j|T|4A5{k1RyBtb@LYmjjzg zwg4460O@y#j}ekFGXGV0B3qQDRRjc!q-qD^Ocbzm`3WyxI9clhJY#X8*uoJ#J;RYi zo4Yc&Impd+0zVP@o4%Haqv;*EBAo0cOP8hs{(vvL4j};IumABOMe-q9xO9>!JU3Is zaX<8b%M|+Pkl?VDB3?~9`(2DE)DeTArr-xG-M^f}*#faTt6~}2cLBeGgfJ7$lSqXJ z%=CXB;vgTo);~Pl2GAJymIz%*K=0$j_xHIKo)U9G&+axk%%%Ur8#r)3;x!GhLLcr+ zTK>hC0*;Wk<277}&DkIy$G)VRmD2u8-r|%HyemA{zzx8Q%fq}A>R|z?&HI-H#ru%G zutNRUHGr<90T-$VZUJ;f+${LtBb~6Yr>u7thHtq=F8Sa^7@Ai=!_YklS8=_LP92*H z2diAw>j(3Dd}IFM3h;5;3l|E@=K(Jf;UMzmo1gfQW#eR(i}$852Gbs=YwY2i|OV5uPM0w0R9&^1>Vk z!j*)%pjb$?!vetQNwB;>ypDnoIW=)BfTo+nW1#%b4?(&ndhbA+d*k_O1)ZP)h?;Ur}_~O7)M7TFzE}nrCrJO^9 zzpZJhsVB;Uc3ni_h~j;saYE~S>|gwq*w`JQ^GH^WtO?a0dMISYd8VM9L>J4Hw{pZQ zg(vQ`5%pF@MeQTX#rj;En!DP$Z~T}je*Np$FDk(Dq^;i#zDM3!;^6|PWnh*QQ%Wf_t%5rHmZ2E9t6U#R32n(X;2klAFPWX#Qa9?myA)%q6I_8tE(%3D4pEGxjVKAQ? zIpV~*jQv)HPb>}Q-U6_eI>%rTNBtBcT$FaisYV3DDL;du+uXn`dY&{M!QA*|Fy-+$ z4nU}UKewy?e*e|G+zFn{1o$x>rv?T73%^xxoyENb)xO4G$I9;Ri8sm?L$-7y@R+PG zV23q0dZP_%rI0{80l`O(w|5?9zk3U zEx9*RgmZ}I5oD~|kJNNxW~_-|?npw@Kd600Cs1-Qb71BD;m=WIVBh)iG7fGUt^q=j zLaa?F@;hbznQRtdn@IDFPo=p{T|l8yQre1r#yKj8@G}u#!wM3_5+@HEmybvZ0cV~; zQvxE8!@2Hm0H7z3yo1R!!9$Yd9bo`bEH$65N&;L+AYf3V=f{2@!`&v$G$+tU4_s&2 zmZk{djdcTL{gD(A)E;4Jq@-&wY7cx!aRb`$WxeP_ii|Ms7W#I=HmZL7#%hv9ZURn; z&VAu974UUzCs>KMGo3dzjaUGBF?&uzfkN6R6hR;rgctZ4ZXnwvAjfMj$&+Vs1EoTP zuQ^HD8p_{@%78~evLfO-n&{B#gTRe|BlB1EN=QV+7Wm$q4!sE}IVRwPND~?u6AF*j z7oNR_ukyg}5&J-a|9ya$CLA4bb>Ly+MPnrg=k*1Qy@CkmE*??y*>S=#z%aHTy+AxrIKf+s`1U;TXkGCya)%BD;0nNN zkICtgf)EfP2XTQAKCZ`?OpnwVllT@FKo=6l|0Bhok!zJCMUiV*q!o0uur4!%8~N zbqSS7pkSd-eT=4RavLK(B;$jyV{jhmo;>Ph2aEv-@?4UX6*(u0rv>;mWHqzE8Qcc< zrdy08hXUij4Llaz?jznH;Rqk*j#q}ko0&ZWzo1~$mQDck2WvTKLI5s z(G)J&G_FDIJuYyEHwsN4t-nIV2Cv)o9`s-<2e$V0A@Tl`yyGv4Hg~Dzz`rhw2DP;B z1!~(;My5P-gy-1ywu*|2^CPJN-+usbBd2yEJ-qcJcq#1eKJD$Ko*LaoDQWe-g(fh| z+u!ovR2al1nX4AmeQ(Z>LX~A~3!7|?u*RpOt!=o>&ZFr7AL1n9N&%v6U}D0XA(y@j zf5wYkHYPR}4DcG1-bcgQ4Sb+qzy1I&Yf=x)PorzCU1jZNdwctW#zv`H9pr8(_Cb|7 z?ZDQdK7edX_aj|35*Y&l(cDE72Ha)QZKNx8qanA)*!Q-GH3FX@Nf^kw0`y>p?Wulc zYf749;vyvd0J$e&9kRPnxCNACn{EBDz+kCi`FiWCtlP?@LvE(6DX%Tz`*91I)q>TG=rvnM$ zEf$6QO(BLF<>p&@3}Fq%`kIRC>qU)BrTX*+5RTm2H~j-Q!pPF!qLAdQfG_JXej~admo~O3io;ULc%ny~ z6#u`p-Xp8%q7jlB@IJ=@ZtaWwiW)$2S71Z1EX{3g1vtXcLL!(#=GsnBuaB-9Hrq1W zAiIB+nMq>ko>W+EUf8VWE~`8XkQKL(yCe7^9RjfzwDk0$&Gm=aCeZ)jU}D`DydeNY z8WqB2na~iF!ch}62GyzBLk%3Y^-H4iKoE6D9?!IKc=@ zNF!yW@PiGuq@6MXas|+;h)GU>m`(i6PyhuY7u(0PP}3|~55}cB444|#Nv8K!e*sh-GTMaL596RKk+crr z^58a82O7u*hTsufy1UB|IbMFuvJ&>Zcp>>yR7Y^_%rP-OP7rGp8xp`9KDiYT3kPz0 zEJJfkOFn?~PL$$;KUumB-Uo`FtGG7R@md^6a3}IraypnLoYpzLHg84;Y9o=pjt+Tb zS%g|ZI!wY!qy~%sv|kV9IdR3|aa$dbkN0~j0W;o2D1;v22f-i$UP`^+gO^Pdq$~E@ za=p;x1R>|pYneh)aU5VhM{zZ8*9|%Y6B&@r{DTX6YW~0Bf`CtNvD~;=L=6Z05>Z4= z`vDw}K{Y z9fl@|SYjgeA>j@x<$xc73`c=b5R9J|2s+Eddr|v|^hyf}#gg}SbF*#$YLSx@TMTwW z{J_+5ppDDWiG$D{D8TCZP+sZU-%N!}j~pbpiB2!+@H>-@1rAv#{|01V2KKNdQj{Mj zfDMO$I{Xl=kln3qF03cMUUji?6W48D-@Cw3(OQ)k#KR3c*_n#9$9d>qQ-{^4yQVU5TzTj>% zgDl$wf5dx8E%2G-bz8TdopAxk0!Huk36gHtBRcel)UhrAza0Sm7yydPetyBVXfYY^ ziDdZ!xutjaY{o4&1x`ewO(d2>NUhnFYyo9AenZAEj(}6tW%$Vmac`s^3oyE+I6_~9 zlCUUwW_$tcs*Dt*D_7wTk#vK@D0p3jRu@D>0n>{_;~*j&RLlxs(F#7O*+h{dmy0s3|mJJctkS75@8rfuIdOwi?lQ%%XFo586!^VwX zpl)#Dh>nD~7%-id-w%-3!Ilj3a9+A^-#(le(uf3T2;ub*(wkUXTG|V&4{*$fU{ZGh ztuaN^_{THgI+k9dTFeO;+IgtmudZh>;ewds`SbY*!N|FUQ`PN2>F+D2ffxsN6NJPt zG91>#g$+rl7GNlIHA_3Ml~I8c117$FVOoZ*0@OMe9Z3lii#Jq@13 z4VC(7%WYbJ<0``1HB2J=0m5NV5)Cp?2L(k!8JrH8%78p38Vd@v$&)n4Jn?QBBu^@E zE~|Gf$x*_NoQBW}uL|q88t{A|?h`(@7U%B(_%w{W5EOd^CttD(UcSW|EFK7$KCf1w zAU|*|Q~f69yo?4-MRdwl@F)t;qfMd< zzJh};z}dpQh>We6v~iLsO?979oL;B939-j1up$d$MQI@dDDqY$SU^?mt|L!w zS5*N~q)DrLRQSEq7+FKoe64am{&!k0To5^#_aB)2p-?+5ULzE(f> zd|kGDIs91}ft~^{A}jF0lMpYl5@HMBPc#AtvDei^V1#(b-l7=GeC%rD9_lAT_aA64! zVhMz+!Lx&VHFEg?=Y>1L9mw5BhweAnI|Z;5q<0Gf;5hu2;cNTS$rXredy;z2_;_cn zB55#QdgAw+K6*(P0pJ%8kj6-$Ip;aKTE3uUey0rL6{M90~ z88-dJIjw?#P3tFXGpp@DFYhEIWm*05DIksBgd#6^vdnMfx(&KegldA_VuNW(%Z|aGwsv z`_V$VS7_q=<052txwDgdu*3q}?m+|&!kTOuqBRYO6o@)NH#MdzRg(tBZqr^v1w8S< z=%_jRpzNEO!^!k6`HCLnjabz}aKon(O%cK4k`+bEvKDPYevT~D^?}B$Ba8^DuIwN^ zW>bMtbwXABL1aY?!e^Q0ChKy?Avc25O;AkR=MHjWm~fd~_Zd?3PSoWvr+?lH5+mn5 z08(n@y`ZTk#nxaQmp!KCEkg7c3Nj+n{5;Mm*4J293tRP&l-fhm--3MMBytGSU8z%H zmk}It7^I%CiACs;gk7eervWtALob(v@Imh;1A*|DETNB7C=kIHnBc|XJ22Xznxf8f z&m#;@upc-A?}>cRIA2cIz-a8$^^lOw$lRfmzXh|ig?ZO!(lZ9X2^~6#J(2tS&Wq3D zG52Ut8!`4(fZiA%j4%6$_zN&PeG$6%z(ON%WW?oq4@*Ye0RqXTkdX3fPlblkBP1_| zz4r6o!#b!fWc6h~BE4OjaVtPlvm?5clUH)Ubw*0O226Q0nvkLTzm#|vu>XL%ZMe5> zqX7Fu=2v=8JOh;ufnInm4S0L7BbgU|F+!Pj98R&exD#V((Aou(L;{9`Eum54Vx@Hi zqDT@gPzj`YaAahpuYb@Pl4~P-Lh^sIJy1nD9{HMdRSauK5~A?{|L|x|%^8=HuR-l9KC)wo7A+Z;7}~3Lv-IsU;^2 zf5awn=SCR+oavXI>ug!DsMYAggsDs)z{Q|i9j`tDXH4$;>FT%#@Mc8S4)?(@FICih z!nSAGeQ8I3Mpc8r6mOvr1;6wpBc;#F-kdAfmU^1< z+q%HW^W)QmCvt0qZbY(v@*VLuim znVR%T44l+5Ui(QD+5^w>!m6GYHdT_*iU(~HfY^Bg1w?X;JeOhN;L|U_;1A3Zy$Z=Yi7^6z7K=`w-U@s~t%e_6NvD2_hldAHW_iFUzWzIG>DH~= zkLZc`SUmE4l}~^ftSfneR2Z(=LLe&iye6I()~FpIA{LAa!uTE&ko12)+}BUmXvrne z`vbuA_>tnHD#Xg3>BeIqXFSl{lRotBvlAA7o(57<@-*upULd{|lcx|Hql8$vt+n+a z9DWzNa-fa_^{e<4+rfn~hAW7`0hR%X>8DQ4!C>&LB>=mKVfCADf>;WVPgb$-73>5b zs$hU=Bo|-Rx{PGL8e{~a7yrSYLs!j{f|7`}`oHn8m{?CBJ@AtGmoyFtEN&`CBW zu;K}QsE^@zu4&>u=@9QXaC7*N&Hh{8$n|d)&vcWwJbr^M4({(c3>rws)G3eN?~YFI zJErmm97luv)&=Y|_f=TuD)6K#bmOO|N!q>NQv!l(lMgp_u6=l_SMtmT@hdbtyuAx( zX!XoyXG?@1ORCD;Zd*6j^V%?E=;Hm#p>LJ87famkKZuX-J|`;fm6nN?YVXtk;b95 zL*u6nb6$2i&h9nRx-igL`sStHy3`eigGlF0Xkd`4$98MTN+tX`?f1lvetxh5f23sx zYw7-h!_=8h!6QQ+Z4-CopRPHyVt`8R-kb6l^~;^dbw{XQ-P*NjyOa>oS%J(&mkf>1 z54;!gp-&+qoqS|tN11)r$*kBeMH@&%r^Ls{S5i{qNR!KQ{o$ zf+^Q<;EcHh4u)ArM+Zj!VYdceb;UryZNC#zhcg!KOS3LCIlJ7r@OdKlS#JHsXX?9m zX8E4{^-a&yP~=^CN9B_?&Fk%GMrhq&7~lB4=WXKT3C4`%Ww*lveXR?3+Yg0%D{pW- zbZ;u-px(&4BWI6gS0ofTl?A7$bZj^JWI1v-#=mm-MwXIN^wbkYE4`d`xsu}JJDN99 z4s=Y5WZQ7)s4;U)9HqT`CiBOU$5PyHMrDp2yn6CZ)~jAerc1^}Yac&P*s}O>Wobx` z*SfG}z6~|Y>^vIlkNVT{=lPB?8SwY6?u7w4Mgl&=`Q8G6Ryg8A8c%%vAlV_ zOB{_~N*=U6V9Qo|aDZ8yu`a`6f2G)|Q-~El6_$@pIG5V)J?2-?SQI&$lo0(Wc!pst zeROcgrtLnvVG;oF79yTI!tBF&oJ=)pD+xe!Kx&MRUf)61ed8UoW)9BINEZGnmzbnj zPLI)1^}OpCR&Ek@_f=gwwLIa;LmZQcF)Q+xQF=%caUz5!uNjooXOTQeWq%Osi? z+2=C!zEja@4a;(~ttNvlU8d&lJ&NOg>zE>+T8!F`($%V6r3L~;8yV+In@x& zlp~&eZrYxHOS)0}#{mufH#+%64MEx0lUo}Ha=)z(^?mJP&boK-0N3iTds|Q2FUpoU z#v@eyv$;wmr#r`Ifw`P zhE@EF*W6oivgC`kr=7Kee!~xKfW{O9c{mC^tq7ryHD!R#BO-|Y1y4RaqhhtRqC}K2B zMI~sG+?_OYdKun6G-GDnPs>-7k8Yo2AB}b8hl}!`!UkQFE1DLv;|p&Pn;!2tt5pAG z@kiU{71x9uO`BNeWZ4u>f`@r{iK-hm3nZ?o#5aP^*6^t zMV>PVMk+0SX4`X8K=4y$tMvK}rjitPUn?KC_w~I}PO1TyhQhN}F*B>~Wgf^+PuxW@ zy)O7ju63zWYl*Te3**gohd!fX3;mT6n79TQ&`QA!fCQ-yRaRF1dmREDG@?QyLS`~u zB|~c^r*_d^raZJ}KGpD6Yd~yo`%;bzXWyoVr?IYI5gq?fwH#enR2AEM-%5U?rf&N^ za{QbAi^FV-sLDh{_|8o-F(uTssr4WIy~OyT{`YG>W=xl^J=$b!|6A^i5+BTrmd{WG zuU1f??XStTC*LOTP)ggZr4`64z{k!jHKsZ<6l=5HlW+BEhoaS%F;2&3JgA?WuP)ye z=uT1XwqUPM`F@cmZ)o!kt1m8L`X6e2WVc;PPhYm!<@0t?Q9ofU{=MpWlM%x%>v1$aeM&3x)K%KpT079anX3P)|!F zrf6m-wtbCtb*$p&S(2A7^V90r%Pv96C9O9ZRLyPL(gw`ecZVt4vV6;q<>VJi%foVN zX7jD_FUWTK;D5LLmF>hLb=JP2=HJY;t*t?U7w1NKBzmt-ZIa>KV}05)J^fnC%VhsY zG1E~30(>FM&_9*x+|Z~NypFfJlb?mh&y(hQ)Gx$;RG$Bp5o1byUsBy{c5p*vw$rVC z@6F#<-fTblZO@@5;j@xUH7c5?9`D}B-ISTV_QoOanB8~(6 z=a>+KLOgN1RVtCJ!MH>TSp-8B8cKI}{aoh@d;7V=ea_}#>1)FE^_6mcRo3!268*=#B7*s+=sb6HO!-4B;^&VoSh&6$7hEG_qIpTw0+RonD z)4!`*gW|<*>J6>A6By|4YARpLH4^8}Y;fpa$6U{j$BQNG9q#7lmf;SNJ^l=S9#Sz> z79NBfmuW|J5x56Yhwh+1d%SXfj}?0=;&l5b_&-_KYj-u=DOpk2$DL$N#d*@QveRqs z=+ltp}i0YRw zUf8#^7{{iB*u@NVr7dCq@J-&&Dk^6ZMG8evN*~D~sy79M%TcpSax$8n|O4cyc zP939CY;b;00X^J7&)yT3mI(-aw%Y-buBuuO3>DPh@aPue`wp{_H}y3Yv=9(@xkAED zGhTw5`|bwu3{Inv=P%!Vskl3|(EHW(M~>ls!KDjx#^` zXHG8t!n>-Aim&pC*mAD_6z8s8rJ0CHMVB*uALX! z0^2-`f8tl0$hWi!HKHG`cCz1 zZDrowj}BgxL1Sg2Q%I4?SE~bkvq$}jmM|_z+wB@3*4#u=A^#tNcrWaz$HU> z2vE=tF){r$YbJalwMScEZe=g8bu!+SpWz8*@{kIH1u+w8|Qaq@zz%bNNXxSaPS zGRC$e*7?-?HDfIR`NYITM4%e$L689EsL%Dttys5_Bn(<-81U2QIs}|+bsj*YrpAD9 z#@3MJm;(``eP8d5hjd;)?Ai3{nnUJb(2Mlamqsu-C$?W^+sSs^AeC>^!}GIL%|}gG ze+bKb6y}mUE$%At@_FCP#;M+2C+eaV;yawGvC5-Hr zp)Ry*t{;*$Dqz?A@Il754`aEnBumExno<`UU6~u*OU_x#OuU}2R@cx$&IDl6NPx}3 zmwQz=fxml&s@n8Ha5pE=Fdb~*c?mjIx+y5&eSfp{TmMYXp}pp3=a&lot;Jsx9sK-M(!RH&3uQ`{-6?SdM1c%e#unibZ3m z`wp6{kKugfI<^;DY~c4XA1A+|Ko4@N_amRbc0;9~(*@Pn|8?b|aF|I2P+hWqF z;OP-+Ztf(RzBdrB=<3f(_T(bxhLpi}p+B^Ox_YH;=DD*!`wZ00s;jCVVBI67+`>Vc zCM+c7bbN}-R?cVV%b@4+gSPzXI}H;vGg*E8a81^I;FZzx$qQ8%cS1u+MLV*tiVI%~ zzjnWeo*sxTrTNz{RiGgZfc`P7n-qSM#@c{VHK3>lcS=ye$;rvg#thV=q8lku9*`OZ zfad7D$#Mx~1EHINc{t5X3L;>EVkHdAyQ@eDg))$l#bYD@&M$ZeYAr1-Fqnm?A=BzC ztw3;Bb|W84QDC8zBSS8rCua*p_=E+;GHN?;uEb+NnQ;X~B4-)+bS3B_Fzfa)M3fgX zMfXWU0zZ&b5EvtBBZ)~#yu!lE`7Bt?fB6YxPBC!mgD7)AJrq6O&lDO>Fnoi4W8XnL zpeq3x>Q1!8egL6iXLt8} z$#g}SmWGX8GD?>6aYxWWefKcZaG4bk^!fNT@H*>aym8)GB_)~Vi+iuP(-H0N@bEB2 zK~0SvI|C3J-d`5z6a^5)ai}J7R(`a$UiCWEexnfuXva~J3|wg$;$FPo(2+h&uiY*w zc@J`LXePZN;X~_Ubl$lHHisXpedwaLwl=|PfoL)Hl|yrN096wGqoYMdMawP$h{WZ1 zi(;dgVv2q(O;;H6|Dqg`i%LXOJGmgr&rxTRI|tn>4IdP*&c- z#yrVZQp*N%jn?2gMcPCrhNYVxeKBhSn+SCfI zNFkT*h`Ra0ZK1Oz{kR(&jZR*-B64zo61(qW93vk&Pl>vh(Nu#>u1Cp~#hY9d7E!+C zda#(9wn}_1zkzQAcHEjfH?xn3b82hRgL^Xy$ZkeNd^KodL1Szb?!+Ad+44$a!^Re;F*Sb|x=Sne zlZ)-mx)%aFK)DHGG&`}8=;J}T1f~gdi&x!l?JY#VQ zXCePm2BYYV#Bz%%%Huw0r$g=q0SeIr#A7%p!~_6k4}*ZUa+tfvw1lqwv4g#R;}^#R zL>7<%3s9D6Ok@Q zAhKsCd-fd)%goGdS$795fE*d$L93^zuKpM_d}mM3o-!6`_rw<*CrH(Sm2&7?$m;WT=G zba!`?CNcoG`CyS?TRa~t~2c3l|$)_{OwlRHSAZQZTxdA^jAX{e=oV zAsiAt6tnT0R~(1p259<`!|DbG5#S57z(^YQmEXw8kpe|g2w?~Io$9!mnI~vJJPXkv zz`eD0bS%FF#|_<^1-xh6TyL*BMYH;@?BI zn3|eGhH=-_SxxZ^tnOiSB#?M{>^8m-HFvxHWy$7vDXv*J4}Ngw8|AvMHrwm9u_X$h z(2H76%*Dm6Yz&+vZaM6Ea;FPH4wy9qSbRXfneEG za-E!aeKLzSxG5xXswRi$4V_He)mz&czT~{zaI8T`$AMnP+mWqw|L2;jDk?sU6`Sm< zJ0Ksx&PT$$Vp>m%{V!Y!oOc2rL!1R?Lt3vwX4TQPKLA|VIYb9mKT$#rmvA3?{ODvOr#D?B9zY_CkmcwpxBR9w zxiqOAVz{CU1dj~+dVkGCba*6ItPJt3yg z&&%6Zrwfzx4m7@9_HW1*5E(Q&-^ztv9^m_FbT%+IH+Rmlft-?!on47U|9RH430TE^v!#sCL7fn32Ik_VFmyV{j7AQiH zDGVUn9>Y`0cY<6tZ&t274C8%aE)$sWa|K};YU^aMTV&h_I=MXto2WE``qyztdRDAn z{SdS%#qiAqKg^3^6teOT4`)OvGuw_G2W|TmYgO_S&6*K-Qb8+Fy6HOp=MFjFW`uT_ z0tNdxY}Crj$H&Uet$MmC@Bc08evq6D#ho3g@1ub3B#o6I{aSpkr?%US>)4@&h6cDr9%zpi85Bi~gq2MClDBOc+V(fbOohUpM?lryEh~xGExOPMjU%$LC}DLvm1Q@aBNni`IirZB?^+a zI5P}g%R}w2vZiFo`NZmaGMN5 zp@Qb&+Bn4x0)m2QOmRu`7>EiOUON88s0JW!i|yXu-uoJxw}&A1&ISW#*m0bd>`%M{aRoENwx!h>9QBhw%)c=FDwo#Xn$=|ZJ zsYIXsS9A02U;2=s-8F*MVB+THe)v4G%~VsfdQ0&@4$g}#c70WxHQC3DSqmX6{CY=c zX9@C@!R+s_I`BZ3VOY?lrx{fyTGObLXQ04R>AdcUn!mQ!)6<2ZKk@0oUc%t6TemI` zIX(0MM-0_fRez%2r(v%9v3OM&0^G`RC^68Zfn)pj%g9Kw>Oy_}{73_cOEG?Mgrrj# z7DQ21mGoUAX%J$nHR1mNEqXNyJ*9~o}h_lDCW{tfT1;P=agJf#k0X&4GsQOMv-^|r!X%sYN#$FGZ|_t z-SBW{J_YCtqG~c1-6+fH!3$YkRaHMw2{^BBaoErMc0pT8>9fCGBK`2uqvz(AEO@#Sf>!v;EQbrRBoFY}`fvT~ zzkW46bxHxbc=dygEUW-6P}_z^n@{bZ6-fhsPD9i}mH=%!>3));Gk_+P`oPP2C-q7Z z;%*i}-B`}dr~#=RLy;|t_=v}VG@rE8MK(anRntZ2BGzq(^L!|JMITVql7k+}YLjqpdB@{ynx3x;g?6&i2jC zs&|EB)rr8YzIU{*T1!u$IcV~9u& zA&n<>I{z!hLKEsm&5*Sn2Qfg=)zu}Pc>zcb*3z$1LSl$(j94Px4TW|1n+xpj@27L2mkr=R{`ZepQAbrcJuKwWx z$T@#YicW&I9~tz(zz{F;IRo%0I>fYzDGx-E9w8`8YSQ1Q{J-)t#TO(P*}NG_4yorT zmx3gGqp}Y`AyVXuIs&5FN2YIzN(1}z{Qw}(_c}tNiHw%5(l+NNLTQXHGu&pq3ai!9 z%3hkMUgD>``IXBk8^eq0lZdQZ$Tlb%Y|HhPPp4=DH76S{V!<6V~j! zxw(97G&jobc3|P^p`8q!l5;$jMr=8P44n(|sj8UMOBI*-+=8%t1mawFv@LpveD3p) zi#+f+00vXj)At6=!t~l1y1<1Xflbv6^YS{B?jrjB{E@pvW83_8F`5mgEQp&PEDEHV z_VhHAD4(2=)=bb-KE(ITx^FhieMG4+;OFn~IcM@tb>n?q5(TR+>8lzXH#_5h|Lxb@ z%(=(6+!AHHg~c6y@2xRsiL)**TN+4Hq5f{Nj-%_S&(nA7Sd9fweWWlmGouW7KLB%} z=^77E!Wx^9pbSvtFa5V4^xrGP ztE%24^nBezJ#d4UhbI~mI#gq8;}kS?-%U~G@NdhVW2hV|z7;Bbo-)&|V|u(b ze45W)B!zYZm99|f&6ysZ#r@^uKgLy}o^-knygK2N9-_un8pGQ(&~HSq%UP*YH_^3^ z$*?_I%Z1hdrMt<;&ZL6ETzAu4g9}TmpZz-jXl`2PeoF-(zA1N4My*=Huj|Cg?dBD`oX?dBoFstaFU9=|W?(^A&eN>9(A_44dNwO{pW{i4!pz2EZh zY8Bg_ZiuASt~HdH++35>m1pxfIr`E4u+9GN!Hi+cyw+Uu z88eDCYwmy1i7$(F{lPRn z7N$C@Ebp**DX|9UDT}y^nJr72oMfE}R+WXMMMoZMch@hT{{1oQ&e|fm*YHg&m4x(;luG@70{SujxJc&cV(?&IgrF{WRiK1qn@wH`g7;yaYn7r%~$ zj1&YFba4Ov{r*!;H)1M54kAt_a}g2Pcp+H8++xm6y66{6SgEZZGQOdJZy&C>! zzn5Hfb=-K;(j)vW)Gh7S(y|kbdnxuTf&pD_ub-KJpuw?G;@waq;n$TQb@gM7Fk63x zV#CN5nUgvhV_WBPAF}1Pe;HMMC1m}!BvX0g|HapPz+>6I;o}b~rDQ}yWHp4WC{aj6 zXm~OTDXXGXlrlq+6;cSv3KP<2=r;;9XhI!o|H3Uf*Cd@pkwfe#vl6?=I_GF|A+z3>wJUX#AE*>zuRf zHhuZh=0wi)`YjcKOQVX5jP71+5jZ;J#k`akBDggNCC&c4)IkCf0QJX?iBirzcIukv zI;xx?ZE0_Q^S$2c{micqAs_GSM4*SG+tA+b)yHr7d=JhJbe4pQDN79F|AngI)jaWh>E7|nRwy+N2`xl zN-w-t3*FYJo!qDzz(beN7i@Q{c~(EPa!IGL;T7YawVcU&gjkI!VS3+Q{`%%qA87hv zQ}d`{r^BNgT+UPT!A#2{NBM;}AJc5jYNYIPSE438ONr4qjK)>lLu);fq&)q7d8cY` z@AaSVHgg^{WQw<`xe2wLg1Np2t$fM}&)v138jPph)_NDp_w`xSr?U>s0hrR1&K`ep%uI zP0mn#@W~bGl9?~r(=z88Q|(r$$JzIBa8P37O(Q}!uwQ?4_wFjgEnA02(+kvW)^v7K zY_AOs%9h^O9g?}pwQt`mU(OotQRkv$%Qep`gPtPg$k}D?ceV4Z_j8(%hr;f`S=LP_ zEA3Bj*~~o4BY3p)^WW2tuJ5U;5Y{i~c)j|hr{nRM9Nt$l75-O`uW_PutTDVQFL(zZ ziGS++I?ohme{w}OB!86iU4{WR1rKc>hgZCRK`N||kdOvlsJYrKde=3-S=xGJmrmT> zM8mi-`f|hC=SylLwn;cou0RY`L1?_>t~@F+$R4-Dm-4akQ-^$K*v|2Cy84T+Jk`3? zKfc_*M{jqUYGpk8$B%!f9a4neF#WxrBl`2_27J!ez4Gr^l;g^ddxRNNoG|pE=P{B~Vm zIcRFsc5L$z%^_tC))v`4pYT2)!hT`Y-^Jwjdvd$~HdIyw+O9{n5B{DV+l*u3ZG7D5S9m8n+? zOMv)G^SSTui=Ni7hDcxKF0E$FwXvctT{R!^X1e$mzjqrns{tW_cT+OA@7$JU+FcU7 z*PgYUX{BHKSoSC3lXiN~ouk#qC7zvX4eAlS|HYwK#_++7x|l$Ns3ny-NfON%SZ;OzZP81wWg{KKycdJ}fcj)$n#+F&>w{3&r3p%*8GHTm zj_O%>#|ZO{En|AS@@(ROhoIR7kLRIFA0FML?Y-g5|CBoldLKF29(w#*@}=_kEza{E zpI_QSw&eBu`PXx&ySJuwUVqko7pi90b4LUl4ymhmm3-L2Eylsip!5GyH@?$2ziwxJ zbo3)G*$Iy;Qa@-8Z!{g;`R%1E9U`ft{4~$EZyerq2jxogr$tqd)SSs%4${U}(0$UoF)}Y?tVVNS_|oYaje^s5P&C_4e}4 zZFI+y&3*P}2Q^yh2WG?^PMd3fZRdM>>#82e$KNW_hVG1I^0E?LX_27lC}wHNc=qUp z>dGA5vGo}NE=2;D+I-JK;%r+ZuQyjbMyikCtsY0iH=z9eS_(%fx(_%AY z%P(=skMHBMo4=kox2fR!==2^D!-Avk)}{F4H#f?)uXE>>rKF$#d~xilVRGDh_D{TW zyrGY>BdLXcJGKa~3f(ioTz<5T_fMA)={dQecJ{t#_HB)+Yggm&KGAA%w(a}IAi|Qv8idt^ZXf_JH+LF3plH-pRnGu& z0kaz&7c~8u^&jsi@twB5U|#If5`!ODTH#-EmtL);dLKWRJ@cyfaZ_?_)C%_SQ(x!H z1^F&}j1-s6wDmtlCyHUAnrZc~`S$D^^b|BNyzKqS~f zvpZU#@pCPS139}Y9VD1X%+Z?>EjFxp-UCsQC7?Hba%FRTU{zV@w z>$sk;@V~4>7ka4Ny>-+2-V2``dLyN!LR2cAM4p&$8ywp-we4_I%CVPElb^hYRx|eS zX!rwH;H}R>+PpXpua&T^(osC(uN~(4SESpqEzHf$R3}xoCTV;f@YGL*RhN#|@NR5N zKFq6S31I@<`(D7yXwR`2Ph#`~;2&JT#6S>1PBnidn&tKXWSbubz)Y78q&53SY<$_@ zxa9L6<7cO@PrUbb+Eo2Qh3R?nk3iA)FMIsCqtBi9jBquQQ+oI!*3A~H^pKirQI@fp zu$oi&#>1e!_+}0+Kgz9SY-(}mu)Es&N9&x2{*1^B{%?v-yP-Kb_D$E%LfHorEBzHmKc$E5dpWVt6KLkbQq#oXm@jR*;Z?Q%FS&9b3zB}~(hSaKK zDghx5fA$7Ij2S3suGGb_1+Hf2YS!4XJW@pvqOd-oE?;G*pk%sgZTB*1?TlDALx~kMU}4bO zP}&fcn-rme`rjy#CS2GZ)VWTpkDwdRdj7W&Pyb8P%X~b` zvv^`oe^l`Ev!F;LvnNlUsw_RHD=7WCTixL@$AtXJz>KN$Yho4i4nfz>(U4h_{ik4K z_Ku_D_SdbVt%44n`cDgRDW$5!&Ro4Kg|(`B>}vDvkIfpV&CkXZM6A7(x+^EQ;56-= zlkx0s?7pDQ$J_-LJn+iPJ+RB({VL+K03p12%a-SWLP?`K$i%P=gU+{$lO{lFNHBmC}?bvxK)q zrKA6PgBF)!#KOfx%KW+gVax#r$^;xfvZfzT;M&FI^4w&nt_>o5EqEq4c)kZ>OWq3F zQ_VobKy{NWp@oYpvhnRXd}4t{-Feo?=nY1T)(KxykB z-Br)HKA;Cj!7_*fNYg6LOw>q+m1`GnRSdVfc(L7J0Lh#;{079#@3!IRh?xyA^&UMU zXhL6|7lLN5bvE$~)= zwU->O-aW(+o0e7sK6!XzVjEP9Xpw{zGw|ty`~`Z${QlyV+@j#$LS_}gKLZWYh@T}6 zpj?Tdkr+OR7X0O_nE;mS3)Ab-$oUzM6>GMBOgB0JLY+a3N*+>z_vS@m)e;JkM1T+v zUg`W6pS7e2MjWk^hEU6)<)sK}ATJ-E;;&yfaC1k_tkzsuBU=~sKg(%MH*F=LFCg+# zFhW2cZB;;goP(1K6bEO{p8eB001dA%G1N1C#|+C=MUZJQpZ-v;O1@9D+CL6}nF01f zPeG6q3j(CeNiv)PD*oK^XCb2_=0o@Zrz;Lbs{{=#d>@GO!Lc|scKmq|Q^0~6x8a`% zhc}QI0mC+a81_UD!org9Z1lmak(ni+PD9{3RFE}Wy6!X?KL_Ce@$<&(y1#D=?#T0K z_`k>E#!5#=cYUxKP6IcHuL=2XK16f9c5NAaC_Xke)=d04tf4`aXdq0D(c(^|eMLn@ zjzf@QLp1cCw2SW%mHEE%7U17Qp3mNx6@6>I1?`V^rzSt+$ZYUIiPQ$Twf5KS{=27tujY~6vakhJ>ei*=xEAy6ZBWw77f zkCn8=D(Cg?a0(=KGTB|n?-A-DK&z0Kf?P^_#m4Y z*=*4wCK^N~fw5)#NzXKq8wQQ!|oFqlZv4Segzq~^=_qX#_w)uGQKS`BbsM2w!0EHor| zJV|RinCt!6AmofVk$#2-3OzyZKcht0g0*!Unn>7f6cJj4w<`(H%>bj%K=vF5he%wp zVIH1TOp>?~`F9bEq(7|-G@Outd3brPz0)9(Uq z%Hcxg@Yy96q(F?gZTQ4|AW$MYPUw2#A@U(}o|0A{B1>p~v6@y~zW?CCWmx?f-W85@ zY<%(JOxYCVyF^`&Kp}qp*9lkyYpUY%WVRbj8j15L`4Zr}@g+h*#N$LWM$92&CW(On z2_NYsgzx9&=2^fddF75k`F*0>d+=5EusygTkJ_x8@YmqSMzPkVcKW^Q=*RW6#POIGi6 zY)xC2rW-yQ-sAY8>RR{j$~`q-vlwA1p$K6edN#BV9Fl5-wpI+zicpEOL39}+dn5JR zT3U7}*gd~|IT60sj@8i65Irje^~A2TPiie4@w)GO4(6B)(Etw&Ga72pG&(rSp`VU^ z`qTr0DgLcn<2jChX2KQ8)C=?x-@A8@0;zl=zxGC?6PRYozkByOG%BiqiCDb!j(?X< z#3BTtxCDIyrMjn`#=p((k(O>paD$Sk6!IJ~>{d+NjN-U&k@f7EFlGsDghAgMP?{{< z-2R}!)wz~S+j~}Lq{2=@r)PZU|h~d8viP&?kml@he z52W14;GUHIffD8}{H_>4^Mm>?#^xa2o~CAI4PWH?U*V}a{`qZCuKSqaiiD^J^SBl#f)cbuINOA6B;P|}-vQU?#BvN(fh=HE_0Cj~1e0*`d7KP;qoX5|SDG zB&#L&Dm~T=JM4p@@ApAo#k*q%BMN}zhQ`9wy3e1>kUTPTa^AhsYDbS5Q2{bo4gqDK z0%Ryy7Ay^9#MgH`E@|X(P#6jvT86{{!*;mf{#pcXd>F|?2L^-?OY-3lvO`fg;P3xP zx&(>jf|ds&6O7YbMFgi!2334H zEhZ>V`7kL1iBQSL%Xz#uoKRPMhwCeuBfzq0lOJ9gW}skf&t1}17x4J;4x19S%8H7& z=nNHe{CkDi!{NNfk}E97eT zx*FX5+b>_%;}_S<-*kK|@G5By?CYQ!c@h;xf&UkoIRmL)tRMVvVy#ZBBBGPJE8I?F zESH*dT;U;m9wRNeo*F;^a|P2cp|f|zdf*|sz4NRTX?1|MKL#07FHmos)Mp^fI8-(> zu0|LlR{I1t+?!Tui@8xM2 z!1t{&58gq*GGeYyw1DIgg1YU2%n|qX92^~wZ?wK4s|)l$&#ql-iQOg6G5iJ%*m*d8 zuZ7GjJv&6{PObLz+=)e4IJ&mwM1_Sw@ElAY0Y8JDMD{~ zQcPG_*v&vhMN^YyWMm{z9_JRB|7ZoBsp;9XF|n~raDO|- zEgjfR`!+gMmMol5?_TNzBLAd@|2W=3nocJF>D zbHumy^XEHHo^-1B1Yvxd_z&wn3=9nSA|sjFzXqMdsf~DfJ0L*QvNTRD#9w6Wd3cYf4?N&c+yMGX(*aJ8MFk|)>OtJSb=Uu#du_p25 z4_NlDrluV7bCQ8!l7JeQM?x7Y9TO}z1WtN<5^j>*P4T~X@E6Y>5z7}}T9wN^p=^Ax zWh3WBBg<`+$YJ|F(Ee zs8*xuS_uS)j53JxQ`EX(DzV|_-|zH+fq~?3rIz#Jjcr_fV~Hef!P7m1UQ@WsyVhHI zS_EO8kv0LWQWF@PF8=(=|9<|c+~g9N^)9?#hJekO(cyy=-v~o4&~|o@+=%Am^2P5N zCWB+0Om2rD{GF*pVkh$840q%NpBK9=T-o2b8 zf5ovXN}gbhjPJi^5_tdqE#ljiq!2UTnD207rjvWV2*tA%I^ys-IJZvzyWwbjIFK1H z-KDJidQeO%C$#p5*34B5~nm?EtJ`-Rk=LygGc`R z!Dj}!tSIKoVQfbpZVyS!E*H-3LN4Qm)xY@Rd;WcR05UhS(J+XDA&p_PU^xV0q*?Kq zVZL6U0G_+eY9CkpP07L(Ql^h}b&6N6q}OYD_6f_%$`V`L6FFCSvFY&6aV9?f7f!ai zF`s}G5v_PDQ7YjR8_c0wyM_KI6xPBvY~(@xw|H3AthtSLlrwlgR2AN&QE0lvF-e{z zB{}xvNqFDLT~{q8n)M~zH~jlzu86W^+FhRD zy#!X+<`2K$7De3^7n3d;Bq`TMs{@uGGob9e8C4~nKD$-)qdg2!K0*Jb5_kW@} zlT4KH_V&iKwL3@@$*5FJ(z_KMZR9@v6&)W~5x-HL>DWHPT6~g_upH&^Gqo`W^&>~h z5Z=h85NjKH3-E&eC?7od7To~o5ERH1*}8QZwv951SD3uVW?;dtMqz?cOXTZleKtFH z&IA>j*sfg%adIZ!Wa5-|K}U0Oc{x3F`4kuV`57|29=)l>XrHKsVG8jThQd90__>w} zw-jX8>IgKw>PDr*1Z@bD!ROwW@-#R+I~4FW%{3%2T7 zltXJNODMHad^0#T>c;!yO!h|Pf~VY2K}K8OOEeRj#gE*C@xcYDPoov~r^tg4vgmNS z`Md@{N4{o2WaPZw?B~W&CdB$cf!3`W4T5+@Cr_RvQ5;1vwl{?gHp374W045$%%ALj zbONI+q3suJXhcTy8+*1 z#_>%@QBzY(2PF_>vuDp9qWuRz;EQ<7yJgE#EVw&pU$3jHyMia)obRv>w*wXSe)L_z zU)9i_Giaa_#W|TY2UpbN&(|e|_hx+)uU&%~#8cDhb&3>IrUHw*)K8r$uTl}@$z(!I&(W6a!M7XNoaJIU zmPzQ0f|(I@O3~d7J4vIDbrUq@EXc&tGcs-k1h62>9R{W$*yxOzOTF+JMt{gN1DpzS zwzjrs@L&a<0M4vJO)Y@rCNmN+!1kThk7ad3(e7$Pvet}a(kTAWf60znKL;_64gHcI z;}35{xtd@avdfYKfiQU{ABijJYqoV@3cFy?7QAgl4mUtL1@}PMtS-T*s5d~#)y8cwKpg$0Y=Fz}0H=p-cFGJr9Tsi`nmuBW z$i^|4BA9sk`;#P&nT>4;8Yjp|@VKlAG{$3H-3tjxFu0B?@<(^KF))6_of4E|+6C%t zx~ziUu4r7O9P55?7sVTC4F@gz9d$$3CKae=u z?&J<69^aAslCumzbH%_;*wrE!+5bEji1;gq>?6rHG;{;7QW^?;Rx7;9PP0Kc4R8a{ z84DR#g!mqvlH!Y2Gh!`H0R-xf_kvbq!`zmk!g)Wa;4gkj-yc^9S86tP_Wh?%7uFY` z#*D8rh09S@Ahem-``cG~qRCJh+XS$vH$*b4Htt#l$vna^9dNq1U!rnyn*cvh=i~qh z0xe$-ne$t;G+<7!2M#0_g3j)d*trv<1l)03XP}@b#R#&DG>P+tm$->j7Fknp;_7;ekrhscad|Nih}(B2@sg$Gdj}BfyBxLuS6Qtp+a?E&SOdg zC4xN`W2IJh7kgDqHQtZA!%OBRcpzqeAp_32TG!}DE*snYQ3>6*^V&mh%i{xOmGld7V zj@G5D42f#g64D0t7PCoeK@fFauQ~eR)Ujhc0Jz)>;TblE z!K#V(HESo}eXsq^X5gDPGK%fa+;EPi@F1k88t9xjk>Gd=Nf`JOG&3$opAu=!HhYc6 zK_4Wk!!&%c!E8NDNo}@s0;i3QV<nl5vrh6l3HQik|1%9TH9oGv?g6S6t}s>e_;}16Ss2xqH}|n60d=Sg;5L$46)SzeFGK zkliQhY4c`)a&tn5JXwm}`L5C9>kdQ{M0cw3rba6EhA%hue;nn};%a?X^dA%g9IXzqPeBNxuP>QjiDU0#gI4oghne`4W8bU6=L6mRIa3{LpO4 zilgB!C@sMzfURd>E1EXT!OWuBauuh;`i&c(U=<2h;>-J>YDf&9_dwo0)qsX@Vlbv5wKgy(6OC6kLN>pzl?$bLN^j^y!AZ@y0kDf z=;q9C6}Gi&JMi9^6^XwLAyf#|h|t4?MnDf7yJ%Q8lzo($DYa_VDy`tq&`=r*x(l_E zoUlSk@(-Ff24(&EryQxr|I-41x6I4S1FA|7Iva49P}8?BUjkqGo)ph<8tY|&A0m%i z5yFkyzU@@;c7*sO>(qq8*@4S(Bgm<>|J?KPRAnS(9k7SEO>-C$vzmf1w+wFi2?m!e zEq$SS=s=`HN2-Ul9ni~kFxqIPW?Zpig@A~NPHyR^#>RE13d&Ies8_E`;Cb+#+zl{$ zN)eJ_45#o#V{l1nsS%ENbcqY&4Qmhn&O5@5?0|i$K-Ne0PvI8q+QQ_s)VDHc6$&F;*T4K$^m?!AIcj&;+}=gRwtP!anC|qM`tZ1!Jr{N zDx~nJxY*d}=m8@mUec|Jmd7{m->*Po+573K|45Q6;Bynqfo`~8{qbXZvrSC6$5pIS zddzTp1FoFF&BQY3a{l;QAdOcr=g>@+pC1FC9zTA}2q9@B%wSV1{DB$(iKanfLbeyB zf^%=7wC6THzUnK3&3mL}Ws`*~5!L});yDf@NhZ;75-**LA~4Pq(9^?+6nr;XAKuEi zArF9YNJjuqMkOcv0Ar=0pioSQ)-PPfMlG%eSdj_l3rYjc0)9;%Ot37szRj3s?U4G? z(mbo0b zCV^;?UBrD0mb!e+4s@*crgw^{y}6&SSIt!WtviQ=B~b5aPFFkvIE8}WAh12wzVx-; zd)6Ie$O+nUoDA?fhKkQFvmE<}kzhQ4DY&j#SRHty3@&KIaf zZchE&78HqDA~aZ=WpJ=bcASlfRya-8SGDeFY!w>)S%Q@vG^nnbCkuis(FWy8 zMe_dBaT1*=-U)pnPm`L=&Ym@fO%|ChiVcd<+SiHj18KuSVwrUHA<;nK*{I|?|Js6; zdQEs4HN_r4dK9V-iKD)Xc;zoMGlQ~)LqkGHA`4&@vG{(*`rfXtr6kRsZkxXo5wXe9 z(a~ajJsYwx3eIB!u~e8H4wHNX3;iYSUmZ2r%`RVNg07Y{=VRWyU_X$NRrpCl-yuD4 z#fr5~RdUC>Fb1TSW+jPNyl$V}$Px_56P3!=L)Sd22=Fu?sZDsn#5Ud*JSJjdw815$ zttb@u0LG&uU3i@B1Uh=J4YhKS-ca;cr&;@?YivD$wCY}1nBG~}6@S~fli+(+X>gj$ zNo#DH`W?Awh5lJ~fNYegU)y&l#Q@6%i>CV?9x2qbyd&BO|7XshKhftf2ap63BaKk! zRl@3KW%}%z4dO?&v!^j4O_NH!#e3jR5jf}Ez5M)iShXov21f)UBAn0~-#ADBXfCp9 zlTcETN%Y7{#E}q_kcihv98z(-f%#cQPfu^=fUt%}R1U@ET`3yf-Q9%UAPgm%S-P}W zW~Sik1h;z{dD9zdjo~Q?(hO_@Sy30ZJcOrD;Sa&gy!Uk3yF~m9w zgED{)h+_&xxa^J9uoEf95DXMao~Q(HsfPM!R+a!Iw(J=>=hyux|E)SoR^*L;))!>b zwT&>A%g?yP{(+RyUBw3?diwj54dy!>U4e!W$cq4w$OlZ|zJ;#&w;*U(KxW~AzNxMr zh;xojzR0}*=^z8^0F1ysSD=OHUIsM<852T)02rzi0T&s@u|>GT=J25JgM+Jc4g3hl z0G~adp?#w33xt)b6o={A61bwZjkVG#pjRG<&Vy8$jB@}RKmeWJd^GM>>i5MKqx&XHz z>sjPu%D@!0{EAU}qJ&cY8lP7{R#;s&v*I(oOf`5z2~5L3kOi-&<@#Qcm@dFcpVeN zXJl@^9eG#@Vntx({NGL{8U<26R1JVeT|z;t!{#5>cA<9IN9s-<8#y?<@pcj%Pv`#% z5&Wj2qZ15UiboV4NW_CGg+`pran>P5l;Ci8M%zTM|FhQ;EkQL0=%7BB+5K?Kt7^QrwlotXkE+Gt~Fra zFd7)g*A!&47ykid--%pb7=Z!g;4ZNU`b$PX?dz^j(CxQb|i^!@0aKN*l?8>9|i=t;)sogy`iX+g^4AO zP(=Wq?%nO*CvUSQSp-bWR;!`mCS2|Rx+(frDWj4Oq8ZJhM&0%wYMKjY|=QL?T<1cNKXd|1O=`%2f#oI z1wGFI)B&7A+j175O*Gs;MJ)gC5p|_sB<51g@wAW0|6kLSrCj@K!oR-S_ zt`QjmK=017zZ%MYcHUNraqJh4C47Jp9!9Pk~+$jaX=K zM5IAc@<4_GJWR?WXOZAX=qn?m=z4Ka$@`!wVc|t+%0)z9@Sn&;0EZYEZA>lasZ)ht z6p@}Tgd=;81U)Thq+n>TCDtfeIvr9Bf(d?@ZUWH%vnak-0$F$L!OdrdKfn&+NVnd#FvaZ76`Y;tEe8BO>)YDq=WaMcHG{|c?`?=oH?_?z z3)Q#-F;x`OA@FN;s>sJqz(6YV&jv0oMx41M^yy7+;TyjOYy_KMo$K~;$|6-{6BZQ2 z&dJHiylx$MHcdvjrKp@eyA`zT9jtd<{X)cvQsj>Mma{AYQyena66#qz1EtRqYG0G9 z2ihMPdEk1gCb_R(f&BI##j@1QFIr&X7JmT8OANv}zAnf6q9O{O#=^b#E&cZlrGaLV zk`HIm!pp$jlPC8-F9Jx_gQQ5Xl_nRdY-!>0^6~<}8@RAAeLLusz&m-?7vTu@1;Jq4 zkfgov(s3pHloHBsV0(J~(-2CRBa0aRtbaKg_+w_ld>^#?X8{fxp(LlO*u}w*iD37@ zif=!7a6EAw$BXLW!vvy(kIRz`5!6s%dTzsl56G#yr31tnM2`T`5tI#d6k;!eJ3;$5 z$0j)Sfg6a1F<5V7TAR>|O>&Ow&CSi@f**y3zJWp*E@2<*lJff<>wh|JjmcQtuCdEn zZ%$h-ptE2*5sk?*u)_N}V`1S9SWJz&su>tLP-I_}4hfdxS77R4JJzfDSd_J^tIG)f zA%s2#o%kFOJ=KzAX^`?(J(gXE#N^n?2|OaqGFw5&K@_QH%*_)sZsMzv{e_%{LIFd* z4-OXI5L$qAKCtzH5fTw{?2tH`kK%P=&g|DUwW^~7DR zt*sOnl;gpA*>G^7C>%Pp;?H19a)t;F1|${=xGBP;L7#F4?u#gm$s@<+@c@m)CnU5I z`6Li^6;)MLck3b80Hb>YDX|LRFQ}BgkcB|1O96UoWp;WQX38!BGhIxmvHTq@iMtoq zWxWgF5i-jN$Ha2pSh5`fJe*kdjKZtr8|eWq$l}=$pk+KJen^X~b2)Kx#}R|cXs{bZ zGJ}wsWMF7ua2E&W8wg6^!3i?uVa0V|EVr3yk=f8tE;eN`;#cwcxA8199;5EXzViSE z$+LCqK{M_ym}R)a{Ru1ulr^4!W(kW%e)$;&5ZwU8E6~-6?o{OFrhB& zx6;z3$SrLOD6D``wjYltrv3o8*TXa!h1G4GDrS%VgZr}}^K*spB;&8-|BE4MM~WlZ zwDtc+0|b9lfpgnIKh70^DT#L|a1NE5H|K=>`;pFL{`qYL8t6CLal)fY)s8!CHWIuA zDhp`nhzBw>Z)iMo$G3W3m4(+2yR{Wqy+M^nFd!ar7ju-D5vISqzL8eZqinS$I%a)H zKmZNpJl0m?-2cvfq*WPi0%1wONVC==U8jSs4nzA0>Of(}EOHPe+p|-CVu8l~CoLey z69zb*9+Wrm>Kz9z+YYMdEu>j-e$>?eUzj=zjaUkRQWFP0)zu-0KF#y~1^fg;^8L6m zS+pR723JRpMv_zcf)Zmf#qS(Q~)#AuWtD^Q~q zwYF{|I})7KbX_Nk^Vx=Vi5@B{B&r(Fi{_4=MWOmKqsKa6TZ#_t)01Cs_i2+E4N0Ii`Nha1umeaZ6AxM z9-uJed*g`m0*UVJ9id}8$Fme%^706IK@%uLVzA&yUP(f!Dr zZI^N6uv3VH-0<5a#41G~c0Ydn5O~49m0o}pAjkqS{DirGFSJ=$ViO-4nUN$H8*re39_6L1Em?Mau@yr zWAwJ7LXASwKI-I=gx+(e7|~4YqN<^xRUjudDc5*o+-XM9MvX0PLGwrDN8AG*TvQ`J zPm~+JCZ?uK92}-)_FZnL=XP|#gN??+uP{Tz0g0D1k`fOg8k#)hAnI{BcQp`V?lmBh zQBQ5f-U|o}bc3!tvC08YN0!uST|X=FvO=UT1gp&h^guTK2H;U;^E@i>E8SHa&@=Z0 zRZc}0nomeC6Bw6@+cJhh*?W=e5)wu$;+8CWTL{X9;$N`A&}wByM@3P1I46KO24r~j z!@pM4qt}XRxWg6~gs?#%O+btD0B~)PK4LJcLz-)htrCL4mnh#z!2q1!dZ&C^Y^>7c z6A*%6?e?pv5cMh1-2@FJO>IadyS;BUgd{{CrdfLO#ECOFD*tvrx=bz!(TjQbt2yR? z|2IrpCs>S5TQDku*r8Pv3f3*PRS}JW<$s#RW=T(tIR1dn(_a9pB{uuBMem?6I+&e4xX`Yor0AGCV%6 zojNf-UJ8@V->|l$jxH61^a;cqIZh}va3iE6-dgCvjrY|dj)P>x?1AO|$rga9K**%lOu zTnc`${6P4ua}CU7(U^mmNRWQiR$xG~)x_(Xnl|G55~&3eUE-$?kcW-}DU=ut>Vlf? zpN_Td7{&HTcD(5AaUbzcr*G#-OxVpxuY|#WLOn+Cm2?Qz#mI1D{S;$05pKX~0L{WT ztcY};^SXbZmFUZ0>sHZ)x`kB6>1#tSqHZ&)z9TSxhSK{)#X43p**;Pwi3Hji)N*oT7B6KHaC zp4|(r)&IZnX7n1Dk<6<$HYs&h?Ck8{<>x1vC`F$>y`V-0tZNvX)=;vc|0i*SzHmRz zel8*xA?ub6Iw?G5_3Ic873u8v@#9AcXOcHVsgL}UY!e98vHO7G&?EN3I+VXF$Qc#F zy~-{Us^B*M{8EbyQ8MG4cy=wGa02M*0+oIcJA@3c06#;zpU_SPu+AyPpt@3gIEAUGaKO4Pe&oDG4!-0{a%QDatdxp#Z?(`0DiyCBv%BtVKH>U zb;2E{19Qnh(>APq(Hkvr#Uu&u8!4t<~Kbk)eT~+sC7BBqnJ6dkE z1<5cR62OLrku!UkWC#$+_Q~6n2>GNWL@UU%Zj;Hoki?ObaXDQY(d^>$r6zPc&Ei!qg`1Y2pIkRey0;wPr5JapA)x(W+r+{ zWb5UG?f~wLP9AftUJvSMe8UzduJU|Zt4Ju!+hpe9~RX1QoA5#fi>PTRHOGu9KLexS_w*a z!^tsiXqs*Q>B>1ip_2q;#d8ehTBv2Dl{lcrrhytp+kQj@NBF7#QumEA*96G05wsdT zK?VS^vqcTEW~3>27>meNrpclZn+`%*4$?^i@gI<+*gqK7hk<84I1?82yWjp5;WS_- z358!3zK5hkai7<%U8{D_Th(cgjLb*{zDS*_W0;t>9so#SplUk`;VlecSt~dbZ42n3 z%VhWU?30d=_q6zoFTsM|q#C*DRoJ%LB5i1fAmUzhOi7;qsPNWJsjnC{8HzL{%m9t9 zgTsmg>TZOM92blucOwzQ%w0Nj#sw)~c+nz$5n4Epbs0Z=I^HTXe87{|yhHxpzDV(d54ABbFEt_5H63hOta z8>H6?=z(nKXD7b*^?A?F^vE+SDjYq^gt9tVW>dg7RN92CzW3;n(1J49qSTm2I}8?g zLYKt_w{PE%|GooMoXwk2dd+GiKyNV-#tY&C9t2+^BctGaM=Vwk_fvI}mXV>Q5YUd# z7gBb(GPu$Zz3Zt{E!ItF5FwRHW{9_EJxvX_l2kuZTCl0~q*(>Xu;5Bp1|+4$DABg4 zsl*zBEMJPodQt|fc0$;!&C156)qQ|MQGhB0n^eq)Zk6|cS^zPzgf1-y!@nzUDk@6v z2-KOkdYEGRZlJ+~w73zg@Z8et?r5^9rwa0oHv;i%&apbM_!7KJ6_Qxd=86x+3tACPd&Or?`08^ zg6%e|Ac9?ZjVH0%YM-6l8AQqhBv=mxSXdHQl2uIRu9JyRV>lP8gqDntjge^pEAF!- za+e07TXg{w)!w+4F0lf43=AOAI{7rRpjr&;S#g{tK!z-?vr=f2AxSyfk4TFT$YO!U z_g&VcipGt@|U{{2Ffo9#)Gl|j1bxbd!umSKO6OJ};#t)>~SrX~U9R+J_Ek<%c zRB%twIU`s)fitrcVUb*k3Ez!E!AK7EjX3iXIIyeJkWDWI;zo%$WlkDpyw?0%^4Br9 zj6fvlKv+UCt98ONpqBAbSE8)(z-Qq2MJ@$&))H^NaTk&?iUQi=Kwt=4aqv;M;e|Dj zjD`p{(UNitz@Qs5u;v6>Xb;fRe+NQhG-co}zV+t4Iy4TW0{SAa8&b8Mj@YLEvMVO79UktVna{ImdxW9N{i40W_#^| z``YSiiwh66)HTLw)UWctv`nPqpKYR+M{}fGRc=^9?;m(^9=zfe>;Bb3kGk)hMR6Eg zP>x?+DVbsEuJUae^=UK*^Xq|iI33ip=5fhu*Dc)RbvT^`)>C>Xvp=g&kBW>8p1Pg* zJQ_Wx6>MvS2L2w1j`wkea$fP;7`N@G7gv1;ZhKmkJ+9&mKa?wCvnoa{In$VjqQTX+ zW@xo_V)ZHN4lnufZLeN`J;Zg};$7K}bFqFdyH$=N58>Qwvnq+^;fHX=*B>rX%cD40 zaVv@ZXa4Ftd!*5qbDM1!5pm7hbEow#eo1sao7kEHI{dgi{RrE^Z+2xz$I&4*rsw^F)p@ z_n?_hThX?`5*v-kQQ5)PG=oT~0C&&1p0|mee+NG~^KKh0sMk#TMSXinDRquB_hcxr zpcs1*@h+L*)E2*I;T}|#XbbcYn)5{@d-jPUuo$TLoi5KSeQnhfF-8~pD$w1tFFx@2 z?=LwWJNM=|>jLaliZz0~e|VU8>(7S?~+9c8B*wWty%1UF_GF1Agibun9Yu|Mp6~BCkDfro5&lY;>_mAILdHG-Q z*`7b=_&c)rOHS7+iA!B?$1hhuxA@H?{W44IC;x6y?s>hOH6ER5yh76}?5|}O{|y)% z9yO<@*bdF^fG3#*h35yi=3j^G;1ij)%oLtKz1lMw`!+b3i1HUkEqA_g*}Zk2zR6qF z8|N>;vDQ&goUXFKPg$|<@XM?hMLhD`-kD}l%SV&ulK5A$%pTRV=F3Vw=_SJsottQ8c1_N_q5N@U>+}eSVz( z`e8-!(;C=;`ez(23#vR(cwVILKC9;N4;DS?>k37&KYPX7EzKTfj-HF=nHRs+8tJ=Z zev;M7`C{?WbHFru{K~)g`VB}sY+gHZ%kkxn{$=MTB8ItkF8*RtNbA^wX5qMI z|A^GpTl1ReBb~W@U#v@ov%$U~WB`Ls3Kdv$>#rLRx8^C9B3`=jj9^r2%M98!{Qjco%gxdXz+{)WZz zr`J5+{YPTP@mukS&#S~8MBko!mz{I7`}D^;`|Ga??PQrYU3R)%^{jk7d$0GM_+?+5 z+9g_7>-n=s7Wn>*-gM<<_S;GI8v!@@>HYo$r(YibRr&Kwv-X(Nxg8u~96Y^`cb_r8 z3R!_T>>H51G8dlxIG{WoU zIXkpNaC z32|l7c}hswqt1&4!gI0{2by08_=&%e-~ZzCpTDCc%FTs#bh}c%MVVLXP=>!vZ;d%O z&-0k`4;M$+dZR0askUP|6bJt;o!l!wn-t%F|5txHST*koSM>bc>a&c#;@1>ztndBF z^XJD>$8>4h8<6XD#m(KNT^48!q)F_vY%CbMyXQX|HGB{}8ghJg674enZjF{Fm25f5Qgl{W{a0 zIlP!)<2`8XzdrL^N>WloLW?WaLc5w0rcI~R4O;C~xt@RCkV7TmzLgbxm&Z7=~5z9Niy?<}3bCuGb z^8C@H)dSJ&OXcSQx@Skq@>+%O`AbvHp-X#aus$`f&Uwf2p4qaGbH1T_yj5If+6(g%x>bG47e@8OoD#63JJ&yJef~#`%>!KV|J{wJ$Ut zh@-MQG%4owT5AceO=lc6DXKH0`_Egr>#$M4cGfmq$>Yr7;9l9PZCh?<3 zw5;RIg%w3jPO@Tsol?(QuZ3K)E_O(o#%zY!jG{es_>4Nu7%7A|2QR_5%B^W_Gd|Nhf zQ*zw;6B{iJ8w1_JRhB@CcFF!Z_5Q}|2d$SYCyP5JOLx26y(|1I4Z8NbL#ISe zt?Ze-p=ZJ1O4+16u%V+R=UuEasAB08~A0l&1)pS!kVPj7LkUTIFFhyVBFea;rA@){8czrS8dHhC!e0 z;}0L*4-Kj|FMJgCb82cTuOa((cMpelgHy;^#-t06Rd=U69V}kb;!|It72f6X=Tdo^ zdf9~|sWcKVGSbC%@@`jtJrt$2UflkSZZ-3^DSLa{hEzqWHAjFrQ|>S4hq#F`=ggq) z9~@ECr4EMuUp7bb3swkiUlwdxr*lYui<OT(qplo>BDq}hCrj?A;8&{I+Qy=zcszvyh*R5Dt;I0nnI0;?)JHS&dDWvbP zx6i@6S{M7w%j)?ry%(&`g;$l_Tlr_MGjBiSrKeo5O}Tl6$e%2C+gf*}5=#5`!TbH0 zH2dlc^$X{1SX{Qwhku;YALvV^G&A~t*0}mIv*g{d7571-*umnN=^vaQ?|KD=hNej6 zOIPY9_NqD>Gsq9_E!#5x+D)a5m37krs+wME*TA!CO~IG1D}v{nsSoc5e;D{EK(WxN zZfQ%@dB43;QnTQH+X+FgU4fyAd^JD)%a&Hv+|FU61Tq}ByNdDU+?TH*;t8jp7%YA5 zJZ)U6DfszO&5S71gV4|ePp@y9Nsp0F3Cq3E?KOLDqusuu;x8wrZzqZUTkKxHe;0Kg z=_%Z>-FIH<^fUV~MX%_9fY03O8}+|U*xy!*kUHe4eIsum{OXN7?n9o(opXnWgjl%W zSE=guOzP?z#lCCF8aR<3o~>VR^l3!&T}4&Q{^aX-=dHJ7hDi!taaK1bF{xuyQe;G* z>A9f!y({|c1f{?Iu;0-c`Jl-;TXufa$GNrsHv>NucmHmQKL9hEPkGVw zquUC+bS)^vQHofh|ufF(JbfKIY%0W^Q(asx&F|l89T< zrF2cVU3(i=JKAb!NvIayOR7$XwAk~swMCs~c*n{9Wg<+?a0~0YR8Z@(e$8N-&Y|~c zCo(H-``Mn=$T{|1@je}_baO{SU#Q8zixl?N#U=F*hGUE4zqoe$Sl)Li_`A-MhD3J;55b zEOFn~(vq^=(xh>9v`3Q^`)zf7`WPR}jr(1R-K@7SKV|-r`*4%jvWM*9ql2p5d4(i? zQgp@5X|Zvtu>#Mp?P3kR7|2rB%)wB9fej$?a(fQZ7oAsqy|20%C*PyktaH3*M}FwW zV$9`~%(KrPWrrFwQd|qpiCPPwBR^QN;b&O3V8E8~K1=(Ke&LYY1@p5`>*8dvis=Rl zk?F?=S@@YfX?$JZsNa>uni3OzC9qEdEV{Iww$+iX#XL){nl#V=O`H5$a>iAsFXEnd4qE{wJzBXZ{MnsCp?)^ zb%th*lF4Of;L;us@;xEvgM6ZdDP^p^!V})U{Cu=JiHoHz)t-H?T4trNyoUSi|s>Mh9+30|48Dw^0jHTcOxa;T94g_>_~ym#(avNERC z=p?zgIO4TcbDjkzl%tx6UXV532sWG{3xf-lAdKo;|Zuj?tw1KKbqW z-Ed?hoK0X!cYEe|*PygMXZNbvtE&v>8Y8csU{?ING2wAZXPknTolD7Va#6%HBuDzX z^`4ylVR~Qt`u&5L#iXi!O6&hx*;hN?lDlLev3Wvxaw_#K{A4kFs94zAb$PR4jvt5K z?Pm5BZ#^KqDJroN@u<+t+^n}uDw-Lr>C;uwrrXEw!?OEPA~89`DJC2Gu6J%|9G^c+!owJOcgXgW)Ho{T-|*6 zsnXbC9=&5)QZ&&4$>tkhqGF>bi}mQudondhFy;HLzvkGU&{aVxK6dMGdt~Fkr`Ylo zn5=FhIV#F<`kY##&B3X9rnHLM+S?P$iZtIpN~ZxdlUMW7OOS;v*(HFdx8UZNRIjq> zipj?i!|IqE|3G>;$Uo)bw0@c8y^2~jIZngQ&?A~s7eC&neV1vR`m^;?Y5ONKA*0b( z#&M74I=oWn3h!OMM6zB~u_a#l>ZR4f#msE)5bxr1cAjZ>H<)TtyOPsy4cp~CJitfA zap=^{X=@IXtC{f`p-$`F4Y%_kPlaF6%9iP#s-gcs@J( zJ=c=0EAI&Z*nsd(2}$wt3Gu^X71T`${q8{t7NIlF-7hW2$3s($xIVu-DFinlJ z+$$;-`x}2Y-G3uavw!M>!9r0_mRYYr20sm#&Q5`NM#4W%&CZSrYNL{j!4j&A-RM&pkVp7s&dAdfU z8tOB_=gyvT=38%NgVseh< z`^AgD2IQ{JkS@N5eu~ABg~NCz3{v`p@c zBM`|NNK4}?t!+`bABgvoFH#n(@^@I5W@TEF`EkMIwC+HYO?3nIQq_e_aY6e4ztVzsZU;O(+W&Z0!iHQ`Ph0OLOzT0QVpNl6hLU*^@ zb!F>iJ026RaPyR^x7d0tR6L$?ilwoM013uD=Or^wjQNoD%V^8LPdc$ni06w2~Hf6E`Izzms83Fzlq(25Xi zjQziUh=1aR1}|hT<3||(-(PVIpo>RGxD0+@1WjOvoZ0^@9vci$7GhKsv?dKvLNH9+ z{m%j;uKM_X%w*yyAeQQ&=l}I@|6I!y|C|5Z7(a$}{Lny_18?@9&-VGxr}XT^5qsPa zG1U3b_2OSLng&^F_6?qO8sWY6SZU+o^R*Xi7PMDi3h~+L3oJa?q-P#`C^2LWPUP(IoeRG;Mi+v;` z7edb+Qh8?mW}|!Xdv1?Jk4^+vBPux6USW03x8FbM_su%FmfCm2GD&4hvL#WXuIICc zpwn}%8C^0eT1D*<@`&`rE_?rI!BwTve%7wf#luoBY{<@q1T{$@_X#X%jh%S$C?t@AJ;rnVT-DPFWW0E7cq?%INs; zeUh&sWjc$+wH8A_u8+@!r?wmWG^-1}oUV{<-4v>a&>tfn#J-xM`TgDMQPK(iw+hS@ zB(}ESly=Mc&Dt*w?P8P32_}CdIR>CZyVV|w(3XXnWjMfmo`FV6l0d4k|*BX&*oK1QqB}R-C+p7H+&wJ}l z^!8<;-+#D1Nm0@7+^BOvd{4Y*FW1I(O4gY#gNtW<#IN#xim#n^&SPWe{#JcxgUy#Z zjOJ%@+kv{e5nPZs5|+Zn^apHyssE`Is{x5HF*>0CpMz1sNedY`NVTh)WZe4 zd+T%TI_NA^~y9GRq`{r_=vq*mO1Z~mAqx=Y&tjPPP$fbpuc#S zr}4Ukln3xa20-FjBiEdi7t=!&Z{4MykIc2JCEvLM< z_u&oaWu-QX#xif;0I>~I@3#3d`Bl%QpYqX1Lh`mZvo71OHG4S`ro*OV9`@wWjn_f1-l!JGHUdVf2h}%Q;>8d>s`m1-M3|~{)D_U_~ zB3h|&pvkDkHOXY3NOvL0BM8HNa!7jyYfE$`mbb9!Eh&*-d+t8z7;vtjyQ*{~ zlhDu<>wGUs-+c6tj!07f0Bd*UklS!~M_UzlW1J;LOdbQa=>y=syl`Y2C#Ol75Bv?`eHYHpuNUA|U z_1wqK9iM4y-`-y^J3C1iprT^^S^4MMV-5XI;g#UO82bdeO)eF+NjzqB-*c}eoVQVQ z?pi$Cals*K75joYnR(i*%XcLCS@&`0Hih%j=dT>mJzcrk{M6~#)Y0-Ss}5ZLi_h*H zX?ZUhmQGmVYO!c*S5y)b1zNH-lDuVVa{-JU)}ILDhB%tT{tp_ z2J$Gk+i`RqZJ{}&b7Gz@SGpk6NxLgOsXFk3K*j1z^5ymX!<7;JqC@rByeD384|<*0 zB#Byn;m^Y@tPYXaZO&=A{d80~p1f}{w5s*EzCt&}qk?nJZ@eQbxtwl2;AE)%(UNO9 zC4QJarDC+*$%EO~$R^F|YVA605vXuB{;$gB>7P?B9lj=bJ)(G|TifVvQMjYWXKSwD zi3`Cen0P{@?r+;dc4Yk(_a&u+wBFTd7Nn$d4usc9X&(LTSvsSsQxwNdLc?=FcrZT! zKheFVb#BzC=HkQA(~%4le=SO%aT~}DoG|@5k+*qnC1a(KYk59S6rn7Q0Oi;6u#GQs zL9gph>ReF~UV2czam2LYb+uiNpw()!T%z$75@U_Lm&^8e)d%wbHU(yCJz~oaX;VZ&{a}h^k%oy5}VT^NmmF@Sm=a1>#S+Txf$Qa@&K-= z{koKX>0@pCCDhbVeyPmdSb6pJ6<;ow#-(jN?^V2~+EQHh<*nSvkiInDomc!tO3>`W zPhPiA^ws7;#w3pF=V|%j#6W~ zK*C-qbztz(&j&pR-%Rm1P$)B~WoOLwh+p23voDN+>6AyZ?w+a)^}EX3jXU#X&~zB5 zpm|{ax=usRTGW%~9VX=)?^)h0=zO1FQ6VZ7GjHkpf>_QE94n4VIyrv(V6uhb6XyVShI3mdzx=R} zr0eqjlAKVdSgX0GZ#hx6#YnJ7V(xuovYNT{O6Q%S{RIaO($xf;cGXRG9uV>oL4+V= zdyNtjPd}S$Gj0~-7U4(tN$Zh`?cBjAFeU?%a0gp^7#kSi8%GNg}k~L<2h2Om!^3I2Q!0dD5Ut#_SO|_ zcFSdbv^|BXI^dhLS1fymVFd*#+g@x6Sq84Y@)m#j5z?5Uo($iJj{2}(h)6Gxuc zC+j(o=*o->Uzo@+?6Ps$Ig#434Bu$) z?pV1FPf4x0L)UiQeP^~g&dls4pUJl8yH14}x4TL7zbBcc=J8|?2}*M+UT890J@{(D zPUCy8lD|XU#Agc2qxT1E3elLX=sIyxNg-1vPt^0S{lX__0XiWchF`B!avPJk^uDlA zUQkqE-X?joGC;y+L9uJNF%+-#?drsI?NQoz6-5CnkzY%Ti}m24Xq1yVlOKBfwP)z+ z&=2v>WO}Y}j!50)Ukkld)T_2fhg&~XKh_yuUtZ5>OkumR(AquJ`g*)ru>X78h@@?f zXz18APw~=-<{b9(=kIhDH7G2{k}W88&(9uNBMoxmi)PwZ#t_41dyDGY{?UjE?g!sm zgQs_YyK^g11rbat~gj<=Y=po?|CrpnR@Md&QY?P$u%cT%IwmWVhbRqL873&MrUuK7CI}}>cI0Q`2`u^oZ z=g+6|Je)PvYi_R;Hdmy6l%G$>aXQ~!Jo+26KSk;4kVe>hvK_ay4NHRKH8^#&aE7Ph zeE{G7L*I7nb-LL%H#9QBZuDfn5JCnC57ltq6u&7qgd`@t!~P2ybi=)lz= zoOrA8bWkSopLJBLrKgK!O5FsV%iBhGC55zAlZ>e5FIqR76YWf*;5r_izdTr1V0@|3 zJ;MIz?62xr=B=9<-94z;ZU+unNy%u1sd@yxe&c_+dTsJ}qC|J$BY_Er@TDyA#B34- zXi5o>l#*>&NXAK*x4b$`Uk?8>88;El#J%b85vJeE1G1^-IW9Up=)N)5U63bWzej6q z>Bfk7#)s-X)n^m0i^=bI4Dn9PkC+|~ID}gBhk)er`U8epl&%t|tP(AjN57Z6*)Q7o zS%X2>e!=`T+NoDsS6 z_jVMvIpwn*ZqK1DV5LrG+Pe9U?%mtBb%o@`*VimitKMwxI2z$IZruDa5Xe*JOs+KFst(M`raYU6|88>qH8@Ob6E>iLSx0{fBOR zcdGWvrZWx{`s>l_cbnF-xfwWEf&^tW8YDB>I2zQM78yq#n9MB=G8M$~ZJ=7AWp?-H zAxSCs#l^_l)rrRLdrh~QGWcnU2&>(LbRjQd&ja(i!Q8C^o zazDmKA)}7#w_!-2^rPkXUn0ytazw_zReSx`uB71$jZXBu)ApnH=Q6L0RSbz}h9$UB zh&#_O(@|8udDEChMd^h`D(D__eCbY? zd#cXLjP0gz{Q)_*zNbi8kQl!xIrxBOIp7X?e~@qDB{a}OZEf!JZ6_BW+w7O+%SCdp zqQd6vn{#TSdV4(RyoT=0*4d8Hgk%iZadK-Z>7EYH?~#g@f9?0CIwEC4AK`} zeUyV-nB9HWb}-mwbaSv|JTCWwn6ij z*+!j5wIWS7>7Ee({?QLL=5uGy)HFM|`J_iIC_ejIhXbW>9FaJpp6z*jSNwamamO`@ zx#qRykJsneSQMFg4~F>{dGUFfR&21srapUC{>tag^QT7ZYjdx5%-Mv#$dO2D-8QQq zJ(m_4La{57p`D3C=dTUR%A5B}es2md6i=}%z<1e_hMgVA%Au7?D

ZX zYFe4?FZl=++vofSrINzN0RIhAO9p5*i9S9=F(BNh7GA)dGJ4i+aF)s7>OZMayNzc?((n^rz2+FzJap& z_bRR*W0O>QSOQN#l5BOtx*Zh=VT|~B_k#vc_C&~~g4(k{pYK0`3bg;08Tk)9QW6+& zY{J}6D4=Sls&(D#PEHyOmMa8x@fOC*ds25}?(Omw_s$TI0&O+(o8mp`+IBx)!dnM? zuZ}3dycGjpc9T(+^re)^G4S%*?5lCK02;1D$#PO{{_9$RD{K zwl&7DX|C<=hWBT~+q1e@x$l#4Adyz~sUkkbHxU!?@Sz!_X>^ICq+8q&F8KhAWTVfZa;cFrq$^E3rC9JokkARefF~ZWV1O0^hEG6op%!Kwe!E-c5;94+qNXdHk>cEEPFur zIR56*)T63S0_w__kEmWI%6ES|{q7$3%(6eyu)tv|eqm^BEt$A(y0$zdH&AjvXoS{v zt466NtE`6As44tgzxIg#IdOKB_SA})lSz%r!kV`tuL$D~#bHN%k>F{o7CB9^6wb>c ztVW0DVmu1W(;nhhw8(e)kFQd>(FMHs&^|kcFa3nnnvRYiU$W|vdn!4R?J`IcWzTU*n+YOXpk8_=V7x>l!QMFLX0;^Mh3atbR za?AaFuU$PJ_C)J*>zCdX0-UdZkXVI2Kc;G9g8$L{{ZZaT#VSpmD%+6 zrd!!|o1d4K7JFIyIHQ{_6`~!e~~0qP$vW& ztd>O2A*f$=#cKxB(m2#H=~)A%h87Q=!)Nq5kM!7ft9{67*AIQAh3JfpQKfwQmfnw9 zGrG}oXgeg^CCz@I(*LDT#B(h|ZBTSnQvOp}LigRe!$A4em?NB0I}M(ozPiZF;PUZG zmU@;)oOz+2>s*ts_!fSG{eX6t$KBqw8BQI%cf-+V206}bKM9aX{kKGjm$Egp${606knC7~+f`;#n$@OaS$|tPFz@3nIHS~(r zFl%}DQ@twx^Yb%lp}h{Zne_+qb%rwwZYpPY>gtyIem*ELJDlSt7|;PeZ*EmC-@ma# zj<-at$zFxQDngTf+-$V}*b{h;!ef(Np@muMYDiTyjcjU4nVfZ8o}RGLiJF$e>h&+F zs3L%Xt}w?dTB2PJ5rJ;g_{d#gR_R;A+l1M^MyMFlDR*2ItI@-uIW#bD9Sm_f_CbxO zWF7_!cVpd}=yK1Q<7*ZKY2g2HWEV}oQPjs9H#fA?sH0OFx^d9FuEtc3V}lm{{e zr>9jQYp#aqtzJB`Bz~eZ3*%ore>*z;)5)lX>XFSvZWyfZo(+hiuvd{!4RS6j3EDI` z4&HNQ{`3^E2M{JRGcJ$_HjSwZuDW|lDxXsmbptxJQu zMovE~dLDfG`Lt%|lH`?AuK~C|3L7fiotO3mQgXntNW~t`3Wz+F$-i&a{zLJzgztBF zPds?kDNm?{M+N+jpXpFSJp4UDQpyyX^f~GL4A>(MM z6Wi|5P<-iXI*Ruv#hn_d&5)_Cj=3*E1M?A~(@}>;r{m4ry4rt7Eufr1;xv5lDGR`- zzL`y!g>SBz#txjH$!p=nS8#o&FGkyqFS;gExAWcGKA3e=^3?oxItXDq)|GHlM+ZVg z%Cjqc%%ue=C!$xz3tyuf-rXJ-AuHaF{Ob)8GW2HP1U;ONwcBeSYX{?FL=a*YIHpjbd|nL z#f!v57ugpR<1W)N$9d1&aB}XA!NEmGpLhna z!J^>(2yZ3IC#Q3ix@(t&2?=Q?XZtms8oho?ujXzSVSX{PdRq-U@OIG&huVyHEQsF< zm;C}wM0c4#%yNFIR{&E29CduZKxc-IAU@_i1wS2patV+*&~fSi{fqUa5jOCT>#iNO zTx;nW9v*Q5zj1NgzT@l9v@klj|YQpR6Go}iOhplkqf{E=-~SKE+6y?2KZs-WCcI{ z$zQB9?=I`fEk#XK?UNW0mFLx&d_iK^Q@s$KW80`w=Z>F{q~_+XYw`t~1|6dp+Pgf< z3hrdS)DB--NkToma8@IC=Zfk{PEnbaN`Slf)hmD&)@dB|AIQ1YB}l!#v=MT7hf`Nh zuRC#u-6}7fm_+_krcNIXU7&@y7%HRR5kosm%gv%BWkobCMG_^d zc0<2_e5u`1oc4wCn7$XR4#CVJR~U4XKiJk~5;i=fsHxJS2d!*mdKa`&b zOYJ!%sVe;do<=1}v2_=_d4iZ14M$e`8msQe-onI+8Z5FRQu9pUw{=1VpDrMJ4BTDS!|c2WB1IZuL$1BT{1O`(v5Y-XemB1Z_#J zn#W80!V{6E`WgN=YGkG?D=R9E9a`-;#RKY5VS|upl@XL{A~2@>TeY140N z?npRbBR7U?Jg*bxH{Au!Z|RxIwv#FYNcY-&G`ysy@@MzR)wNk2V#B7oRe=F;=c zd5$tOmv-AJB`%Vb4%Q>Fr0<6eF+t;fXmOpg`jg(TeTQyPCP30vFntwX5kVis{rr_W(@=Z=X%u_EZ_0)rH&ygefMK$%40s=EHFO#zc8cK>|5lh zVl(NpL47ikddcE>vtHf>#6YoY&S@Yuf?7V~a0y)9VY-u^x-=Q5-F^hf^izr@Zi^V? zXNu$zXu!N~hOaG@D^*wN*a@!F_pXd)QeHFQTPGkO%5U8kR}xM`t})cw<0$iE8E>3S zRvZ^tutvGg1$gLwgzNfajEF7?&&|vm$bKiFf&V@0~7 z$@{XcCjx9Ivv%8b55F2S;f*g)-B{`VDi3-BR>C&#{V|*D>-ka=?n$gQ1^pO*oxWU`Pd&poa*6Aa zjjQu{ifuF@`j(^vva#V;`=M2DafeZ3!d2g_w)3%|jk{j2ns$qgE|k~!mpfEGgdvuXoW|pQjSk`+Ecp5 zXQEcxwj>!oBaZIB2AH5%Zc^+|NqJ{pzL;)LZd_Oadq3<*c8>4X0of(`u!dRhl1*65 zL9qY$Ii{mfC}qGm7VNfErw&$D5gf=1g8WT873%amWkosCo=YFMMGwFNo4Eeg$PHcM zCv;SrkEXokv7gVSNM@w3@9ACpeXtCx+GF8ljPLeo!zpXy12*&#WY0Y#U*{yPWE}{9SK_tPO1-HF#CK z83N|^UYoH?-7^R*Y;Xv_d!{S+tuDeA;PIJHkNp^qv&zi1%~^VMG$lYW z#-7|YbwU+d#FffEOIMQdLlIyWRIvvr#h;u>5SzRYHi;mykGLae6U?EBNw?P&*M64Z zuJ_M?zib|0DaciGAH?>|uCgQmm1wgKi()f} z9_8IL;%ZlVoyz+k+p>d|daX2eNDPn=aV3}O8%hhp3=9PgS5gwhe;ZO44JJCtQY!ul z$mjh6h;Xm1Yeuf9Fj3Iw&K*UTN(db*tMnvkQULG(7xQJ9%JwiMQRPd0hz)w)P)&8- zd(;1kh*^#Pj)>uiI8kJ0(bH>5@_ssU{^w#Y8Y@S({MAQP7e%z?P=UO5%LB z@2wMueyMgsF2(DDeGm?<866Wq>RgiQt(Qjz+taaZ zr$%+A=WNjBDI~@S2tJ_SbtGS%8XC>LW%TF zu+RyihaQy@AV44iLMX}IFU)-BKmY$c_ndp$fAR4N?rD+d7I38+i-e6-v|1ui#qS@F^v%zj$Gklz~IDxD- zhG)vH_;a~jJ%96;FR!k}WZW@4apUuyNX6U>Ukg3Tqi<@#bMwjzZA=Ad!`bt`<=uHa z_ch{n8o#NOG4wcB^jAcn#70gcvw3( z6cpCh)+zy5*sfleudN?A1pWF1cb(@a=$8YTNB#l*{p=3MkKps_wF~S&fX|H6F;Bq< z+nrxJu0o#|euAHdK2Q89^$Ya*^B@1t>9IK$J0x`$vL%Agq~oV1(Vn^9ENua33*Ky$ z6_0}3c(n?1b1EmCE#O&KlB}3KF#wTo10G)uG3pazkIvRhS1bPUN0dmGMnp$X=5=iW zfMYh^%fx?U|H^8qM<1UZU{F|0ipok8J9&3<%pI!=RvDeU;*20z`p!fJ?@Vdn%QEHY zeqc#97i0&zprAmzV)N_AkZ9Y6wDUVVG<=?Blpv3k?JbQ+{u(Rt`tpRbJ9gK{t}EfK z2y(HcmDQtwmSJ6cpJGvvt<3iS%`@;zFjipdAv2B z=Z?CP>{O)VXh|+ItGF~XV?J*AZ4c%@j=mF6A{UlP1#GT#B}$8mrh4i7jKJf)=KIcl zesQ$(dvlm`ODdvUk@1}yn;^pIi*Jl6z%(ksq+EtSq=4_8!x=rpB{sA{Opt8dQl+`i zsAFErsYR7 znce(EoiB4OPu;a=5+d%08RuV2ICN7KuFz(2ovu|Aak{BWGk+e}Z9A@~@|KLjs!At~>vVuql#RL7!2yLVZEmAfHou1OC_K7vR^b#It*dm=b+o*_ zfj*gBhF4RTw(Dxs3*PFlu-j@yc64_3qm2q=Tc8<8)-K(-c%h)v%p14cCLA?QOJ@a) zR=Oq!Z&w9t_vh*$S_OQkH>aZTTtO=#7GAwK$rZkP2xaiz;7JgXw(st$!tO5ZY__WF znwpv#7n#LQHu#g6+Z1c^CaqgpQp}{dtJ-7A^Wy<-Tf8dWf?S>KR(mlvHa1Z$(64gR zk1Wy-E#S30uIM}N(W)8ul;x1@TG^iJJ84QDt{_+?tFst*#fA4-5G9l_M)Bi=gM*6n zaYR2@R%L50YaVbrY8tR09>@=4r!{y-^983s62Oyd$Ob0>_KCPbNVSdvEJ@Bai4x4( zz9=T9UApB|AGEtQGvQ0b6`GdGJ)U$AnBqe&^-K`+p87JNeFmH~iZ`|Kr zFOCBN{|dl~Kd+NCpj3&I8xw(@5;rUVNb?~Ni9>QCf!(2o^T|s<_|%qchWKs>-Jyer{~WAD4>^VZ~?cq=W`8ig1ff$(3dFhSWa4CKqUE;v&l%52&? z-Z9|_idk4#IGRScSKl`E0eIb6303c+&U7N$&V)|;Vz(`~*2z_q2mr(dO`$+9obReP zSj1{5N#5NGm2J}+>6~I->*a{be)sO(?7hGx0ch#;%SUHBnzmMk%-v_69hSH?R_RLX z)0P~peN6!$mY5od{Ly-9iV%Ow5-g(eK^tZ3l|!NDrXcARj#4Qds_~d=IR{hnU9Pu6 zK#CLy!0#p;L!l;C$a4k8MI9N}LY!g|KrNV^*G;LmLIjeOiR}qjY&^RaNst0T6Ds`H z7h=(9v>Ea6W@d>^Ywo*uXF~=3=KOGw#%_wk^uWJL)GeY!YG#w1sRIUjMIJ%)312-R zVA7t`Z2%bp$oZUjY8`P`Vw-{uHmh_=ycu>1-Vt+=Y(a`McWZ~oTgJ`elI?+YamQiF`ZouK0zM2e00T_dl8ZY(JoxOn)~hKBGplk- z6)Ubji%XL2oX<;7gCy3pF@VupOal@k-o8LNva{@qt!rUGD2CKvD#6TTS%k3~6`T?) zW-{9$hUqdgu{1*LbXqe#i|eT!>th!H6l}bImM;9NcIM&A*N-q%Ze0e64+i`w_Mv^T8K9zcch9SNZq(EuZ}}< zx3)M^Hhb~o3QWdn;QbK+rGXGA&rn!uKXwhj&x=jibna(m4LtPQL-q zXCD{@#tYcj;h%_`E%JR;?nM1oaDpd)j6LjVN_KBl6rj&17z>!iD*gN+EsckEn}bWR z5b^+?{0fZ_nPWjIweOv4n0_l{lQ!>lT$O8U6kV>nMk%l|N-d?9>ZCxsJgL3IA@_?= z{r0uxoo}g$%;B}mLH-^}ON5Jox=Yc^>~(~2##Bx4ZTM8jk)4*N@u1!%B;!@Qft6{R z<@Qq2RFJ0yx)R-y-ng6B73AjAUCj(HcOjGx@d|5(ckP&vNZP?W`fk|GjmxIxBBHws z*&^-Y+~cU6u=v>zhq*cfO~&I7Rq3?lNLnU|AQ216HA{6>GlPdPb|dfLj>LvQP9Be2 z8wyTQ>m3{4s$)A%W_xIyxImV-wN=2^=JKUJx$V??>kXb#y%wgO3sY{(-Sl_=Yd)hk zc}Hqp?u`04noIi2dVXoU&7*t{=Gjkh(;}&*O3#&EOI|TJ0FFOg9`p2Q(=)}x)UeON zZ*S60<o=RiYBs(&Bf7YvV63ujQJ9@xQfjliWH%x>NMdZc)OCZm94NT0 zSQU?{?4)SblKXB~N*AZ;o_@?6DTBi85i`{AHF+IYUws|zl9xu6=_!|AXerKQ) z8O+?4C6R8YML&0!=#ssTX-{cKluijMpNHA0s0o`01$!q|Ii~I6ht-XA&{Fe~(Vn~W z1QLX=OrlbG_lO!leRoz%{JmM$MA)sP!Ku!9te*LaoXA?(fzaowRFTi(;sdC-FMNTW?}@TRjWEqR zH626TUSBMwOTbW{B@`~3aMdW;Nbp2Yimr7VkGaH(K-2jHmOm;|zY7D^61QM9ShMrB zQ+ho!ABJj(A}Uu|2l$_lOuSccB;_&riy;Qx9VAMjzg3*oO-^=Wubo*V>+01NFX>xa zrkYFCpNowULo8c&>lADhp+wlbdEOKpv`+Pxz>4}G9W9vcXz+$__k5~3jQHXk+1BLF zc^kFpJ#NKVdPB4u74G-^R(;rccKfpl>y4?0MusrzW8@ql(sgiVhQsQ+D zew2V(+2!LSyZtqYLV5z!{I1{Rg~KMz3~7z4RpELCOXhHT zh!1O6B{Z0{QcfO~fJu=PO%vC<3cXZ6IO`ls5IGrfsg&|++is>bHj(-@M#+Br4)P|F zBf#}4j?9_9{6=JcYoxfs?a*Weu^5Juw+hIz+JNp^_qC@wn#9M)9T5fqX^;TXKL=Tp z$5He+5!h^l3TF0RSfzO1dyg!$sU)`v8Ca>;YM)4f5T_JFGMcuSo{nSMFQzjv-FPuQ zEb4k=P+P0oP8KqGcBF>TEsG&0ts0}S@t!P)JXIgF)Znf0^^Q0Z@v*AgH;r!M$XrBP z)N>nDj8S2C?L)@vLFd8GR*+(Gfm!fWKQ~ti_CRHqdcZsQVhLL#=yl@}|CLEcVo@M& z6Cg2gz!0{+z*^gk7aVaS7fG#dyn>+)Rq5VBE&BFp63?cH#2Qu9%{04Xx`+HoL(bur zC0^dwOstr6>@)B7IFX1dr~08YWIrvBz{Qhb)26sDu}{sbU6tLEMI6U!Ty=Ms@+-QF z%xW%abBpU3^`qC5$^bBa0%KxROiK0TG_1muJ5RdqwvX9Vem%LXvU9;jx&?PD>bn1c zk~}Rxv4ZN(xn5!dmS!{m)r7D0J(RDBHfHPwr+$F+epac8|^ zfAyc?Ur!Z-L7G+pwsG}UBJw+9Mh=#oYQhj$o*c>}E{#J7RUos_w0UO1ZA_ zl|0+YCHYQfv8quchz63wV51SF@w@zdj5X5bl`z;9KS-|XBx z)d^vzv=?=aeq2kbbwOSxeu9J2I%YB>8H$m$o9`i6zHXu(qlxh^pM8v#+~~{@;u_6R zlyAwMESTULRfWtfcqrqp8z}v`^#~?v7{Y7v4O4G#?{>lGzdv62D$)+F!UV<2!otgW z*1mpz0{sPzHm)Uc5@t!J`0kFrX6^n=|4^5P&qFzphBm$CVN%Y0g@(pPx!0pOAr8_? zvNxa%*16XwpT{L$Ft`&$_3xP71p$Cdviue*=^wcT^=Pb2dz!~uV8l>+bm0- z;U_X~lM0p_6#a$YoCoV&X_;$VP!@YO_Thpob*nPL_7xRJGrw0RPGS-xGO~6C4;|=n$oJeQ` zAQRBXoqb>JKOVODH^IJKLZSq!0g1AvZT;o46w(3_=ke_jU(mVht zx?bd6ARPu1TzEU0^b!K~#;XR7Ro>#ObssE18yH(vzUl`L*#e|4vJ?9M509nht+0k2 zvw%j+Q4hKZOhMvCTlww!z=yfvsi+yD%X+|Z=+Ci^UsEk|?+_ab3f+MGm~&xq)7jRk zhKNPrv&=HZDELBqJNQ`S*sOGD@Oe1g`P4nKnfvt1w~10dTBzw<%tTCgpptzLJOm~3 z|KfDH_oxlR8`aacTMgxkyPDjObS-gDq{XK{Rs=FS0t|!VvS@3LQvq6uLSN!0S~(Nl zYQDx@wyc*4wS0VUx$b-8?P}LQBm(FAMB0-Lh+VG4262q;W7*DA;@aEGONh3Y*GpUP z`WKp1aySNK;y=l|X7iNtQV)FF3Xw3d;$Hz50A<+qt*Av=ebaU2ox7sC1` zRk+1nw8^twlh-6ICAyR5v?Hy^JCf?_o|EY;dg!y;{;bv@0XXIJ1Lt9-_q(6%d|$D)?XAB#ANU7F#&twSQf=j7 zvFhq5Kkc(kA8V0GuF&*Ob>GMyG2Pktu0kh~N|z+Y>@e5-XJYfL+mFkuz2!O;^yUFp z$e6UtBn2i*BXv(T*8EO5(r_MP?@0f}1lx6AajU?cLC48l`mmDKtL6XWvSgOq@UfYF~$^!w!vB?jQ9 zcV>r))8Ail6D%UCFVYvX)T{wwpKw83I@wA|9g}QSJ>Gh|&`4LO-uofi#K;~SP1i<+ zv-&qP*5MIg(mLAw+oqaFw4wYyF(E3Ai2`whm@;yQ2}n7Bx)I+ zSvXW(TPS~j^NAs*u)8bIZuZTwP}@-a;v12gDU`AlYmmX%YOuyj_)}0Kii^)ad#=AJ zke)?dfpF?lfiw1=%vw)$2bf2!)Ep$Wg+<*4K_UPZi5en_9|+AYCIn${z)R_Oy;_wL zG%@c;sIUaVMMUTGD#%0V-tDY$1O^Oc+oZ_ux&E~PfuEt<0ul^b)KnF_2$?DPK8W1x zKE|_&dkS31$T2aj0Cfyv2nl}Q2e^RX%>^Oqw2q$LWGBrj(zT;u>pdP)+h4Dzov%;( z-@P(Kz>fG-ztgrcWK*L>4hF7g1kjl7L$%M`f&b+=igpA>I9t~lw=>q&l_sOS%dD`5 zoHWWA9DR~R5Kx;q1M6Eavvi*LwcZ(P87<;ozp2r%?*P?;5HI_p z|My0S-pCC+-i{@`frsu}5F?+lR?nW@!24K^!B=;hkt+l)yV+=IX;uXT>AYobN}WWR z-5h95RQ7g5*|y>rt6OR|CPoQ<4Vh$Is+>HC#hKPVb_`Nqq*rtY-aFgvkF-jqy0ry} zk5=BwYKd@)aIRfn_yaNjJqNZ>Np$Y7*P$(rJCl+K?zqQd?L_UG;BufVin`SUm;K<{ zsJJEb+;`}Av+JQ01}RXo;9jbCGxcNv0Ug=^MT<5SijR!5sWNjkD@XkbAdQQ%{Zw=u zC9<8mMmIMW+MHo$1VvFu!ND6?)0&5HA2QZlcWWIsUv8bJgrPB?4HBPLRembC8$FuS z^P;%wG4s0!3|v=ZW$ltp% zxW0pWT`aCxVy>f;#l+sQEIkLCD&oa>x7DWUTL}3>_rw(s;O72tiO(dLP|_f^%!=&% z&Iprc#;wkmyXPgD|G|wjkAbdExGL8)irVgkj2n|)Iyae4G$(nhT`tO1ms3@ZS~Mk( z0}>H-kTyeOl~a{4RAuVe>;N&%j69BxEhzQs?K6}uLL+8%RIN*wcZ(5~md5tUSKKuG z_(!vbY7+Hug|Wei_fh|XwyO<#i>XFp#xs!RStQvJ{N6MxV;Z8tb!;AKh&O8<2Z~#2 zyJp1O%7$d6ocHeT&0J>12D_MX(Y;upiTt7q36%z8GVc8; zSbb7gvR@-t;CK*WkWOeBe^R%-()sM(Q6XxXEHdJhT;#c!)0&GmHF8tgZn3Cj!5NJ4bPOUQOG52Z=PAj~Y8Fqt zs9V5USgAQ{LJEW;YQA;K7t;Oegv$MumiwD?%Ikyl>o%4cXCa4zZl`5 zdRAC7vg^&6>uNsZ@9$>B_*i>?%V&|2n8gX1f@Rk!NBxhvZj3g}83%hlZsts>;u?EFRzj=uYZT)MX* z)(bxkWL>dou}u%$fQ51jXbC*`>sj9{9&yY| z)!NGq{_GxhksW01ETNGQRsEN1pbTDq7POGey+-N(ZyC&g1KR)Q{ON!1?hvX^;Hw(f zi$GLetRk|)nIK_e>22iJ$e1;2#egqeu)7<{(t-eUS3`1+*p@kiJEcaHU_;9)$kdh% zku7~jZXWvyJpOQ~MkfKc9AltsN=nd(;GGGV?tahu9m4s6cVHqJ_QU8O}K!|AXY^EBoQlokQ0DP=ggY^0c%JiG~_+7{0Yq1i{1rQVdGe&<^qcE_xjMu^Y3q~}T(ont0Csh;Epx znf=!ugVN20=g7rnWvY;UT8vOsz4XM!q&%Lrv>L!#o9Zw1P7YvBOY4KZ(&|}~HH%_- zq-l4HgiPygW7Htm6(p%ho)#j>iRqbaFs3c|riB&UMs9iO>@b1*6Ymh#2*kslOUCIon^1h}dM3()cm4Gm|L zLR$ziajk*K8{J!rX-Fy)I~{&lvc1IqC>|XvM*F^*=Q%zi+*h-04>AD_z566Q(J1}l ze(vo}bE}{S`kd>urRB-M2u7sB zQlUuuXWwy@2*WEq$T@tz&t|=-8ex@Yg(x&4%gv9LnG|^_`^+cNClk}hig&%OQdVpM zHO0Y@G&*;KNx}5CJqDjr@;A1Mk6ccmpSpYt6%&tP^vXMpPYbo+tOE7>I%q%zFO^3`VMOlMqix4|l1B?65e`pvgf7Td+W zz0oEV%qT0o*Lc={jhv2P(FlwIwe7$~O5MdFC~@d;!Gh=B?@K;rmaldEzL%`2uCVU3j+41ZWGcP^zV*bz^iQ#%vw!~E#8K@VQXgBHw2V(08$fI2nS0&o2h0t zn%3Sob;q*|a@C~OEd7^b(dTuB5-1h%xzioA#w3`YN?f<9UvBALqw{J~J`pO>g25s) zCr{1;<_5qNjbL*82GMJ~J9jHyHz6_j?r>82_At@chOmc{TQN@$^xJI0H!n~}l2{xtz#un<#Hx5T79$11Sh@2#!U zRJjx`q>uVzT_$o}BL#hY>>3pmv5ancsim4j$ggqgv-YHvMC$7fejh4#f34*gYiL6%NS3vf{p{L! zFipslnQ^_$h<3`f^s$E>J|RwI#4R)xCBpLWr_lP!j?4M&nTZ!)Tt6M~+NzT5?xS;) zf?1pDHSPluFqG=nIzGNJ*ffZ_E=dUIQ&@|+Y)K6YqEAs~-l`MhMDpLi@4}`$a?(`< zj01{vP$|@iU_CSl2LUd#X0FtUw9AcEbRtcx3}_a zl9+?Q4Kv&u-{#_y3AII!ZHDZo_$TtcLD5|z@Du|UtM6uMz~_20FMd>e4?IjB8p*O|RS3G5 z=#b4?&8nQSuzha*DZu_TSn%JUnD{R$i~P&uc?lf3dqV00YL~(P!htIgDEGt4Z^k~= zpCp$d=tl|g zY02O?5#;K;D1y?)2Pu_bMYrojmm9>Qb{Lho4K;9#ColfsxfKD_s@<1k2WQY>-JPF( zCzC{Y{I}*=PyQ;KwWxWF1;j~A2< z{IEH_ImS-6{if`0KkUmCZcSmt>bEj(X{H6ua>Pcnb{hZo^%@THCl!sAPsAL9bfVhZ zn5P5x>K{Eo*;c92(KLUwNs*Q*W458mtI0#uZHT_-ke_fnejc``UB_74@-P%xqTvD% zBNKJcMd$w9S(GGa`$d0m+}CC0X+BM%l-A|Otl1sNq2uOMJgYq!jGY~pj#7&`%psJ< zCxI;=vZ&hG;AV8c(zIuk=6HAe;pmcWu|!@g+Lj4-N-{BsveOkzi`XI6*w0vRuXn|7 zQDUuASQTbG%+)|_i%+FcOwXcm{4${xiV zBR58Qv#mi)5Nkwkm=Ot?v|`5wOQ5tkQ)_LvBU2aZA$m8(b+~>Kfki4!5&OIfW|12u z*xAD@Kc8AkzkS3NaDA%TKppOXkM;=iqgg{|5o!0ZRud&E2R_nZQmOoprZX04HBTAo z>CDr>;>bW2S0r)mOO|5|t}>A|%a^!IU0dwQ*Pcw=wzEo>lg&$`PegR91Lsh2xsxpb!k9#*=(EH2>5AlIp7B+ zs?8^^E&AWzt4oND@LX9HwQHur6E{~vAmvr*iVZxk%D9J`Za6@hh74MD8D0l<+i&Ns zNV{~@t}Lt+|0H;1&c#S4Y5L1C?rf0Hh3OkxrRgm-SWgDKKbUQ}VxL7i2=;XN32@{H zS=wF%rZ$mBh3c(JdA3vTl=3pwbl8947A<1a^4x;wsOrRPg}Pl1Mvni0fNw zxP;jBtxC2q1C!}|36L}{e94oZf7Mz_bDQXM8mpWi!368@Y<6XF!70p87Zh}ifaw}= zP0}fa?0Hk1E?BW*%p+c;eOk1&$YZ&AwLs`pUl1u07(QU6PsT<_&9}Eo&A-y5C8`}f z58D{V217JrZhUP(I(HZ9c!upo0lc%0WOpmk=MpkDZ&IpcY=Nt{kLV|m>1VpLIhL%Q2 zZTtx$H0g$5Hq3B@o+IZayrh8?!xE!E_hD#xL#k9uL zXt|oXFdq2XTeg);nGvQ)_E#^K^`V4twtK7BUJ$5TzXfhg1b#{l%U*h^<`uXj<}}=V z3;^#~ZrEs_khh381$w0~xg|yhQgK%U#R??8^`8V0ixd$aRlhZ**0|W%nH_7(c=kwz z+(w(b2Puua#&#;|7lCKmS7IKOBuaOCHH&3vB+0p_0jD}%Nh`$LD{qKL(bM+D5guyE z6bK~5W?m2Vn}hPe15v%jM{GmNLG`nrz-En(9MOn7)g;d&UD?N>c4^Q*x+uA@ahC^- z%mC~ea267U)Rv0LV$1z-phNZEx5_(*_cQE0Uy(_B^!#_{^8b@2bFUO&$7}Ng^nWk1 z0V<#SP8o1S@++1O9DA>MJ(Lso@|f^mv7=^Y+9H<4+z0h}RddYK>o1O6^!~>}FQ}&O zuX49emxhq8O8viE-teD9$$$CuWJ?xv7b=e>i^4#Wxf%H$2&K(-B^rD$2k2ru8Y=Y$ zxBTU}o&_lZM4i^i1W8aZJbdI8^zbz((X^8XQ71!V>5`ZjV)#L5jyHKIS#f_r zE*KE0x-o2MWK^_1-VrN?=(w2YGP@T!Jem$ddk#Z|rjbeu@1f{n1R{b@zWbOa|K0yJ z_Q#lxE_>Mg+N>2SWWN@|kptdVIHv_Aw0a(0WWSjBnN#^PE?FMq?ZTkWD1y=2F;F}+ z`R{$&4=culy_M#(qf1>zN<|9XkEwy*hqhFofthSB zDj;^~-E-|=j_X(@%$RsK0K^w#6!sAEIp;`rHh5P7I)?~sw8}l5ol_y>%N{0qi}&Js zz}Jmc`f+{wvlu(5?t+plD}H^s_k5*iNj{Io9VrQg3wu+|;+3F~P{Uy7u2q$u|<6`LLgDJ9~Juj2abn^4c zN{3RFrelPY5|5(d2)A(;PsE(yKYQ}-z>Ndfc)BTE*feX4(1Js5U+b-%ON(2?{{|9>I3L@W17P+dGWH6TA>981ej0p$AE=fSvJ~Kl?;6if1;bf) zx9nE2v0mqT=PvAj-}Xby(+^vLL^tt-ofi_;w|-aZQ1@A@YA}-~37@m7zwdQoZ`>1$ zP^n)SS_6&N6%V)kx5U1s^xoZHz*C^(s;yX9^VzUwDz+7Fr}jdoW-p`&9t$h}CcxIB za0+>q%u+RI<ap&%NM8k+9q)qjt#u$TxkYcR`{Y~7Dt|JhpkTF1pU83DM>COMU;Zd`S zipWHDk3xL|)S&CILeaL%3!i|xYn`8?U?|R|$;?z@a8U@>r5`tNf}w7zH~5SfeKX#w zw5*DdH}ztnDbf{{ZJHJOUhBSUgVbt>iz!qK3*yu28Ar;mh5Ar>rW-#>DWANvd&xPszXYpz89JM4ECzNordEAXWlBA7*LTR`KCemHS}Kn_@rY&WQ@La+4Au3MRXgFOB?9nrs{%P%F)8v9$ zK+3bwaFiGM-1&IP6vA};vy=WZS|`zi)fFum>|9ZrB>Ii>gWUp4&667|Js$nl?YoYhOHs_xZ#+3|*=>RnL_&98z$-h?qMb+v;49u)Z)%eeN~f!um!~}jnS_2`Rr|MEChvhRXk67E8B=y(;=c{`m0o9MF3>SaquT}v zTj9ys+*?PD?KECQzMvR{{&3x>QOn!(a-62sUv$ELO?)bMeyb^H*JJua8ktdA@l#AT zYUa}W`NJQFh?sL$LwIY9LCzDip|f{T{QNkl-kf;AS_=fB#=%$bZ$YybWZhlGe>|7( z^<2vj{{0}@Fso9RKeFf5=VL#=C$^cj01q`JD{X$*Xx>rYdnKM_+ddNbW%Dw8oOUK; zZM7x6bkXXzUfsp`Pj0noUQ20&WP!anCL7x{_u`TAEB3lyfHzlQh@C2v(le}N`i{3x zmEOCo)sR-0AF^%7<69S?OeYCly4Q9SW!R1B5TI2o59*=TI_1gdV1o;?^)|PkqpRGS zRkvFYk*~;e=^6d}J{E2<|8uO@64iS2!&d_V)ek0z>~+bDLZV)(^JZ}<3bwg5Ens>t zKE}pIJ?z6Vn&ABE6-q?8PMNh9se6Daw~R4w?b7vpdaT!RFF|0f>BOeO>XMD#F5+ z>cI`&TXnvQ)-}k-3r!nE9k=v4rF@pgc^1q*n_M~e4SnK{`RtDemJ=vzB@Q@##ap5D zMSTYP2hT?`pj-4Q2=0$6hx1mX@d;i^=}6 z`+51)7^a+__j_uDYG(RS(ZC$HdD}>{QGR>-Yh|Pb%CJ5rN@PCNNZb1|wMiKB_hB9- ze>l3JYWK{|LT`wxAb!5J=Oikd_b4AZ_Lb1w)PvoksiXt+o`?am8I?IUx;nuDM5iGn z$#*_qP?#9gZi^CG|Bd;=w!{JO(9h;IH&D}7>3{~#4G|EyOvXydNOS$QX_TEm3 zgiA}bv|ikpqfqlV|Ev$_m_%4;d7=O=FP&2~ow;Q4HqSS|E>gQ8$g)H;YkKiJ8r3*6 z^Yt;QQ=!Vthxa zLnXnf>Uqy3Buw4Jf?_IG9Yz^`4&eQ;h4C6%VC(Z2FD@x4+ynJ{|A6@CYEWeykfW~t zx&hU&2CNuB1c+vDrceE>pj_a&BuY_kDRQ_n=rpkQRGOJuP#5YX;}SraDqa>ejYK|< zZ5#Sq1F)yKQeB8LfmYax($iIA=X&4fHgdZ4>G^dHlx!>3X^_8u;vi6d>Y;3DtSNEo}HQD`N zJ|8t()x41qNs&{jr0s)5W$(DE@*xYmG?=6)baD0jfvhYC+~U01*0*d;HA#8P<9oxd z>LPwuVxvaB55zh(uhG}0`^62vlqF=Ej;?X`Mhp!6tD#v(ZltYwg%(GxtL#mFvhzgh z7gTn#yqm*>y>z+j_4g7j=|&}}oRd8nZ?_`T-#vS7u{K|wsr11HygV^9cLx3IJw43r zovWyrfc4jLI?357wgEXju**~aX)~Jr^Sj`s`wgXs@w`$h1NxAZ@#$^-3f1I*F~K}^XQ1MSu;}9 zaD}D$P;t+fcqK7aaUJp6^*Dnc?8SE#$t24GAEREHg3XiI1 zh1pLHyk82Vu1Vww5sG`lkFhWNO*RxC+5_`6_M!dh7$4dTCe7&mkI8aX%jMD8zrs5m zKW2Np-k$o?YEJ$huxv`ajd}tcM~oPk2G7Ad)l#zml+Ii0H8Eeifm1?dyM1nPdYImA zM*S!v8(0!%5mv2@>fU99S_RQ8Raw*`&*Ih<`8B&J?lqgapDPw7O+d|LKsNc70zwNo zRmE0WqHVTP?Wn~3Z+P{rm#OMR@11`QU*g+~FePBEfEqK4s+u3v%e-qpZJ9f6iLZ-$ zYzhpaZtRavreRlMz_t|tpIA$;F5fzMbI7m$&I9=h@)&WeuDx3~_2AL?=7+r>t(mko z!UKsrS1m&$W7};@yD8S9w+I-^!DfWL5E6>O0dcL5Onv$W81sd#EeG**X7P(k$Fu3L z_1^pKYW(cdce{dDU0m6JyDp8}ms$-JvOY%CdEslLsajo{mrh^N#FHvuq$bSw}0%t(y8~rmrK@>2!p4+cM+gi)qfE{DC`o$ zRJ(K!2r_uP4lJvNHa|p8Gxik?Rmq3r56I~gK6kTOJN_>H+tfhdrUFdCUz z*L&czJ(6EM^5%wwuia520d=oChefexV1A{J2$=q=z8Wf9$XI-^q>UXSKCr#Ob=Bz)ZA2dr6+Xct z!CddHXM)R1YQyTj@lNgPI?}MM#kITjxk~9I!qC}HuU}q!c0PY-$!NWOD~k1r z;;FNLE(UnL==y=fH4f8VjHD|MpS1Y8+N-+xcn{}f6o(<1?8f8^<=iol!+#k=V?I_IM`@TS>$W>xt4dvatAt=EwM z^l3!;(&BPvX%G`&X))5{FM|vm@Mgcu=M-V0S!aPFgUoqbf1P0*qLxtnUh9Vn8IN`| zK$wBm>j#YnI?dH$cRY7$1;2ppNAglJp&%HO)t#zL^tEtsc>gRU1o;gLL9($a+K4N& zKbQfWu;X-~17rL(uod}uf?@m>C}P73w~&Kot<6O*ozw=WBc~8VUdC0DeBhaTvCis@ zt2ROd|8ls@BR7hV-NE*6~(BZYFVqrbg@N*RH+49+_8=Yv{b zO7Ys-+T~V$e*S^RK>wQR>fd7i8o+u727n-Lpyqu1+qap^6LoxhA=1C+cG;CF`7B;2 z8*x#c{r>$OXfeUJS0)LbLsK(B*=KIDyt@{PUqE0Gl$c(gIlZ?tTPYX(^pU}GM}w2% zW`?MkxVYS6%R1f^DE#{O+{LA(!7NQKn10E_W>9)69I{tO2{3NWsd4bh(k(5mzWM&V zb%J&9d|!@st=BxNudi=zlG5+(?R^~)F_@zrUmA|^@rerw30W5f-+%k|4V4WvYWu^I zd#$G~+_~D=*oeN9`3!A>DF%_{_aNy)rO~3>ZT$)fP_hpY#RL%jZK|rj@q-5sMjHd` ze3r-aL30&tRDFHDVdftP7z{=}Ec!fkW5f8>uji&`XV1&a%abFo!bZy-qd`O*zN;!N zZD4}28U$@HyjH*8Y;J88Q&Y17%3UAm*IeCHba?m?rVT9WntdcR2UC zay+D+N?$UGGX2{c$X@!9nc3EMtH`p>m`~nq03?Z~r>CdCg_mrw7Vue|tGDDI;U}@#9CS z{W^6JXep~$Jo1G-EG(=aG#{!6V0hlt)GUA|0syW2=FfpRy0)&aZj^XfR#w)|?(PhQ z;w~Qb@ZrO9f4bY!aLM(sQ{wt>&)x+49&Fg5b^-)X+H~%#^Ec)IrH#O8s&pB7r-+!H z1C37NlarH^WSw)tYp-ahL3GV*oJDlIh`PEugkE3DBbspP`F4;fn23!HLLZ?AD@^wdE!JiSd~{`|DyEu?Ol%1CrA0@ zATjksAo@$;orV}I)hyweLxO^WTc8aU(26EUh14wA4{{o6XuRkx*~d2XO67|g#9?j; zLy(?Z2d$tWE-Nj|Vg>%Tv9aN|K4@Z8;|Xs7{j;=g-RgOLQVdA5t(VL`o8XZh2RD$c9MEHmsN^~Ql2^{F6ZM^n(6R;gS1BX{G4`J1wzl(oZ`*m{~I!NVim_bxCWqw#RG5 zUC!52-!;5=@gmqY7KK9nbIw&T=Ph1s`!=P4p4Jl11M1#+c?3kQX=`f(qUize)LA2-JL)l)vUp|$;+>#e&_0JYrBI_Q=M5{ECj8dI&7tQ1lsL@ z#M}mX)jNQ3-(_V*MDDw(k4$2oB8UGxbi5qAoY4S!k9qp|$SFnQK#MGbVeV}n(BKLN zP^Gcg>c43VB!Eub3sA+}!pw>-VP24XH6oZ#at0Gg8bZ`e7(;&=Y| z;~B7=?|@1-2H2BU;$-Fm?C>i46wY?17OayAe18o;eYm4Z#R1r1&qt4-PJR)Q&+4H5 zbc4A%d`AUU?g6a7H5L{YDaOSX`2ehr$;rt%h0Rbye-V+Hw5PT?Zi&1NQ@zwdwqG z#n)25b00o>)X3Ab6lgVjuM@WH!1xU28)(uaR7nT z2Lc%fkO5=_@Q@2|*xrLCat`YYgZ~8WB{=NQ05s5a&DQ?^;qE=4s?4@@Q7sIm5-bB^ z01O}*l$=3PaHB{D$tp>5&RIoD6wpnQ43e`*&L~P0keoqs&N*%FTx>kuJ^I|feP7@4 z`o6~)=TsH^d;crW`OR;BYp%)&+ETmGAYDrUYUF&@Fz@)mtKfzts|}#f4^Yqz?QRi^ zZD_#^A8UB_$Q1ARD20aBd6yes*>iDjlY?+b=)+ z$JpII|KSAG8N>i$PEs!}xH6@yrS)vKCto?^sXJA5YpXoiy6#+)cCZJ?j!T5|E5W1K z7=Q;8;Nbn-NUC6&-MV!Pa2y;X+H0w8O5Rb9b7;imbB)z_xXGGZy(xv%85B~aV9RAg zP-oOvqy#{5b2_u3<+JwvnZ9Bb=sNcM)0wcG|Lr zopb&(4!5(l0xm5XYApuCGJ(o(*D#-Ra}}XzB)jvrtv{2B0-wz+6b=1$E!96LC@6Jk z5er^4Ha^~9Yjx%yBmc7dr9}|9B3f$dyI_;pwRXTI-^XsQ&AGG(26A(AKb!8z=*lyT z2Gji6(Ry(hgZ6k=sY@yB23T#;U>vKetE+jhb8;$V>eMxX!Li)gw31Jdx+UPSDhCTN z^T*2NA>+QHM8MgT9+b|=1OX~OLq?VWPN%A~vy+#Zjt&E**;>IN`+xY5fIup&IUD}1 zc{6&Mn76`LZ0?#;njmKLGsVfy_o#n>TOXG`F!)%+_z6<%t7R zV&muk`t|Dz?9bkzEd<^V*3yH1$0aWr85wqPTNs%;cV2=|kGp!;Gr?(V)!f{?$(x*S z8LD0;0I-_u&YctS0yGm`vWs(oZxxs1UwAnb^`p52SJh;YUL5!9*RRm^3zYYSgUE;pd9;+k-*eSb>|K3na)fxtX9S&cFhVx9$sF3-1f%e^0FV* z3Pg!e__G;HeSaBNQcY#$m|KONg98FAU@%*t2ArN+=DzQPbrK7PaU6JsXB*J**CnhFd2~eVvGLT9k4L|>(_7byb0FE6y=IQONU}>2JyFT@BgymGu&{((w zg4|t&Hgg0%;1u6cxqEmpT)$qyn+C-iy9%t5VIot(SnT89Tv9hIG8T}%qGMu`fJ%)H z;|6ui@a`t?yq|~hgIR?!6sx%lb7~1|b7i8r^4$tVFU%14!aoF;^C8mqjgHm?I+d9j zYpm7|5hOJRrn6v372*aAiYP}$M^}Z15FiZHa8?5FEj!m&ET5*zgKQ&sM8#tJ6)otY zZ2Uur2w|sC*UsL;h(CN7K=u9mcY)<>oj|CbTK(nAt@rQW?>^(YzBgKBe7IXf)CLN2<(IpUA=a#6}Ep}7+<@6im=-sYnMzi9$AjmoW)>s zbJxftATU#Ecll|CZHbqpOjXH!2+LI7?T^FDt%@3Y!&UDm+fqZA)e5x2Y8xA+iHV7o zoO^nDP%N{tx!JeISB>%Z_NJGOdkglGuy(Ie2MkZ}5m>wj|5pQMi~4>5JL3!Rr%=2# z>eHuB4kaVjLls^H<|CdEtWnzbb&vGHmTZOkj*Jn2R_-;Wj_nAH=VfKF_vV?2fcKhg zPmcgI7?1p6HI^EwzuMB&k^#em1i;ae-E8n1qlLDP&Lljf)Msh<64vVEaJBC%9ei_g zUWB>j-!n9vRx`N@v~YR8XKb#6BC>1EnM?% z0BC-hkB<*bp}>4zzw?#^WE7hL<)%iQ11Aa6%6C*~8G^+@em+aLA)L>e7luj>JvRrc z!Ls&ur;Cb2%qr!m4dIu3Qz?uX=^%9}n(m+k{T8zF$KmiPLvT<(# zgzQy`>5FcDj7l;Y85#dFf?&blzkGQJxB)V95m+Sw9v&WoxOGj4jhgIxv{>nG+>in+ z5EvNPF4DmFxaXS63HsI|kN98ysj&(F6kzZos5B zyCcU)1tB>ow+U7v0+H3L`t509@K+WvUq9R5hw=0CBhmtr&gGeVGwZtD02Yf;l&LRK zmIl)VfWyo4T{1twetOlsPXOmNEze+b)wj){Xzurlikk1=@1CWXdzYEHbgjW*V^JH% zvdWM%8N__%R#rQeLo%|mR@>`lVL~qI#>U2ojDew?BeMZl1CNfw@`B%e6{2Ue_zJ)Z z1@P3@uPgV*7}nkv5jlJ55+lK`(q5Q;SxBX}9_ASsBzm=itj3!u53c*dN|@W!=@%QA zolT33j5Iv27%6-ZT)YGj7=?3aWLi^{mZHoBiE~es!CIHzE1-SA^-?7p?53EQ7{g9G zkD{T~em)+a&RM9sKbJNF}aZLF;Yb9nwufE`UY*(g8!XI~`Ao7Bg$@twq zUCt2d-``pJ0&v=Z*XNwQFa^(#l^rpa)MtLm8R8!h5D;zfKZiYyR!api*HgLu&dnAL z0yYh~l)I= z!E2IQVGfGfz~j(fyCwp55n+h0U%#^JHBkXg;HBbE8Qtw-V!c0*m2S{!I_0c{G}V0U z%+hRssRqP%djJC@oYnQSZP8a}W@Zp6BcHCWCgy#q5poZRR{(0I@1~fJPTUH=Rs?*b zq@?5vUwp$G=6oRdS76A=tDUsvv4jkQM=#(*$_Uq_}Ub-$g?wQ>oZt4Y-HQGw09K0jq$R zvT_d+MF=WVmrzq69%zR;xALF0FdRS`Ip9v(y#Mfl7L49mIvJ|UHhdTJykOiGAR-1g zmg&^@;R6|9_GEw}YnNmJjWP25aqa3=cEgSv@S!`1GxqiMwVv*X;_9;%+1Xr??Jsen z##DNf%K|1_o$01gR8&MxPYAaj{k6Ot89cDJieR3kz$T)gl;}L`ovHTp+yF}-dmNN~ zj%!zUkO0@J-&d6Hd4}#I^hzIKA#hL4P;eu4it76HNubRXz&XGcV!M0y?gUiol?SlX z=d$aB$@Tn>03_^mAV)uEW;PeuEtyX%c>;0+FsIv&>R~|chLM)Q;xF)=fYj+!#vc3& zS}GfR#7J8A;id?9x?mxfk`>Z63IY3P9iN^cW-r`nRmq=DRuqd$q!$DBpKOlzp-O}m zg@R?&LGuyusXRK_I7BYC1CNtZNwKmGPN%A`PhG=#BNl=+Dh$H0P<|o{0$hXhw3vm3 zg~`s$_#=d5mwl^IAht5ot+=%|m+K9JVo8Kpz^@`Y42zf;2y>>~N=rS!(B(7s4>Qn_)A6W$||eFb_^mC9kjBf@Rj}4oyr<#8TPVtQ{4k;pJ5ZPjH-&@W&z}A1o6A zC>kz)|9&$ptklYx>FE}rE2icE5@D$hA335!R{(aIX`*eW(q)Zy%m(;906{r`qkwO} z4&aoUhDKMZvbq`r__FO>Oh;!Y-Q~-|7z_q+N(1E{R?{6ft;T?WNX1k61aUodj9^uop%W`d(IsSOz`ucjIa|3=8@VFrm0fPW3GeCjW zh9S)WZUhAd5d&&(zbR08S5y=#F$4tj*`%#dUw7#zx9Og_Iy={p#~>A1rZ!eoOfctS z=eMK?#3Z0iy@kQ=5IVMl%}Ye2)xm=YZQfd$n-`4nLo^6snZM-}Q|<0d>HTlh?}CC7 zVYduL84uuTYuhsrs@((7A(tdaM=s>duQIj*TrMnluzs^VI{$*NQmJ8w90eaX{__fC z%4K3da|{W=9#24|7i_7`&8c(_t4TScOWab3YJ%{O(!K-$lsFdPk7=bYEJUBswbC#i z3sG1yHcZTT9Ao~6_G9MLCr?HKo2@k!ikKBx#_TS*?ZR2!oLU=_YpaGLK^6S^T5%;TDKS&Qj zlZ4FscC81$9fM3&CcfIr0e(&xf(N|?R!}a(Kh?tC-d-Qha~7BmJV(KX!>S<)e^qh} z8xYX7KX4!4y@7H)2E~Ln&Tz_DDqlO`jIj|iUeGPW)Ruewzs=? zfGLsx^!U~e@TpxHnjQ}$1XJOXfrn7V#Pj%B`WQdIGhy&ax3Q(gMH$$-me9cBt=U2p zlRS9vw$FBm2vEGX0}f1XGK4)!o)7kxl56O<6L-vmASy((6>2fW{Q*WxImW%p`}FOe z$uNVBUsfjU1yMcxv<(5jECntIrNkj#Q1L09|9S$L(?>ZY_Gjwr>k+ZEQqMPs*feYm z07?TG4|=Ic*SH5k9|79wL7z$Z#{hNt1qUakm*SKG(x%3!m~c>lodUANFLmP@#0o&) z8;0REQ@@-akOBkNG6?;|QbCHKE#c4ndPukp1B3y`Jq19meszP1^bPpQcp$L=SEC?g ztvAT}-@vQ3+_}-~aDZGI#>JDAi5@7SsW)QahCf0YBM}TIiNF}jn!yM{W~zRZ@yj%7 zBw%d=u;~zisN^(6DE?C&U;bOTV*}#9Y)HRcq9Xut!XD0w`&DRWW7BZ9;b;*hB*6s0 zYT5|QLJa0Jy1ckp^I4X)5aDA10X2Z5)fpW4O^&4>B;2V1U&je#`euU* zE*ip&NB#DDrHl^D=K%pbg(+}TinxTtNT=Jzw~^XFpt(8W9Sy&kGEsnd zPhnvmCnjzKAg7H41uw+1jC*+jW8Z_o=pVciV*60}DYQYr)3*c1dI)UE9!;iIbrK%0 z8t2{{nKJaH=3ol#AqL1c>|{lZq$6;>(2iDEx&P=18F0Zj0Oi3P8UWxD75gIe4y-pz z$@=XVM~F~v_w7F0ZHONI`RR!Whe@C62~x&ioo;qNA%Tq67eJqX%n3IT(SwML2M>iH z38HLOi)^J)N)F<%jOel8#gIEio7XUWR5U%0wu=`(ip0uY0mn}B#wMiXYR=O2%v z{LmD+zqcnW%(BAHIRrLykg`cI@j0k*ijSoFYC?QQEaZf@byX2-VFgS=c z{PG%aIc9xWZ$#V(dVugDP}Cn`X-z?t&(14T5)dAqhM+L;1M-g^eE{?1^*d#SQc2gq zB_T=!(kd}9uJoLoZII|f>=U9`5g`I3F1S!OUOYioMi52#Ll-%qPoHl1vuLP;U-%U; zU>M-%7X!|g4x|O}0YG!?|M);O;SUcqtdPVJ*jaAAz^e5(NC)&RV;l!yVi0=?fC%kx z4#SS0zmrxZVCf=&uSCiKl=g%gY2!fl-Q<-CCp~vA9x~pbltjTwl63T|#`pMPpa_1^ zc<`MCh%8{T_<=1Lhu{Wb-etIS1MqI&_%pj2a2Ci>RW&q7_B*YGfkV)@#D}x4E-zuw zLt(+d-5-(Q5g0%W00%CmA#!NUSbT{>$sjf{pgv{bexxkukEih7i{C-O+4c9kcPGIh zOu@=R7#u*Bwq=Ye6R@XX6PjSpBW@GrS-@M0R^n}}EtQ`e=ePcT4P?q8s zsSEg*jaMNp6R2>wqe%tg(U!eHM8l=`_W*Umxt|J1(>(*6fOG&Q+l%)Bh~J0c7Jbwd zvi4r&d%Q>-6;Es6mi@ zL<#|js`R$k7hpxO@?O~c4q@*o!k1LlhMWeFb4RELh^Q4Psh12=xs^2|VRNdtZXn3DuUsQ_{pQ89c*vdRoZ4IhDmLv%Ns!bOUoDFDWeh&O^1?`Ws) z@jl=JVSZ#FnB;(@AH>(Rz(Awy16TkrZ#+*s-+|IM`LYmRXFdM%2bi0?7Nb8JA=8wt zRecJX3%FV-6hPC5WRt|~M~t4Ho-(i{IZgUfK)!+qdX#YwcU;6o58?ei84S~!JjfYP zq7>OexL_&)k#1jK9z$e4nmHI<#GRR&ncd_~Vr6B8Xgd*v9{MmGo<2T`0B3>1JA{@7 zuxwnsSN8_bHHwyCK^U)eN`REWBs6k!v6cm8ZV*t#%B3oEK)6K@u@e|o-^@%4Z{3<7 zKUyHYi_|kOj*{rZScr>@%OI~0_X;IKas&3FsP`CthN~Fh4L>0X!fyI&S97j@=rjG& zu1~4SYzXIHO%ox8> zH~(nrkzgbE?nH3+=Ha`*&kStiOpD?jejYk(JWv$wbCg7&J-^<{sorYom(SV|@0y*A zq~OLLISHu208~`U`R1{L>%Rz;QWX&9x_T9vXDM)?XD?hx0t5|VY~hb3HE_0-K9q=$ zu!1!S_Fob>C6ojL+7^In9k15Ads@|xb;X?l@Y?#v!tY;7qO_RtCKf2PD7mP z;NVjXHX~lTnbACu|Og$FwlYc=H})?wh8e}hYua1rJ}lnR4E#zB>-c?tPJq8 zCr|O@GJPmyG~u;p$;qAS1R*R0=#T}bnjwP~|B!{5nKiyO2M*95eR3K9+sCI~y5axL z6>}2LeH}P({T3PCgCF>C=$}2<5vnHqjYN?D0D)q+39_Y_KQO(=w(fey#02 zkE#vx#Q1*AdJ5Uqp4XBdstBhYnZ-rwt7B(L~q z9t)_oWlalmFQA7kb_XLaJQ2A^}Yb9A$+3;hl|7ExH#HC*qIFyrBgEs$?_;X0fRU)HRT{{p;`-& zZFpirX?6tPCzTe{=&9&nW!9aN}{AM4bf(aE8yw^QfjC^(F%b}qm^yNv= zM$i_X$ z*N+^VoJ<1AhazM^d7U=1ik(0pfgle6_2HSBRH!lvRH+?x+Aaj>X8^2$-~}nvK;Ty| zZ$lO?=%kEH^lZ`6L-&_2pZ$!y1w>}GL_bhgJy$SrVDM8eqMadO+6upcq}>gqC4)Qn za|RL;67VSZfX+g{m4|kDEcgJ7fbY@P)n$P72u6Ps)Q7F5 zE-nzaNSwa@Aa7u63vvk5ducH$B=G~N6$nv#r~ z!(S&)KGxFF>FQ=t&3kMKf;dwP3%1qiPEE2`PuL+gnqOPv0^!rl4kR@VZ0+pMW8VS{ zxd2876iPR!&I82BT7$fOmX8e(mmkacL`O}QO zHxicLK(aHSDxiU6kSSnIQAlotEOT^ZWSO&s(&!R2n3#h9*<2t9qVn?dp~3LOn(9w= z?dHve;He`P_Z~c858c+?+Xt?bh7bMiAFPA3#>POdFK{y$@|X~GCc}t>uqQjWu%JL6 zRPBgT16nx_LMVj6>=&zPuT8Ul8x=#~TMJ0rI+KYt>llA;-) zCSowB+K}Q9g3uZq8z_JzA_Ubykc=Wb!0*(U&7C=y^a$t8>L)gX@CmdH5Ifp5(u+mu z7}tXY008yo+Cf~7l$0omS>6NBBjbXW1Jk2V!k`jbwa^Zl1GL-@gpKiBJBZeap!!Bn zJ65n23(DRW7XwyAX=JD~90#a> z9Ef66J_PKXCJEQtTtCZ%IoVo2fZH+CB8aX9mVb)SGcyM~Ap^;&1-C}WY z-aEUyGwlOC{^vCF3JYtYuGc_3T#tF-=F>Z`a}R02ZZ$eXNolfsp5OKp+(0>TBao_i zq2gk}Oink7DaRpzMG6I!i-yEPV;>E0$yhlsPgC^{lPz=taLX*pJAj@)6{bAUt^@=Y z7_8IUVEWD`^|g^~1^nitrIgp#jkM&s1GHU;;MO#o!1jE(;>;50tEPYT zx)F5&E}u+*ggy)>{fMTqAnvnqWYd_@0g?#VAP-UQA3Ey0iEN4jSx8OM7y+Go-I#i>&ZxvNF{7(>w zh=@RlYC;D<$fnmS5T)Ge!Ua^2&^O$Ek`Cp@knRp;#(X@pqUFC@euqQ9qUj+*1lYb zsVYKwET%Q#;KoI@0!uS87vLdZ!5aIElvD@)64!#FzeZa?IB&tUfg3>ozq=%~tN8%f z`FC2Of=&ew45+KY9Wb^AM{dG^|7*A44UtLo=jWBXKuqmH--4(qqH-wE<$aYSD=V@Xaan^YaGYHbA9-(v=nhH?TfH*>w>0 zR<4Bd+sVL+N|KHD#s`AaGw9fimR3x$iL-MF%&vU2C=rk({S7c_EpB-tzKNEMG?C+4#v9YmfJVkjx>bT(u9wHxj@I1^d z!4k5KDZq?e>biRM>T;jM9Eu=`XI!?r`_Ejspd2)Uhk|k{FX35`zC!du-xwYcFlq`O zFb*gJIRz#Bt6C|1jzu>C9NjK(0?8!@$ZVl-jd%h+wk^B#CxP??I=lm>6sHpbiD{G- z^q2c<@6)b4Cy#uk1BpMiqGG#2xJgsYYrx<#ug~tk-sPp6sVDINQ(*NJ1rW$aR}fMcC?KVJc# zX9tfdYi5=q5&{RB01$-kH%)j&lk0!VfaPfblto8$YJ#1xWwf;RQxnkDw_ z;FQfBe_0?G5>( z!D$((gXWVh@`dkiD5-U%D3d$Q@5n%HIG#d0vH2jq=A|2!eKFjD<8s`K%v#~Y+=`EA zSwN-{=yrVZ%i8_0eL4avjQy>0=+@=Ewh4c~_TR}4-0|mh8RDf*)0mmwCuWik!B`zI zu2?oHkx{?p_R4x1!h8HM8rx0(v$j&477d!=cD$F~ASpK)Wyz6fQ@EL}moBw?z$t5x zZ)YW{^6amnu!FfZ* zkXHFvSYhp?IZuSM?IwHgc?As{t`b?cQk9f!zFdY+qC`Yf9ZHfbw@&w2^f*7mBcaT;Aoe)9&m z*1UFRQse%Xb9GDdHPQQvt={Dxmi%jeuO2$c^AS_ZEN%XUjma*|G59&U)sHE zBD5G&9OtXsFkV9EJ3Ka%3RFNg(@>;NDo%al@ZrtJ({o{;#P^$e9h$0~@nn3XLGWE6 zfSCulH(&pFL50zD=#Nih*)e&I+j8MaN|R6-FE?wTO?s$?X(}R`(z3{)x9cL@u(P|5 zy}o=gfwX}?DwZx}=1C+~tS)(TGRfPv)5V+jvFc>}ss!;RDK4o>Q66(MiKHtR?_tds z>nHQwL{@srY`3y|)t%KDa!pb{xZ@j?@DYd>Kjq0NcB2OFghsW3?ia!sPvSx~0!?1^ zm0%%43S*K#sPzozaGTDHQ>&Myu*|9jR6;B!VNBd}rl}EuxfRRPxqgw3^QX_xiEV!JIvzVst`mFjlOoDsI~i*|zix(+8BYIN%y6NVh@p{m zTt$TG+EwzdkFWzD_9;p1jxrc^m5O;}W?KDPmktwfQNMyaNbGd&_GHzfMn93QcKC9N z=4uWvE_>M6!fM=x+VBSpBkj!0zEN_RCcKjvCKyPAIX=I@WR{kYmv;wTkHPC47e=Lo z>qC^y7f<+FCMMhqn>CD^D0L9ODeWb`Oq4jG@v*M9R|GpeFu9t{xn|I2NLInofdA4D zM0^C&`YV4J+{Zfilb;mI>&GrLa=J>%p5Lbcbj(qokKworvoXe7b zMadQkkzv2|QSY6PYk@3Gtt--2w?}3Mq7GkdPgcoP@Jy0b6q@Z$rCSmVOpgy-JW+43 z%OW;Yi1YM(!?8FgyW(+$gqD>`?hU6&i%{v7GQm^c@w8W742uf*K}e!TJMI3$wUM!r z)S!Omwy`M%y0S_|GxFK@5rKiYMQyC3Kv9}qs*gu}pPgk&MX`_?NzqC$Rds*2Sh~0E zy}nrr`@!Da)&cipHJi`8sV2h5BbaKXb8HuA4ZPAuq1< z!mB2QR%eoAjh|!qJRyHi&+yz#v`rvWW6;uAOY~iXk&l|UQhnKQP4AbENvoPS+SEEk11R) zijn=znm8~L8|dMUedXO0WHB~8G(i{W;xeXTrz|v*YEFrl%alz6t}0I}V%+DkJ{S)I-{t-Aiadgmf4JKvJ+1&=B9+@bj^ zb=#{3Vf78kRg3tZKJd74^6cnLdTC1xi(odTqzbp9@0g=??-b{3Z|phTO4Mg_NeDe9 zwpPdM+%5doDyHzC`|)*j`j5vHKGt*5&ri6Y<;1-b;N#Hl8=LI>oc(-TU!j8^|D62U zCzt8#<_r1w3isz*_8Q^KUWD zO&m?_%8(ypQI*=_yO69I{fydjmN~h$HA2t@mr3HMXsSNP&spCWfX&fx+1~t~fT_-lU+bzwwc*n7~y`jX+?!A{G}*X+76cU6kqh z+Au*ntT54GNns>wAH>NM^q>E;aK6hb%WRIBsjH|lYNsRdQk`Q;cIX1{dHDXI%PBkTBb*}Lxj-WM5J&#In zF;uCB`+cl7i-i2t5zx+!9VL?84smH4{?@=gJzKxHwwnK19-pU3;O&6n6Yjq-*_IN^ zIo%mM&6dBqx^b~R(c8t9x4%;r%bsK7;v}?T<#2-rbBE6Z`?6{af#igTsiyg-&sfDIFe2#U6civTAn55jZ2qp-PFEM zxCle@WxlZRnO=>K)6SC5h2B^X)`waXV+32vmbw&2k9ApO?) z<_43ZYm_9CTiGsZ-03Y#Ehh@LpUsmL+9YU9k$G60r};dyTN<=6c~s5$^}Rdy$agG$ z*m_vp{r=6QQ8=Tg zm!9YU2tC~u@LqkANBI@kJBWEW_;UHOH0bG@dVue$b%Z5;d^t7hY`2VMMncsmm`N+qXG1C$8ER`;cDvYVdGc?M;hEVe znm#8La`g|9DL$jBHr^N!G5XR^QW0s&ae#)eU?y8)663t>?`Z1aihnG@yYjp4^hJ@B z?kYI=P&6u@-i>)5Hrtwi!rgx2jpWLRkI+3V;4@LH$vB}3fy=d}8vD`m<0GWyHwbkl zD1K8q#u{(Feh8y1HVpI_RqV4OageTaZuZ8cwV^TvkfpTu-4kAiq; zvX)F=&Hh;|k-NK9c*sPjL^IwT{Px8BBQy0ccCi%Viq5y(K9)}=8VmpO>ftn&68`!a z&&}YtwvfsQ%E0@d-eug|eNti6uuizB0E2}Evlr?+xcZm`RU}#<=9Ze$;P# zY-k{nm?*kd4&9s#WhSl4ZT_>#_y2U3{{Q@gDDSWIV8`6MPh%l9vv#lP2M#2~(4Rgc z2%=eV|7V{8YlspEzygP<9j6z5!@M@%xyKino=9MQ@(UE!5S7#agi~hARo8rI_ma){ zFOpN)zf1qlRKw>yn@&eJB=JzGgLm)V z)$F6+Nad@9cVbTv6I=helLqo*Ywi-L{{H75c=uBC7b4b>b`okrac$+{Q)&LPc&P;v zl6-99c=l5m&h%~72_xlT-M$_;exOc|laQnjC*c=D64zH@ z?(aag%B!k6wpxH70CLyR|Wz>Ks>(Zcl zn%UU`S2+Mt%#%;Q*Iw7j*T9)sfjqIgyj%osCzp%)cZ}M|$cVNJV@GqdtkdSQnfYCO zM_U8yIuHB7%?AOQ_k|C`i76jMw3aw+O@e%Z*I_j!^ZxC_^Kc>pj-Bjpx3ZR`2V`WI zE=^CLlKKKi;QjpkhNoGjZh^G+EG6aKKBsge8@8mlO!}BT-Dk}h#Eh*kXb2_Ee9AG; zSRA&uTCR7RD*m99mi}V00;t z?}8VS-yA*{ruM2i*63a{mhOH_jPB`;BOAPh_!IoD{ddUzzSv}JTX?aco|aR~EW^)Y zU+Paw^tDWX&~es|$yqytc-vHXgJ$V0#p$%w=q`o=hpgg;b3>)?>U!g5OXt$uS(xQE zeJ0E^@@Zk{kCX~NGFCSZUhCo4Hm+?M_YqANH6`-kIYmQ7PjgMIbZgK?nOiUF_~P;9 zGCsI-W>iQ>!+pbjQXnHYpfp>RP9tu-)hS&Z>pdrs#-h%NLM^p~#+@GD@;?V9LJGBJSZV7(=YNy`yUR?K6eJKK%qVpgN; z=ZpE4)yR9zKRzP<J)N6%F_X{n{ujv-t$0`#U$GTMP%A$ULBRV}ZR7b(5 z5+c|ib9#bi+Gr$SL9NotUs`MYm*mk_SS77S%4?!RxADCb|I+<3InN@#p3ctr^JI34 zE=ev`^`uU#+A*8N@!RiZ&c%0is|l#KPHm`ve#AWImznqI%wMOYp0ge1bz07H{KKgw zPel_pkgU3O_)fv_M)hLyQ2eVfNT3Kf;eLvJk^X{>xu)VXUe$KZ)6?rSgeY28*-5?g z*7OBZF#-&0eg!!UswvEqn{tXrBBk$Vb$5emzf*O$>V z8@L33O5urfl%U=EX%CCTx+kbGG)P;s#Zx8TD3xj&34AgAs6Zr9 zBR;8ooVcdCsxc~pB(b}>*Li7#QqWB>sr_^K4mTf@N|Vd&e2o(Z6xR7T0wwJv39SCh z7Fdk>LM7&P=>%e!&yQHz?I#9fYc1r<_S2_w)YF71gNqi6)EG{i5LH#ROU^}q6t(cC zW#7Kw-mz6^pQmm>Tb;Kp=}RFoEZ=-zh1B>xRV?GZ_f+5cO=rYjQdf_Teqb7lC{k4s zE_OZT#Bx@hcwAK#x5V%$xQnW~w{|B->-UD)b=Z)eACqIg+UG%r>xhOBaujr7H zKY2vKOo1zTQ(`jydFJ8Zm32n)PWwAFO%I=qEcSDq45H8Gi6ml7kgH~LBBWye`>|G-|M0BEN?^!`-j}Y&Aa5<_=(ueHH=zH!%r6x3;24}8N;^V+fxGZ+^5U^Y>1byE zwH?=X**=4jbAyV;`$gyE;40t$TLrM6cnV!V({js37ESZ(2I7(B0Xd93J>i4UcM0E) zz5b0t4kSP!8nTWh7YF*uSz35B@@-%K9Jy}b-c~sv_x)?xJJBniMSM-afx!X!Q!FR< zzwy6U0z6Op$*7t$)-gy%uf3N($$kglW_d3_e!^%=@CI5uWICEvlsMJI6>HER>{KxI zrEvJ-u*zSt0m&DO=O*KloQ(HA6ghd7u;}nNJdQEhEXX3#+gr)YhBDN*WBRSW8+F3&+?iA4AB|5JB--tcYA4Nz zmwtS6cXGo}l4-Zwd`3M(0ImKgC#aPxpNNiMv5v^~x49~5eN`ikU>L|fafel zs_Ul%Gwa44I)tH77?tatHrE2ilzXSg@LY}*=}WgqOi)KXWtvzh+ zYjsHPDxJzpWA!8?-=2sx6Ep=H8($ZoKw>IDujH=pIXXbFk}M9&xw$23*p{u0 zzTophvKkUNaWo)-rHYCA1_CkJqz{!7m^6&c{v^m{?6?};A@$?$4+d@TvxV+^6>v7a zpcG8_mGJodtEi3BW3{8k&Fis4V8CoPoOk1sbt>vwY?@1ClW(O?^R0;&8tZ|6P79wVbb(^`SfbeqH1!;P7i*!)qMTldHC0) z>c8L`(SL($Ea6PbYzq%{gM>nHz{>PM02RbQ1q=hsXVqwBCI?$&+DA{1M_2S-U@#I1 z2KznTPTe3Gpg}IgG2JuIH8rv1A$Z*TcmNghc49dc(sG7=0)P5ZB`3?zyvmhrbiX>I zlpo)ctv2w~O1eG1pvy;Oj?ka#Ztv-l`Fp0Y$1qJ9y_NJ@src5?pXMVlkuJGaVs2$) z^=v6+f(`>agt0yU^QtaLI?j%soAmV%GUHY>%@M|0UfsD9q;J4g6lTrX8`Y97+X`z2 z7V7xV29K7R3FFd@MUn_%%2orV2ItVQ0Hz6{sGf8P1qxMn-BMUvuVXlF>sL+cWK(M7 zkG8u-Qz@kc8WdYDG>zobKmOewsL*60oko$?E~w`>r@Fzjfu#ad!JNiSETn!=U7x&N z`HI)=8ab8!2>VF>6YOK4+GS<2W?rfvI8_{*ceM?#W{v#(iS1;Pd}#m`9qrY7(Kw!( z0aY{UjN?nkM@A9l0Iq2os@TTR4BpWTe>GC|4tm31%pG*>KQw z7SJ#GWIEy5jD6Lnzk8UQl$7l|*jHH)$edMUH0RC*lM_rPDD8nI0VtDG-kr2AlTos} zwN}#;uo1#SoF7n7H(3Ml7(xm0(C%F02*9rTE!omRmBrF$f9pa0-EMrkut2gDNVPLx zO;U5Ons&wNKLvwSCI3q>$avWTAEwhXKrkx$PoPR-x{=W5V&>4o$h#~|GBtsLrTq=x zM?V&d_QsaDOdGgdG*-$c?^3uQ$0(hNs~aAeG855Uj%d*UOVG-uh(OQG3_0aw!mV_o zf~cD&qIT)6b&hy%bU^#bO~P`$^qYUbdE<)?EUD$D!z~qPKL-#fD_APYtIuYr+O6AG z^^Q%lRIf9++qv^%?e9Qzd)$jyzOI0SA@b`*-@vf=Yu!FoqdKzBe5LGKm(BGBMv4`< zmhyWiuOE6K1;=gS43kS{KPdlD4JDq1hEsdHq*dt@A3^e!(vw0MnWoHk5~bMM!nMiV zR2Gc+nwjDYVj}vatdPOe!_lH@^mAJW(*a^l*8H}8(ti>pXC!na$gCv)r5gR50q+>f zQe9yl4a{6RH9GLWah-gVdkjCs|I0H``&{AtZf9Y`v*aWZqq4cg<*UzK55@ld@!NCH zj~xy^A1_Bn-i#yTxS)B8qw(2dkik=Sc}>jQ(XN}T26G(xhi*yB3EvSueCOTOyWIUG zX*4|NPTcD>Z(Dl%#dI!vRDDT1{F+c-#%l0^!%=(rRoCZId!(`OE&NRak(d9IcXLWB zO&+M##HDNm30PStYN7!xyP)SqPmnG&u_+^l{@azK?SkwPK@c{_dY92>O{Eo%>I!Ws zS$Ap&%7{HVI{_-x#B1=J@IdNK?MQ3_u3O7jVmlO#o5Gcgf1|JX-)`1_u~+|heOxYp z_rP~H(8RX1v@lvZf(X_Q9c{UU>O0dj>;$85EQN5EP%C??ztjaZtny=$*h>~aZd!S# zX4dlt-P9C)y;tDBg|bFiq0o)ssPink-*mcUYgVtfU`78PA~dB4(H})f1lU0gAHiol z|LaTg6%?Kbo6!;|&7gz2PHVkuY!Qba$;Icq6MNgT{k7p-@{xoEjy?OHNj}YRiJ>qH}~M*t$PLXBXMv)=_ynLH_C$| zI4LP9wjRn;v2RUhN*UBFDVQ_;%wb#H+CuxjaovfpPOF` z$mla4e|b)IzvL>v?ffrbUH!Nbb0EAE1DBlUbQ>31f}pt?oBk&W*^T`>PA6I5ZdB-M z0O+8pFsN`V)QI%Q-;KH>k2b~R$2ZLMR)-WaplH+Z;2RcZD?)|$aK@Vk4%aYQp)??E z7DpT*|G>-BO?i6Fa`IfXR%aC4;A}Ae!m`lPa%O`+@A2fa9}{yUuPz$zZBOCXutOeW zLrwcAKbPZ<2J#+BE-R%dU{RUNQB2hQ$l<)4(6f+xpVeXXt;}9Mt@njNyW6U7SVxB6wq4q)R?EHVB1W z#4U&X_`sRnB8?hXVhLA?Gen0w@V2YY*FK0?e3%%Ge(yK|Yc5e=Umpa{OB+LQn3hPd z#Bt9@`N?P}5=dQ0_qh}v!m{C1QN(_Arftfv&tWDo#xT*$LhwUSX~gcExAv=vjyHCS z4U1ZHI+y0ZblSHIOucm&@((osp07K)@>ls{Vs3tQprq4FTNG0*8=E9<%$80Mrr>cma+qQu4CxL`fkZm?d(V&zl2FBJ=J z={3JyyOG_%g4pNBeIA{8igS)u-GZGCW$)l(@ZpcmlJ)a66p+pw5TnuYO?4Pk(T&KJ z8AbRxfyB@4@baKgL{)4Sv)1U3&C8O#PGY$Jj2DX< zTs=XvDn`D6yDJT$5D4h?C0VWV435ijJ^P?bSwu7Dw3cwmtY(5|`y1tTrO&Ucw`BY4 z8cLmQD|{&Y(=>V=t1n3kg|i)9lknB4%WS^o5o?#MU^nnuY;g?kxSxAS77;BeL^EaG zdD*07RXh$NZsM>)pd9NlpTF+SSj)MZvOhL!8yfD(TfNv`_4dfDS8ThFy|kEm+Puk` zul4-ennsu}-D#R(uH2poqTCL84D0w-{K$0l)>zF_eKqQ{-V*5s`UY+IhQ&?3Cg&Yy z47Q|yQ{B5i(`L3{GrjX!t$O};XtS(*@8!~xiSR7_Zf1=6xJ^l51bHh9*Y?8mX;a~T z%kdr#469zx2gCek-$)K>n$M}Y3ZO^3UUP%Ws1!(-=(^+oT^NcK3~)~ z567jKjOtmm(H;VUB0hXPYMhT!Zq3hKJ zVFfzvBE;!VLIxFEcFQ>-O-7mF*>@jfz%UwelmQ(e5V}zYdYKSKECfah8dH zyVaFZIrg=iGR0zTQ0&AxG+04=4sG!`C&h}rxMjtikKlyPU`IQaDvYbsGSOnnsH1pw zCB{q>_Lr|6D9hSuF3IZlRE~7>nUuU2l?c9$;Wf&tmTB*Hsp9gb^659Tzz!!DZ!vPj z8Te3Zd=J~M=;+d9QqF#uTW8;Pe$79vmvw4qB7bG^4XgcnW)SN_>H}v@!_oS1`S{_j z(mo0yWA{DCl)%z+{?Uz0ogeqGet|4I(dqg>=Y=XHG|W>9ZqX+vO>XOzC?rpto0g{J zK3Q|T;iQb!nz5b{vZ^QPb78?u_Pk!ta6d81QZ2hM$1mU7df%&XbP1^ZRRXmXPs74tb#8Qx<_Gxt#7TSKcvLt5RD9YL##@ zYc;H3*sot*S+{5K?wiGd&`!Od3DU22=M)V!oCY?gZ6EjZ4p9_rY_>3}hvsM~31L_# zJL8)Ta*Lf+?fo`dX{a?>@>X5Rrr^$*7)6X&a3@4hw=TF zME&pOoC@-+NqXxznZa9mj%kzDr_Wk;W@|N!H~wyN+j(4vh|+T7S6)pI+(z#CJiDP-E^?|)+M5*P^qpNM7wjg2;+tYCbrMsO>Q6}Wf0hdm#BJQEVW4lai0>8T zSZcwT#Of>z8s;{*S$US}nMl^;1!`>nD4p}HM}A!@f|lS!oYfX_r%P3a-BN4dk_TnP zhYS|4syyjy7JX)7y0a-H`jzsqUaNfSYkE2}&A(Ll*LMVp4$p8^jE-hZ;jAs(0u)P1HNPH!KMevSKy9bm)X!Bhn@bVGOu?MTym_UH<%5e>s%VEl|2?S{upd_n_h^li;L3C!*3$amI_b`IFt`(jhQXk^z7nU6&Re_7;V5! zu`O}xH?h-%?Z$88JZOe@$@yq8O(`SF8x=ugaZjSJZ}cWsj=U_+X<*Wf(|SX0T`ek* zTi>gw@&2`@dToAtZ^>=^lBLlGBf}9JTm)zTFjzQN|+Sb+()ZHCWrFb}r4&i1&pUQd?N{~+!? zprT5QUUMWufWhGc{jOZ?hx+l;O<3=F3k^hRQBEL_l6gxnuC|5s{ws12;az-XU zBX6tIJy;LWQ>J;*Xg*Pq)J`>}8ZPlV2MbI*moa*yE0kz~$kqEse{Xu1pWEuk390YN zpx5S_H;{e{oeRN_kF+EDyD~SLK25$Qwd9mh_&sbqx2L~W7B2X%pu}8lk(!bM_1ut< zkSjH{`91F$w5eN@VdRyVbq=ntKE2=UK148Y>6aK@BpPDhsz}8iOV_u^FJow4(&tJ{ zTOSB~?lpr`8Lla|Bkc>6!TvfYT~}G^LB{ti*7P$xk1zS~!KIglP!i13ASFZyJP{gr zjQ=*sui*4>OI91drs)zk-O62ax;I?_vF)>;=9jW&w0g`ND;Ry=6L^eW7A;pEHcUi2 z(Q#OIAMwd}4dDlGya%K>wAW|rN@fWbYO1;s1iK!t0r53$%g%xsHC*x7e2~!kD>=A+ zx#?AxDYbv4y^GA}dJNX;QQN28XBWy}Cb5w=t`lmyjA=M{k-GjT0rF3t`=i?k4K=ui zD}^q-z6kgVl-%;c9#c8`i>EGLY3Yv*mf4(VTv;Rd&}j~vUT0+_O&b+Yt?4aFu2)aY z=T-*#g(VsSnB8 zV|?fuU76YhnKjkelC{M;+>!~_v|Azw-AtDt4@MDG0&Vje*){^Iivg(}uR2gSslD!A z*kL=LNuF&MQfV(c4aUTM>iyd6lP9i?IW<99Lys{np!|{-=Ofn_8LKcq$nBF^SM}0c zL{QD|(H%LoR`)_Qti_Hr{;Bb8I{$4{#yL&nmj=?Y_Zk zN%Rh5M-A9x1gdCV60U|D9ZmyQQQ~MBN9worsaBdfDaM5~9=0x}*~Wx5jUX3KOth#0 zggo9qbEkd$+a;7^wP;}D(!t*J21%dIHn9jnhil$w(ni-q-*hZHgaUdE(nUTQlCuSyE*SSnos0FFKU{Tsw*GLIThj+Wwnc*j zQd7fDU8XHZBTvIB+|_inxGU8Z@9ulbsm|4EAS@&-X>%qiJ~5F~K$89tJ8WIWhTCWF zlbIK;fJLOfWt30Qbuo>-2Bdnfl1Ct6$!M+P*kjZzz_Ho%>_&zB`usE@+I3N5+yMQ1 zflcllo7hnEW(MD`FxN87ZdOACsIIcuVp$JfiC^4EW2TXtdhFy8NC$UO)76YAW!zr_ zh;n$`leT`(cj+_$kh=096+GtFJqS?{w_g{ta5b@1$o(1NAiTmId1NEOg`1@)Nw-tm zrBJhrPerl37v{}NWu)?ryj5=<^bt|#EDNcz_p1j;(VS{q%yECfKrpZ6Rur#r75#dd zE_s7RfEYfoIbDOGt;p8+tkj~06&PC(vu7xXowdj&v(@WA1F&CzNNh%7Zsr&iE2Xkr zPDZC$`i7#g?E=>^0<%Gbm^UihL%&C#dmJlX7<{!nzag1x3bY~zygK!#k#=FrN@ zgF(5jlEwzHana*uL+SbH8+D8buR^$iua-5rSuR!u{HApkQ|Z{}?hdM#St)xBUO*DZdT>IY?12C* zM7*Y6Fv(+WGRKdt(4YAK0{yzuL>} zIzFmn<1tD6Vk7xs?acWglU`RWTuxrLnis=wWrS>y&EyPZiK-c@Y8P89|7IP8#1EI% zxk^JKzxrnPNi()z7)>mdx-ewq{c#bKU!SMwT=ATFrS-mZ#zH%lFuIh)h6{^zPhHfB zORQIebP3qm+71Y*gpyQ9Bjo5({j<(_hXxk9t~sUM`tF9pwVnK@0+n zhUqG{+7E%p+LGYp2}m z@{wo_fp+8-Iksw6ZKFtZz?3$5HAJ(Gc{VN^QB;G6v)CDXq8LA-2zKj3zjobwVTd>x z?|XTglSBS0KXI%ZN)TjY2k$=P!x>y6J>E%aeRh4(OMP+b*=`P4Q^a=`7_W1pnv+p2 zh(DTLJ01IG({%z%SP`)<)GBirEwt=RMtQqZ9K=irlW0=-1+m#+A#7x1uUFS;bYreI zC#qd9$yOO=9XY$qZm7UX*p;cm4jpibmPd&_HU`1)6~m$-Mpx*{~?Y}5V@mknAd3%X48tM_F@)}MqC-qjrQ!D3NXyF}oiZ>j2gww>w9v%8sS&*aPGzU8n1AuPqd}2E>TE+93f+rBakx zUxQ(2H=mm>Z=+WYmg%N+f9X;)ZsSU$l#@Muc6pz3B_jQm{wE_2lIu`OQ&I5{8j)hG zHN)vs5|*twmNA5z*BSqy>D~OLaYFEQ7=l72f;2@Xw$JTsek9PUy1=laX4#wOzMYe7 z)wd%e-~GQ`Yxs|u$G;&j+;zvDLm>$VG9{l&E#5UR6HRZug#7TXe@>m+X+hw-!@J0zw9Npfb@>@znPsup8x+*!1G`I_N~eUx`Ig*GL8`ntUli#GYUeT zLcHn#jO|G)UF^p5ATv8?Bk^r-ew03jwmclKJupEAW6O-Qt`?rNMu$1T6K3;_hb<@ z1IBICTLd6=0-)f`iQHl(KJrBHOU2B55b zA?$Sp3I>>z$9wfGe6m&UsM!w%iWI_L;G7^Q#|3b!{@W)`w0XnCF^&EZ!>Y1O}D=Oa_N+$R z5j`)X*%{+%%<E0U4A(F zqHylGtNDcbeIatG*m#L8d7iCTB5I-#H5pD-DUuFh;?5>t@Epw(vL~@Lmb{&zja=MsNaP@tYWlOG7p47b_n?4_^q~pxFm5y&dW|i^G;|1Iuv2 z7He6;Wvl-4ai>qlkQj|(pZGXDbdk0rv)9Cjahf%FqGx|>mW1ba7)$6c_6gccZ<@yz zjz)Mbe|jM{ntH#v2u+prUKwI`ud1GSIB_5sUFT(CPIq6B$^P1rcl)OE&3?Bj|GF6I z{KfTQ)u6dg#ixg}Dg;Q#zeDPYSa#;6Mu)=TOoiqg=N22E`Nb4FotUsG_X|D}6#K5F zc}(`w!6lp(JMfaAJ~DhX|<5Lh7ao(hHEb!|blq!zwXX)77Wr zEA)+l##Q}v9DCPN4hD_dRB&7q%TMZfWMo;QHN;*szo#X84OL8jlzjlP9ahIWhX3jl zsJ0F)3uW)3WTrdz=;n_+>oHDN-(B{Qv@b>1Ajmebb5R|w$SlOs=Sq!@%ecaATb!!| zf9#|;Mm0o|>rhM4!N(@`*ikNP{dR@%10034zt*zM2KcOLy=x^lr`PV*GIniptywgl zo`_)Yn*GLyw%k+@8hv$1Y-7B|n&3J)(=pmbjjhiSO>AS+kFd_|*;AZguBuciBP#6~ zT_PuL(>>@*|82`k?JM3x7#uB=5v{O08M@$+p{;Mnfr1^%W63f+#-|hf)-@na%BU@q zdo4OorM5P-_ky;TQ7C_pOHIkaQ8N%3%kCmek^cKbex4o%<|;9UYlSAF2Nir5`Lde$ zSup0Y8{S#Q-7eNq>=9S)-Qx@h54V$W!Sn43ivN&$7F3bb_ZV+2{vffau1i|WNHw-k zxi@`rqsZ7$6xD2-WanJ=N6A$j6PBIk=WP+P)jD1tSsC-#y5Pe7L?i~EBiA)*RvBoL zdrNfZuw;zIqg>~M)nmMQv;>XKh=^(J$~WMIE^3zuSJR3kA$ zs*G4=VMlKQK}+jEmv%3yS1BpY($uD8oO?@#u%!{-Ktbe!GGfc z9s5^C$@&2CFA|myb@H;^^sDRA_F+FXw|kzpzqEZS#|0a&6ehy#HU3?kd`v(udL^5j zRHVA#XqcUK)y#tp@<5wQEfGQ(<>!5pyfOLXC9N4HxG38Vj;=~6DSoNNDOGasGP$y& zhA5u)daIFKC#XQ^AyY>pI_k1*-(|!M+}wA(O>C*${&e+tj7novMZdumb@5~;kFShg zG^5YvUCz=g<3gebRxY!OP~W_!ye?U-#w)~lZoKMHhi#oIb~c`(daL5CwnD`ABemti z1|Vh!u23JatEPhGmyQmI6$I4%{v#$O=7GMS|+Zr|fxNa(SoDkEl&t#J9R zhG=TTF-H53y&VslGHxO^U-6XKrRyrJk@?2%66%$THGSm1nBVgQtJ^p7ji!p3w7;b4 z;B&<@LfG9YsY!w+%TG0bM|g$qCd>#Adc0eDTbg}$>aX6JYze{jc8RG2dnzX>0Zq?WhxX4J%mNk(!J+_DzkaD*k)z$Rtw zSw`VmtwlZC9MY}v9OooS_pd?jOS!J-&y!qfuRGyecktZQ1zoSnyEX>gUszM{`#6a_ z7)2&$=%C@j){EAx(z{=elae_sr(Cx#HkUocNli3+5tMMF#5ypN5V z=s>FEsL#&ouo{&RYRR;$xcp$XCMxPkUroAl1Cwa)8z5};87*M3FS!JS5^nO#Tb)&{ z7T5cjbpkIx=sshZPt&8Ez$IHoC61R-DW1~b$+}(EZ66GMLtK#1>5=AFp{tx}-{}%v!J42A1QUf9vg+ikl!45Ys{RsHo zeT5zdbq9EWySb{n+Zl%FkZg!{P%u}D$W^P{Ft(RmyEU~|kBJa=UXflhb6pakHB|U` z6l_OJS}bN~aMnKoO)VqM>kj2S@-pD3&hXr<{}HJsx2yprsg#tJ*-_+6bPTx_SRZjz zN?zg==v};Kgc)w*U4Lj&G(UySz!aGm|JjpJ@?#)o^6B8=+3&Q|bzgozob8g7n?3Fw zm6(=hlS1XA7Br$OYD9h0WTlUcFbeF9rG+dV``(50G>3%}ELk#NWl4%w7g%gSg2 z6fuH;sHY}}vmSiywN)bF;PUF5O10FZ!~COucX7}J_LMgg%2EMhWyPM2a*S+vLro<_9;0jv6pQka2> zjf&DR>(~x9PdHA#g7;_r)=h{qri(dU$i|!$whwMP=-)8H zAkg@Uc;$-3>HImzO+Q4&LPG?n74CM*JWnXelC(G=k)4;>voYK^=ga5)7TjqnU$mXr zP$&XgQV!xGwWv_8neh0WA?A}bM>Vk5t>W|TQ|{yw_O%kE7tVAI4NEnfv>bq8&^7f* z1JSj+L&U~ogPOtMaNH;6>+_GhqNa+!m)Uf=l5)no%?n2(7b|m4c};!l8YwWuvaiK_ zH0GBc{vnC37aB(^;+%Rv3<;)df(GV7Zp zvAM$0;*R(=yFPT?Kp@KO3CE^2FV#yFmuD*FJ^k%qbL!A$#>!0bsA%yG+HiE zB|Me}IfgW1+bRr*Rp}88n^&ReRc70bNLkx7~<9%ej9f7cM> z32;U}?wG%zeR~Oe&9aYwBZzuJqv&euF)vheqIHxFX1P6=7UtahI{bf(hk83~TR*6} z&>M&}KOP+4RWxnr{Fq0|nT+(5;V z+Num={Fgef3@p>^TaP^^NenE?*&9P;0kx*oZ9%%8mxhs9+9+gpY02EeJ!dbX0N`Zx zy_rPx(XRwO{8MAH94)?w$@bXU3R$d1OL{d`FmFyQEBaV_BnHGpx@V8Jh1#kcMS=3w zCK~Hf<|d2s@tD%IA+0a14(dVdEc0D=Q_!rUv;y|3&I=?=pNL^M?}9-^ppc5^eLbw1 zK0dH2Zy|W>mH=8Y5!g;gxc~ z%^x4c9>Ns1p2|idri9FxtIU5glE3V@8bdqBtKR2+^_{7GjI(g$QO5d6I|1g6iB4 zSf2YmVvXKg;x#}j9P!g{#&k?sX1N57#M<`?4K?4#6;Ura41a+qAfTf7Z3UN`KRH*z z`&^l`XjE~IqcrMs!t*~8s1ssPnAphgR{=$|artiUT${)oWNzB*8)P*=08Fi+p*fse zWrUww`FfhKVNO(v2ybxXMZI|ZhoZ9t>8yG5PGg@R3&GmKNuZ+e3hI^0HLV}jsi!LaK?TkK$`&NO%HtiQ7zhc>(iaH=pepd&Z zB#QG!W6d_Zas~=8wLGLSy}+ePVps>;#!};ZC9((-vfPrnEbe=ZKUHY)@qqN3771Le z=!?=q8($>lGyUs%1_hfKJ8JyZpxw;gq-nn*3k^ixEeIG9V$YhZWoMcR61*cZ2w=0W z!A{oMMUp+a=rBP)>s3|ttCgN69})yeZAOR@4E+C~&zuJxXC>H*?{fx3hrdrcP>)2R zaHHX=xv7SiJ#wa0y1u}Y@@JGMavNiTJuXs&7ocr%`57-nXK zH@A&6BM-SWK;VoXx*$55K4?FJX#YUqh9|st3z>?McA(qtwhiL%ajt=-$AtmyWCYo| zsqCf;eX#)v$ugwq-6gZ-JjksbQA7ob0TK3vG1^&nv_3KnPsBJQL7U}J;s~NbNFv7H zrR#i5{^a_rE?BD_TDy%8^MBl}#A5m#w|lhca0Y4Q3BL;VvJu|lfY{O-4bhYa;VQ$R zxjY$x8DpLA9;?%Rle9^9iY+v+VMVh{m6#ydLfeN->9*#l7MSHiFFjfVX$Vb`Dx~5LPAVYB589WU0v!H7C#gY ztLqv@ID4i=Rr0a;ytm#oeS$MBjirwCTqni=3LQ;9-Y*aFc(Ywxf@rHwY_8r#*Q=a? zkMi$C}VQ%pFSM8 zIbdj|@A9Bfaiqz@pT&bhKAwqchOlb96%WPEo5cyPWZUjU%OcupO(Ua0tQcK=DAYe2 z$sbUbM|AO&%C_7UVsc}Y{b0$n!m)S0S>A*Nbamki@ND}=`K!lX+)AoQWDQ-ld6LM~ z?7Wu26O5SFyY3el_NDd&^o3d)%;nV+ThU(38p3JvL%Ao(GbFY~(9Z;j#RXkQdl5(x zA?9wrd)-?$kWzUhzyuk~y)xdjRa=D^5&Lk4M(K-;qvdsLca2V7wtbW381)#w zVTUn)t*lPXNfCRUt?*?BF1e%?$;m($%D<=`X*-~zm8{LS8pm?~6Hr|;`|e6y>r<1v zd$efzhi^8_bT@m#B(l}{E|uTe(ulOil>O2kE;#+Oef~X=m;tt^6h#DJySY3$|L*V$uJ_P!>6XbEFX%9dJ<6Y7=V*UWI7ae)mFHUh6>QFa`xrPzKsME!P{! zhI^oxtF1a<{=9g?KoyMb(Iam_Ju~??q^P0u3MsTqy*w8H8OAP)fRxnK+9tWvr5;3!e|q50;2pcT4qZcCJ7e1~bCEm}{>vY?9e*RQ z>GD{=jgZRru{g2l#}k8M z0)sWs7_>b9IXYXi06ta+2EeS6+h2f{8cuy^0zasY!S+mw@+UCqN2IOP(-kPMP}t_? z=F7nO!J>_91HgjsXZJr<0ve4eUBm2*7Nuo zR!pIuMd`^Wq;JpPD$h6?dGQXma$y4Bo=fd2+CTh+=F{_zelRyIj=?oU=4BC4roWwI z>j`%p&AOPW)wS0nIe?f>l9hg-6&Gp{y|--~smMVoYwRM^zNKIPl12V!$D?(aiL~{v z9i9J1vZegBUfG5;`cP5feVX8VVCx&9evD)A4o*ni=cnXiua z_;jpF72EtANczaoAybqH`_l}R0L)mYI^PJ3g7IjSME*7KK7 zKU)T~wRUQ{&m-81Fg*Q0v~)^}!tn>Rn!~dM9aLXr&YiA5I2P(FomWlAZimcspG6fp z@~P5zSB>-!o^Jj6)J>RRq@a?!?}rQ0V+5~s9A0U|89^dp$!EnxPc zLtFf_f$^_h#(Leh(mBRbNy@Fw0~M`*zIz{Q#S$-c{Qg31w62Heplhg$$%_NYL-B`L z#XN64>E4~?L6VRC(>^X<>m};uZF=^INfC-*8DJjTh%TyFLzLYD_ zdNbvLGn0+^o?RR%&yB^DnCJa=;kEK9uHHJyw;I>4U0q-hraf~AJ##UCgUNEB>5pIJ z)%x~^v}U_olW1Xo9L~Aom3>k(-41@g=4%b=Nzbnr#Dq18Hk%t$2e$bZWdHnBCwm*4 zy1H89M9SXDk>XOg7i-sWGT3gnhr^zk$g)+74TsBWNdgAnK89z%R*}crzru?4RQB5F z=U+=u#O$XnC%4holy6#-v|EDhPJn6uTC?g@hw^E*tKsJ2(-97U&?5MJ@3Z{CagA`7bBiiLH`-W$m z;|lSj!D)Wu_|}}ux#`-*EpqGE>#bY!Yz})aq_Wr02$@qZ{9K&LAf1%WI}+(VrRu&F zqWbBLD4#F_$8GG$uYcytE2!kJi}C+ygdL^~^IFJp;c-}_o2^N06{&`0KUjYb1wCH} zI)iGS2}ehS#=MpiK&4J;Yki#27PkB(8N#inC?ONgDIn^qP^=Ov@NvEHUPM$rWlHi` zf)O8ER`c4XZSc0g>ZZvX%f`9vVASQdl=Wf)Zl`cNu*bDo1 zFzk-KIOA8?lU*tJ#T%zP(Ao6#&B?xm2&z_Uqa00dasb4BXzhTSmm2m~!?L31hd`>E zpKgoxW?EpFfL5^wQ<|O3_B0E(Lk>|F*--~Om=Gf?JnVZg#O?$({sm>38kZz@8(Y%b zD?HgnzPv~sKe;@It)+lm)_>Ox~k>z}vKr6-nxHCQKna9wpor3$XZyMzqphePy z*jJ7Syw{C0&hZGMr7LmwTKMSfg7zH#E1mHY0y zmku7pWZJxaUEqEjyE_+iqXTRI;PZN+UY;$U9e9~kRdSh{qcFS8JsxjQJ1BorX?ntk zLq_BqaGgQ&=%(*fXFG5uFQN&_a&uZMhM|2a(&jJ05MBTmjU^Ddvb>o4XwPqMSvFQk#=l+mKU&uC~FNv;t6cnXeF6b?3 zG{&qJ$3~}3sdFY<=0+l`u5ie0vXaRbCid?P;kM{*C)FFKTeX!Y6>Z7mj=kqT%O0_^ zfRv2 zEn>Wi7O(0UU24^2j?*0;<7z_c>dZ(-#LjIv^Kjp1*#{&~TzAn^Pqo&H>xb8*I2JtR zwF!nnVa3I#+e2n_r^I8Ep-~GP@gp~96&xmGl8n!&c zYyXXecHB9CH1*;%KmZE-?JhzNCc~e1cepi7zFSMJQr?R%z_pJID0I4;{@-(wsy~-^ zk8qE@7pDV`-2RR8!ETR#g1rA-n6`bR|6h^cc_;eI*R67-=P0@J8pUUvZ4NzXlk0rXphvIL zddAin7e8zY07)+0x})v83+u?(_jkuF8t;JAVdH(ggbe{ZHU%fe)@UMU1mMr&>xjqI zZ!@nmwmPW242YJByK4KSwb+b>RT$S)u97qsm<&^39=CAi@=-EZF_OBQeVMmL6hcD( zCumD&wbSbX(L*^VIf3{h<+|*3sWy?@7W|cz5V7p}#m_W$R2yaku@3Ahk@mWpGc)!7 zJdj_+X?^8XH9p|An568HlL5>C zF6X@$oC~MZK7I=1;gd4#3A{!5TIIq0?p}z&R-ZqB-o|j|mlEBcvjvN|*IFIr=AkhD z+-f~>s`#KoamTcv$aGMLoudIBsm;W{(uvN1)8cbzp z!zi+FD&_w27#kcNnF=QQH8D2W7E?IBE%)WImxKzy0sKe+1kk7=^H}^l`o+eg83Gv*2lP4N5+3HM$J}?MeD5!dXfS#iWhj zSQR@@(+4XA~v~{Qm$(7#C<%c!i(-Aa0^&_=&8IOQl`J;S^F{hP- z!lH^3`nX78y?JHPi`$X$T^bzMn>wEqah7p4>K~?|1f<;2uHP2(vi9hnA+IfB~H=SDg`yC+=6q|)8 z1n@h()T(gQMvn_Md%@^h_ATOrh1~;b`>obqW^fFAd%A+owdv79VT0o0kf2I*ZGUi$ zDHhf(QEDx^(p$=qK?Y-t)%e_^5QIC$t)eaJAG`2OuXN@^gHQIINs92DAd0ZCXN4ZC z*jkde_r~O@zXhD82>hmbloCjqKx)(XSl`F=8q?+pvXN@wX!>k1Z*OHTi>T9gr<~UX zp2?|!ID=@%ST3&r;GHe34uc?KxKyf>-1f9_>-Ych<15D(s^h&6sjJI+-H`%cYe;>4 zl)N9&^N;`1cAP#8TyT$p|JJj|1^Pubh~68f6(-wMsf=A`&3B*GmNYPI%kfAr{>LBN zMj;5h9S+)(6g-Z8)EkZZ+T)LIX56x8k4k?;{40X~t())oKlLV`YubMuzU9>{Y*^s& zwcTv{raPXgz2D1V9KMloc}+q8?}39yq5pgmTb|t_%QevkbudAOi>>HPIpt7RFbOm@vKjh z7*Gl_BFxCZr~3_NSqlS!t~0H_uib1*76El1fBCLFqTYOEb+x<5@7lR5S9E|jQhRBt zZviOw@b&{`MZk=hPTEdzJUe7^X74GJyHiI6j1YF^RsrJr2R{A0>VW}c;`*%bKiBQw zy6$@)6*vJGX#<3J@6N#28UjrlOp`46=Q7C7?2YcS2?b)tbkg}8`QY>K3(;gNzxXT--n}~kktNU1I~f){e0=fZL1@L61IpGrKt>K7%yRyx#MEFycOE~I zRo}6rw+iTi-rc)@KO{1;*7mdem!2Mdg;3t6zkc19V_NeE(AYnKF*E;MorGLnXP(Xe z`|iarbo(*YRbE?`zC9Ln|dtZJ- zWCCH!2D-pA&}sh98uy=#Wq!HKOeD(P==8i1QXU}f8X8R0NK@CU=l@WE8&&)?%C4>Y^FAbNJ1XknPi_nK7YqNTNh|Y^Y|<5 zG*dNDzysR>+b4`!brQDg)sb+S$aLPOOMLeTx#`z0Uk>f=4hNI;^7nHHNvXT1CmvW| z;nu3a?cE5D&Un<}$?jaz-*WB+Q?+*; zK72S1#OHJQ+U-i-PtbLrY+8f0YWnbN_Agtj=Jx3K-5<)s>&;Ox;_{$ox}GLb`8ETC z>SdTJ1e1j{id|`WKcDkVAad=OFJErMwZ?to3Yl&*KM;n_YuBy?W3l(B&08-o%LZ`q z@ujHx+4=e4q@<*Dg-^B`cr7L5#?bGU+$ghYa8PRg%>h<%)1S*K%Xa4HuguM5J2O}B zgzRwX^yhuuh3F(0LJWI=Ow7;8xvU?y9d=Ib1T9ijRNSBc+i^i4O>b*Y*B5s9aYHHm z;?=EtpO&`ydGB&V7^sgt&eO-w-UJ1~`~;RHFe%)^!XdhUZ9VXjd$3y<|K2SI*Ba35 z!NEa<4i<>s3GmE1asj74{=EvvL2d4WjQ_^4uMo5bHuBp7wWBT5 zuwC$Ahk&Osi2ZXpBN74T!d?7N%NW_0E`)+6B*n=Eb{B1D6W;s^>iRM~`~>RhNBTRiOSxh*%LuV>pze z8pdzvv^sD3x0cWW>hCfUiPHRVV^3jGQFqaiVIfgbQKbk$V2L&XI?P~78@}qN>6n7N z^!Y=;tl4|k#sc`@-vi$=iT4w*0c@<0aepsB@vZG!?qQg`1HuWXSg1gN8~XF*%S99` zrFx6=T_*MVPrg1C0aVQf1_qZ+s}Jdi{k3bl7u{&47aANK4{WRBaHvcJo_8Log(<*! zAd-AQ)+DrT88P5zs*d%kpXh`be`9^FksBVy=-4yHZzH6zk%Emz<%PZt8atRQ<)pB&;BWIlGDF2yruCQm!-wWvwxN~RV#9` zoa`+`=nR@#RAdm+0&WqJ_McimT@KP-=KJc^Tjbi52TdO$fw>nRAa1~C(-8Qe5hwH0 z7kz&W7Q)LyB#HwCc1k4Um!)hV0k(v*dI*mQ6mYf(5y0aA+^|QCdri*F#(?hv16~_^ z;XYOEA&vx63#`=7Uc4ZI+5lPgKhJ$JdT&w1^z=hu+`a?OxtM1D8nM9I1S3uOifmi- zwnaBD%((aU_1@op`|i?71U2PWPxL#*$_gU6_Z5)RKY(jJCUZMlvn^B>!21cA*L|xA z;sTEE48=(vjpTQr&v=a#wG|_M<;qu|?~T-k2u9U|g?k3P!9wW=_w7>$_FqI-fDl$U zz1VH$190?m3J3^*(h9WXv54BWzB)?oToi8A2zxOYPgRpqE$nhfW9zRxz z_BW_#$NbMyU(-tFn zS{)?C1~}STuzppieI!XD#$Gee{;YAEs}~hYUxVW2e$?|PPo6joetxD%-PqV*LRA90 zH&ew$KcEioeuzd3u`~Dg_anT~BeZnzKb0Xo+EY4cyT!N0Rg=o*b>R+G);FH*J-Rn1 zQQyqj0H}HI!Sa7K7aq~DtK3S`bm>*wes&#i(>88{PUm!QErL-xjANkyg{?N6C$lyD zfvK<&8F0;(RdxG?k3jK3e`cXuz_ES@#@B%{Hx-FG%EE#-sd#=tQqlnEe~WEP3;X1}S@WYk z#uLK8Atv(rz_7drL6Pxkfh54A3*FX)t~8|6YR9kma{I$&`e6DU9~NL-EKfXVuQ3%I z@st9!im=1a0lNuUryN(B5%_0?j~_pB1HX1zha(a6bg*E6a z0PlqD`wj5gggmRmCr_pVowYJ@TM(_wcguIwhhMsGX=&+hj??<`jAD=U#(WE< zw=mo6R6!d+y36+~_oJFKO)BQW50MKRdtUkZX}Gx+z|lhj%$ThG^yw;CeNq%^BYWL%3=qijp;7@aByL10aw44e9`qHm zGjK-RSU1#5+qw46t{GJOuNR2)PI-j)Ik>~8!Z6AezcD<#UTbLx%}*Uq_Z-$p*AoVh zo(i7qq?luT8_{q}rpaYY3>BY0!%1#+R&l4>5XQ6u9=;$aC$})wr%x+^^Nxl9`VsG5 z@!}vg!XmZ_4u&3OW=@&vE4HS#r^C^gD3J^2&)=RSH)?~rE&zQ!1rTs3^9}b4e0)m4 z(G7Z#LupNcqzV8%m#N-}6kAZo&!0c@8uPn?2>{cR3a5|7F{?SqGVMLejbQR7z}|B? z4S#*H{akI0FsbJ}9K4pUn|*8N&Yj(vCQss{-rpH0!vJ62$jx&;^p&KO!dCHM(9yMf$vtOKjboGpAY67!Fm`*HnCg#sEy4n; zCn;;zU^%kW#YEStrwT@v3g76vV{8jr7A_VK{lUhTVeGTfmW7LsH5N-p#5GdVJ&VnI z*#rXvr4Yzb;y%|3?<^@1itMK2pi%6VaASc(C>0e^h^G9p9115b8lpe)swgXqz#xIU z1qB5s-oAZ{n29C`@Zz8);uh>hLLXS}{XsRRl0Y*C`IAZrsF@&m8UmOVDgvhWISz+Y z1&(u4R&R-{>2USiN^r=0GO+(b{XUzRH~{C*0WMqu_(>9h|H-9rRe%!gN5N&H`zs1o z4!NQ#!G8Hclmb+8=lJ;Qf7`$RGK63*Jw;R6U;>_JXJ^jNIZ$yit?E^D0K$LuJS;4% z31TaJ;~Z!s_|>C4TBd6eNs1BoZVR40d)Bed!UY;q{2WJWnUNRdjRL6pXHED;lWU2K zO-)VD+3`gDF+M$=${bLl6Ij0#^(4K3^r)yPWTElhY?ygf(!R^DkMUW7H)ILoQV~H^ z!8Sd=c2b(|-MjxBt?^l!X^Ct+3~IdvG+BZhy&k9LX1m|CP))R8vfS~P!=vl9C`6w{ zqhn(HLI3nIl@Mn2zH#b7FVMCN4R5%n8jru600#tyTMdj1AW~hDBLoUe%z`&+YHIX| z-lgz5ITnp<3u8?c6lOKBIPJ%cz21YQqp(KaD`vI>WrkmUkLTy0l$Et+etfm})bF0G zixxvHMCQ5^r4@y^CxAyE?D~qv!6X#oH;Rgjt$%!Z6-t&AyK*HN7O2Q^V*pzdO7{P1}7)e;ojE&mI&b$xOE$-I1CJ;5omw}qYxtG+mL_!_%K-Uf=5iO zi_-NM{7MJ(yggXy0|RPbGV>gKS&=hUZ~@oyu@+dr0422suu2>OL}{7-py_M~zoEFc zP`qQeg-biZm;ec#v6Q6PEM96GJ`5Ibq0e=|%nIi~wWzEsR|w;`Tw9z(be25@%p-yZ z&GNoO^N`cVsx!&&vtPGXhCvEgDN2_GPIJ6DAs3+V2Gw{|4N1^|<%Eny{~~&TDh-x+ zxft97Vi+U5!8XteY_+DGY^%o?rj;E%z57x!{Nx;-K7X#)XC>`2p~Jz>o(!XjU7djd z01`V=Ju+re`yhD4#v@4$y)XxSCs3X73Xrw^Lo@#F1^Df^->O^ffM_mM25^@CaRH-N zz<_Ch!EegR@c^O)?47%s6~uz%qySU!5|a`Wxy8lx!11)qj*u3b6$HlT8f6+6yh}?< zM@C1@bJK3T`U4yP^y$;$)V5G0R|MtWSxsm=n}@biYp`z~)bxCI6o*2n2Ec5H)vsOq ztBt6Q3f=tuA>eyb7T8`>kHKq|zJJ--hW(jzKZG3s&28ih0rV^Y9z|RNL_SeKUwF>K zf_%!Sy>~t$L<~K88w;pA!F2%Bk<|-{tf~Ok7KrlOQZ;$-wruXhze8S0p(^ln2bj~4 z&`|TEcR_1(vn?960L~?ne>J}tp$LXh3!1^=qXdrWeZ2Q^tcnR!u)zC6@j>Y1Oa zk%G0IT*?Yump)FPdU8-2W?05eb@C@6`%>_AO}V2&qGoc%IjIOBJ7LdZ&M`;|X@IE) zmld}-*~0@+3Ni^04c!m0MzS++M5jm*j{)Cd<7-2BEc9FUaQ}Z*kQ2^CL`3c&19rBIOgACFT@kRJ!*mgB+W zRyo&dH!=={1`)dsULfBOaS}Q;x3JI#d%%J6JuVPxa>8S_h6}wQhHfDb7?yeILL>)P zzqu|#NZ91_FC>*pzuGh-c1J3j0Yr-jB7b~lGU$gq7y>{6;IfA&Cen~p2^M*)1cfjw z&>XF``I(uf>_%z)fcKix#@bQ`z^Pm?l)TgiUyK1LzZyMepZmC729sDRTR+OKkq~W= zOTo1_&C&dBWm*8(13Y}9MHRIBn5vhX zIzp-i^=Kvf_~2c@*=l-t6cxElroMS|DCC0rEl?kBwfL6_^$=ssmAF9c-dFmhQDOtejkjoMJA1!y_Bz{{%$(XJ+d`Sfe=AZu3L z)uju=%$pExC~-%-ySt;X(b3WK^DQcPgF;6wd$%&k(LR|U3Y5|7FQ>b@e+=}yuNHU9 z-%ITH)GZ?;7Dq>^Q_yn6x`YP%p#^~78StiNhJmgS@^?V3SOSsT^YvH?%>d z9ugZ12`&%aUmwFXjdt)&a|v_^_)$Qih-L3v#~x#4O%Dz}q7cQvc))bUZ0_oPk*Ep3?RCca{d2}RW1Q{SYprj8^71~<9oKzb^O|$s8!%qVNW)2hqj;~T-Ro3Tp&omY z89uRUscC>-G$#XoBC{vw2o8UP@e`36`hwD&IU094b@Go={QU1X$6-MH#$8>tf8ZPT zBZWmrL4NCD*_uFZ6O!|=Yh^2fe_Vy`Kp05jtSl@nO&wlM&U5o`v);b##mXVU3zw_z zBSpNV=iahZtDokbUFrJ|*MCg#C~a#Qoc0aIo-D;-MK;nIF)^F2FLc1_s;QQKNE*6l zQpxnbm7JtzzecL+XdSBr(-@Mv?FYUstD^d*jW|zCNQ6wj*k(r($!};0WWRdvgHca!A|d^q0zupCGeIf2Q5&^-ec-O z!Nn)?ZSo;@61}I;#B)%Xm#Lwx`Lz0S7YxJjf%n=LS$;}X296=SFTz`j~P$<2|U=SDM?G7YLq7r+3Ch7|q836dq9 z4-uU9DC$+tqSwLv(&(AIEIBr`khqOLGZ64zuesfI$)1YPdc?l2{{Fa`@d3lxUs<0% zogHk+Q9)Iygngb6V#hqcnEGjMZcZ`NT&vV9o@8%0MiuiM#+_z2&?AKiL$Vr!DX_I_ zGDS9|gvUV+R9Y7qMsxHJL9Dd0)O_8Qxwjx6<`RHsveoedTATH3$+@pENhP(lO5TdV zk4Y|#T%!)ZKMlnW5?EOz=4F)&R{Y99EiNE@`THGJdxf~&^>zU{RouC!2b4Vpij6`i z*MB@_J=QCa2-1fnCr<9=E%MWY=RX4-RW4!&tDc;w=gtvCn^iI9QhHfl$oeeb`uckU zTC|bC&f2=VrpbXr-X2jS{Pz%6Iy0_*b#48>{v}rtA~cSO(?rU@ zS-uTlFYUaSKhw{aNBzP^GC1I7aC{6@T_rjli(G&tcCW(2yE?6P@;{ag(bZ(RHSb^Y zMxuUtHA`J};Kns{yU_5Pb^rfswUtb|9KH~ea9p?WDl$#L0Q;Au9yH%qwn;p-fc-li zZaPOq2K`;{VwR=8+4tA)Pe>o*=2qIYX?k)|lHBHe=gvwYT*$HO_?5HyO-BKOaxe* znuBIY8v2fDsC|cXwfsqWskwLml(1 z>*#xeQeuH9CTr(eBe*jtEp^`C4Q^h2pd!tR9Fj;307lf0`CSf&ituYsH2QNvM#lU zi_vYiXN~V!&Ckadi^nvsiFx`YZaFmsjZ`ELk;s=KBO>(FEq1Q1pj{f%W;={&G)hZ;2Q-QU_+ z1xZgq>Vu8(%c-qE;+}bV?fZBv#mr}9?%m>CkH{+3KU$pzvc?B6rRWVFEhw1o4r(Y5 zS-viu(REH9B}|UlKhO7fOy30L!Rq#9X-wVx&-uty{Oa^uKH) zk}Rad)da+k4s}tdV^#EG;zNPGum_e%1fX{44lp!T<5_wV%WNy3+~eru-RfUAc@tGp zi75{oJNuJ@*)?a;FCe&V(3qBL`Aej-aTqvKhIs7edqv<@z;NW6QY=gor_pAK$9~J; zpktBC*eybSyDHo3W2nI2P{*$MKa##_vz9?B+ePe<57%@H1;kGKA5~p9*i~P@GtlW? zKRWfZ{nx&t10iS@C`lOyZfDnwsf|}I26$B$xop*kx{f94k;$r=GKY2%hj^=?)vFqq zP4$WvbmJ5=9)Edu)Og^c*P4Di$J#n(!DAhgsJpIkelE>XJdQZI?ZNEjIr2!q7uU6M zF>G5W(|>%$%3}B1w^y9rNB;dgga5}=*!G*YOfxU?7wuD%4l$vVOLKR_(i^+2*?f85 z-m9C}$X@MF=yK1quC0rtVbR;UG{Axn{!?6~dXE0y{*T9i9tf>5qxrt0kk~uZyw(AS zm?mqheS3oNjKkecXM>fcjN^JGCSKdq-^xE-ubY5>GW;V3XFZIxzVE#&xx4a7=Qi#*4RM*}aj|wi-If4D#yZ-6Mf`MTT>93Q zH5V|;@-qEOwFBV}Us}pOrxeV~M~|1a%kbYAyWl&?9=VBqpyZj3>t@mYpQheVEky+| zDT*hYi?|FvsIV%3QV`_9dxUS=idYwV0l}u6m)@z1_g=uS8Zi#C&WUoM(+(5 zsk!B=Q5$B-%&=;&$G;JRRBG}#?JIzD=D55F@{UCJ-*Or3d?J1=(cN$Ve| zSNb&Cy<_Ndo&d`fJoW}MZu@bLdRy}xlKtqAS-<|m8iEnD9OL@&xngWKghe`t)_~pr zQL4{&r-@fzW}fBRn>wT`47)nLD01B@`t##|O(OjbgG>9bPkpBs4>=XTiM?MfzgYUB zFST_YeS_Gzv2x8zT|?{M@ATa9IxSRjZ2%0r$EOa;_<9FM{eE!JuWQq$O}nUP9ZQGP zbfa|>jI;d**G!tmGm1FBC!dCbPU%xMII&3@;~o9&36(1Z zO2_IG_O0=uJn2`K$Ljg~NxQQ3<6$~%l^Gi|v*`nz6uMHsI^8{idRAwb&A$KlV?U=? zJ@t;~-!x%Ml2%2Ub8Ne|rOg{* zrAG@kv#sd^yw>d%quvpXu6NgF#wk}Gu0HXu^_a^s7qY`eSATeSroGDdEXDgm?3ISh zEQeGXwc#xRb)f)x- z%I1$Kx(0S6G-hX?GR$E_dd>V(J!8JrT;rYgF&DijBLsVK6FRWrDAS4ju;C|yu^hC# zY3i9*IyOZ{Gq6t9vzzB^dLh?jV3XMRi*Fk{g>_hib@5T&ZE2M)V+}8SS>?Y<7dX)! zxL~JJnw`9Tj@G2W>*6LZW^ za_u#uyEHD|y4z9hzGrCJ_`;mYGk+hS@P+Rt&ZfxY();TlI4132`L$CdrD!Rnyleeu z_4X~1?mH?KJTohw_F6d@&TQH9fIpFr!R~7FnzoY4%G1t_X0a!j9~p~g-nYiG&6^F4 z_=p^<#JLDW|G+QCq3ktZEy}a>fT3*Vtfg{T3~9LLA21 zsMKppRpWZ%i}pXu1!f*u^tJc5rTe~mRblj@XT1%U{2pOj3C@DW#4THVs3Sf!cUN>s zPW*}a-x$!$v=2qij0gNm<{D!WOSU| z+h*0WcztfVW%^Q2nUROsQ{4?>oA@dM>Yk;(OgeBt%f>g`Z`+v~CGP`$`zX_!4{ou1 zY;w}d;Op*g)S#)uF0OAU64ReNF=K3+yk5{^o72!U=He_IIy_lzR?!+dsnM4F-L!$1 z>+M5r`;7aW^$maa|B4ymyw_hw-!^b}#EO}+*PLUf=e4<|t@XEo>H>v|p9lDlgoa(CbgZbeE)W(@3rfScT-#f3_8DF8?p4BU=(6r za0M-NQK`r+I7n7yp#U zH}B1=M3<@(%@YttyXGMvek#MQdA>nK>ndnGi{?<@^GIhQrN;VE7wHx5>>-SN z9n<7Md$;SD!9M&i{#;`I_o>eIZ^^Xkf2**i{w+RI{%;lbmOD59eA>a=LNgsE>;Fc=|J~>cmREc4^pLF1P2~9hjCcP~-ix_hb(m-2u}f37 zMr>x)m}cmNgum`cQ=O0$B`<+`;kJFoH^gEKyScr(()?b?Vz#VwpBI`0Y-6{zeTQVWrbTC_TZU-iJ*GlRn*JM4q z%h-E%@KJTyGsP#nzEecXc5Czw8Ws=j5`-Xf^?gBeRT=S{Peb>R&wk#Kymp1?aU13% zYTk#DbJI@KLiU2c42Oq;7u8ajd_qp~pVbF6e1x@LVY`hv zwQZVls!UcNcTD4Pn)TCmb-nqu+gfIK)8Xm~tL=%k_XcOV?AonT($nRqT(_6JZ`K_-gB=GX^ldG@0xohlfyudyElilp~cuFP;W0~hS zcblhy$7W)ePj6y8qG;vO3 z?CvQZIT^@r7+}jNm^8p^9%uad#I7yWw~ajUx($aiZCVCfn@fVrCBIBhKX004r8MbY z880ooYW9#nCg@14!`Ra=>k_oY_wU(NT385kT;eZA%bsUyjzpIqanif$kv?tjSd8#xeV%o&CpLvSkEv_X?uO&} zgpkEaLmX&I6%$XMSr6V*wrGvq(HauZc}OUv{iV&D1o;^vRfok#0jOmF4B( z+d;)8?_~!TzQ-Tm+GBE~U=!~6w7s~uKf5Wfq@<-hTKkCO_*8V%IAh&2mDza9EN}bP z#hQRe>(<+AhOf5uhE#`7Nb5quDKU+3S~}5m(Vt}NhSrP6iGSP`;E?IdHW*)$>figl z#A;Vfyf0S2_^?RS>%$uH4x(z5Ovk}sek=L5^9-?#v?^XO9LT&KGCOG)xbV56zcKr! zV6!Ha+*>(%yI!``6iBY8R6SVOGyZiTv_HDGiLNm=c5lf~x*z#1dwm|I-d=bhoUkCI z-dicj+B3a-cw!Gvbo4+T{Fxmc1ezTOu4*2vDTx-xp2dfH|L)?QoL#FgC8CeW5YKtoDKs( z@O>alc+b%evxevA$r6{!wiPohKiq8?Ao{YjS1a@2A>~=!3+jF%E$IOn^S_g@+)8W< zKE;=)zY&&7kMqsE!{@DMVHjVcT=VR|xBy<8PD|zaM(OYtsruz)@f-FJ>g%2`x_XrX zWHl43n$rQ*)i(1IQc;Yl4KFx*Ze$MsUij|Ae{JlU`kLC5m4@DmS(+3QFKQiP&dA;` z*VFk$8GIxZD`69-eoK^+lA5YC$+OYdflWr^)u!(3B*#2nd!f(*Va8EQoi$^ov8$>j z)$nstc{bAKN##;f`A<7+z6|_XMJZwxqMgO1;dhLc*jy`SXK(m_@4p+~Va84>+E0r9cB`d%M!w~#GXF#J7QoGs((Pik3Q?{GBsGp7z`=FwTb$38umEZ78^zP25| zz`Mi7T&Zo#1jXBe=cimD6aA)56I4q(r5@$+lZl)fUj!<8EIHeLNN{b^w<5i95v1Icl7Zeot_;%&N*q*+HNy5737jiGuXi5+AlcIcY!9vs?wSEd0%CikH1tn_yF!nP7{f`!b^+P+4!vGtf95WNA5P71i3YIFLA@ z5?!wC&7?$~RJfaxujXN8I6b+i{9nVidrn1pu=ne$XPORf3MlLAZ_W6r<=L@V@cV>i zg?zf{*tsa1%Gq|NdHWX24(;Kx`4<<=hX*1*F_9puy_b1x)G*l~uwg3ouF231ZTtCO zkHYTW}}QKv5ma!|6p zt%aKAS)Yyj`b(}g^APmvb7Z1~x)vyxITXBXS<}H@+AbTCnkx}>xujnNO5YW8qndUu zF857@6tz7zP0Spexfa#+X2g)oeCEvHcV{M*qFDLQIuo`TjVYs-{A_Tj>Fer-8;moS z+h|QXSPb#DvN(?^oFG1&plnF*Yo1)SG-Zj8-W9P4}BE$cdzRd`S zf{SLlOn7TZ^cD_T3IEmG;v75`{W1;LmZ`s1$9MSRoS4`v{BqKO6L)Q0RTZlW%`3wp zrj$dw$3HJPQDWn-2rSx}`ZSAl&WVWjt>puyp>-xmnT)BuVt(`{kxg5D9|!)v>(Y{< zfdL!uvx^@eta{S_EjOvZ{_E_9_tVpRCPu6k7Kh4gj&Yo}aLrY9+_a}YC0$_bi=$3y zU7f_s_cAGYt6h6~29u(51f%BP9Ok~q zMQy0}QuNamx7cqgd<&X2}>_8>!FVc9mB#dgEQ86jM)Yhn_9 zxRJMR{fb*BWp-A*HCOT-8~G+Z>~dsuvGAjbYL2_M>%v3FIDvwKj|XS$6tg2Zw zdcf{&e%}#a`{wJ7pcd|_9QoV)N6Jj=c%@zIC)s#3{O+&jtz@g1C{~ck%Hy4M5auf> zbGEUut(n>(Hs1Ysc>KdumNyeE?~ZvZRbD;;{@(Gvw%DHEN8D!$CWmHI2|7U2a2)nz z=Qk0iBNMT!va=}Gt9qt_>n}#LCmVO~-RHl>$GZCioseeM23IHboBUdZ#vqHM!JpS* z3uQ+9Ix7jBcz%L(GP`FMDD)f0DQG-suyRvo5aplmC|jF(tE< zf0?{({f}V|ZP}`Y@aWD<7V-C4ebc{gapJUev=|z9vb4-Z^5>s?$i(#*W!gUHNSM=+ zQlYm_+-fS`-bydOh6?Q;?n)iqS28%-VNRtwkt)tBTzqW9C!rB4%YIJJ=6U&z+YDtr zCgK$GUiB*`-zXZKO9`O+js>vMcvWZIp2;ocWR*`U>e^Cg1%CbN49lLpwsE1Hl#boi z_T3Y$dWU&?e0-XlRS^-VA<^|65D*Y(`2MXS?G$xn_#->t!9wNF%Hl@DR!mUDo%c_bDtDvz9+@Wm`JYlx|HqL053c(E z&z^Ff3>r{fLqi;(=#_c`nFCwqg;4-Ff?Kw1dCajS&~e`*5RSg3swY@D#-H8k)cm3& zB2*X5P6R;IVFs#P)UiKm(|FMeQ5w;g_WiNZ2!4)5W~f2Aul;y;CT^)6mazWvKANy^ zQ_#SI8fCX(Wp#D+_@}KGwp~#o(|k#YjOyJu*Tn_27b|Ro6BBWxwHcd^PTQHJB?pm& z733Glf#oP|YuiAp76h&sdRaRx#sA~;U(gK&&9a>5pR~+OqSD?N^B*7nHa^%ySmFVG z4Gj&kj*~;I@~`fkUUC%_9U(q?EY^gh`RI{!ls7|7Z;p?dQVSLqf?+A+1g@ zQ#K|ojg7m=2T>Zg{l9+wsv-VD-pR?yXo~8ao+cl?wCdMm4uAu>AlgCM{hn$yP1Q%k zRNA|_xtWOqE(T#{>YoVEv#{WVPzEh`RTohF9t$qX!dZC8H%|_NNz#~V=m}+YBzl5u z92~Nwy$`;$Y^ZQ>hYRS7DR15onqWFal*s!HbArJ-C;AxBaE|QTCknv?^c7M4b%|tB zZt=5|hhwoNzun$)ppG51VH1!58&mWjy?XVE$YapA1y!5^BAyE2ZM;0Q3V(>M8)`W8 zZ@uo{zYm6L7nnVKtzbHW!LlRZ{2>%}TePz*#i6M43kAiIW&i%uVAd0Q9HDX$7Wfh` zg>-#o0=eYzowZ=xOM8lu2NE>^V&j%Wm>qxV*2Z8_gV-5}uTsK_1%vYZiq#wBz}Ntb zk+0!R9X8mcgMqOJkk zP!Y`JlXpGI*Ip_kY403P7YIbjmR>wPL_gCM=Xl-q6h_3b&e~ zKd#>Z*5EeiuCeZnL6Rrr1x+ojTF8~iLT|~jD=;ZuS|gw7FYluf^?rs`Pt=)25=fZt z_?#@LW-6dIkWnDyn2NlGb%f_jmQX;z6roIGP6*N7FvMP>LU8mk$al%$l!4Ww4SMGB zj7#JdSoAP*m%1RMB!(A6CR-CC(^I<4!+dNt`MG0=&w(6y9##vmvWCqE^;Wc#05F)UHa%>D#$ruQ1z+UCb7W`NP!R^*V=fA-M3>-E# zNLk~-oL)yus{(>tWfut+qWK3ltaQ7=ap1ri&{2p1fuNAk_;|ezc(`grPPQR<L5I;d$5!$#{P!*gsqI#^1ByaXk=pcDBJq&rAR!&C# z!i)*gK!F@959Zlpj`(wgCk|pFmtpyCW@hFi$BxO7wSZ;v_>az=3XTYk`%SQ({w{A7 zsD0BR$s#&29v;E&QEB`Iap53z8j@^69+im4WBB8i(7D!?dAaE3Ns#Mfz`byP{I~|J z^YQK=3q<~w-V}FcpOOd!6zB0;l{mN^K&=!8Zv4sjWDmI=xJ|x225dTV5P)J%++VvZ zLMzzb-~CiyuR`dAgwpH`77Ap0!n$jhR@)XbcK;&OA42BWahVneZH5R*SU^4rFHPY$ zYfP)ea1+8Fq3Az9{;}7aOy3U=^2rC&V9C{jnMkN}I?m(1U>BNNQj8N1MAvlVt!Xw>GxU|`s|`;;7c6ws-N zCK*#l|7V!=NIOFO9aOQk*)}Ga!*j!VlOdpn6rO)i8;1Yw$J5I}zD2~cu=3ccwS_#d z-Ul6Tn%{1Z-~s@F|29U(1EiGahCkvCg2te8Z7qQ?gBwm;lc*gG>bvyyo~t*NOg38+`0Wg}Eg zZTl{|aOtIVxq)7b#ytcp02;qTrBv)=wWVC8z{S}Y^j_pD?oU8_BdZUBfO?`*6cE^3 zFRpa%+ygN4$Dn2=@~o-$c!KT)G_p=F#_O+!NigZ|o!b-BAg=w)#@ zL;vdc)-Hc^6NhWxU}jt03qGr{uO};5goIlDECkgf3m?vg__Av#gM^777z7D%YD*y~ zJX~C9UTiDe^U110#y~WnL_CFa5Q8Ud51l5?$1qpuge+}s>s%K`8UMWKrMtcK4;+bc z_wM?J#zw3FWkN>Vue@@ba}ciu1kQOin0k=G{r0`yNsKp^7B+Mngcx4gMdt1GnL9V% z9>&a|hfsEXKKR2K8Z~hVi8!pIb?equVIS{QNZN@28>#I&Z{z&?{Tc@>qsJTs-gWc+ znJm-171DStS&k;rZ#X=`&OR2#8g`Z6W$rCpab984SOo=N?EAKkD7V3ADT!8Ej`u6!z9Ml@pI4zHzr^bLb$PHy^&Ea3HMy+#0FdQG>8hBtM$-63Vb~XITJkgbdZGkz$;%0DQ;JyzI}LyuP808 zR+5#S8`0Q})q1t9HP~VJw|Thq>1F3mIjf-{VBU~>pFD0hlKy_HC`wK z7N93L`4)DX>~vVW82^bM?g3}kuV6N+s|i^*D4%&O=fpVX(;gNz@0{d@N^>U`LYFj0g;%?*6m1MURuHy6P%u zu?T?v$Y+=xz8$CtaiUTXTE$`J!UOUd{(L4i-!?BKbN#q8``gwFf-jNXgqNP)tOwi> zPsD;m1T^cv6TB07j@$ARU~{(Q+AF|R{y0kS%a9h;LvE1-E;9=^cWtjF`OVv|BO{cf zAubmrNkDM+%gM>9>4sKjBzaYzN78I?9SD(BkQ} z*15mIoDeG~9M`;m=1m-4VU}sro~LA9Yn5f{nm5}ja^~kNMptwXK3^rj-m4A1 z01D~mE$XOC$g3wI@18*>3Hu3?zl&-o)?n{sqU3ntCnc#t|Qqvi(Xb!V*c~CRr<6j%5 z&F6gT&HcDlQwIEnU7rRV%+B25G39$Ty2HCRO2Ng0TlmFyG4C{Ug@o<%Z5)Z(c5?L=5&drpeQQ(H-0qEA9MxzN8SNUBFB)%kt?MhO zJiJY=Qc{S&7M_1!EQRd^v|VbJZ;Qa`M-0-CZHQ)sUJu&`e*`6>p2KNW0~@qIXOG-j zaEML|=*sqgiandC=6Wt#sG#byo-CvV&kJWFoids8clTjbd zziT?s)YSa`w!^Z_arXY6HbZmV8+%(s_YTRbr%!}sPXxppb>#GacIB!UQ<_h6Tr~Rp zS*HrNzDnuT!DAztPu>_$eloRxD-c_;)$>bzlTv&=V~v%rYt7TQj~qC2QzM%8YxP8T z7nhW1=VjOUJt>LcJ(T1feP1EUs-$dWD06#Q*pe$wlBPVIBgl+>0%IH%ZOw(`R2-!Fod(@ddt__jV9hajMp;U!!UO(PtCty9* zYhRk8Qj{#GImc|KHGXRsZ_GZ5!+`Imbwl%6+VlmQD;#*gegDbs>t|5sH&`^*A5#~n z5}&|i^uC>?p`nq*wkRH?VLzt46aCj$)r7|A$4{s)f4f6RgH=*eV1BY59q(KIOh&&R z_{G&z=g;>%4eDsIU>7~Dk#?0VPEtEzcO;xgjZT6I@)WhDLTfi!q~bKg^4sMzGI$*9 z&$5*c+br(zN;lW8-7-VD7;m7&L~*2j@svp`_gMmFEB0r27dz>Xp89V5-K`LUAJjud zo!^I^_h?+54SeeNTL#2J^OEY16qTe_FDA2=>*p&SW&5UIM-lTo2=Fzk`7Qb;4i2i^ zR=8NjL|L;b6hx?O@1z#R;g>pwjXVcMA3e$O(#@%j_pQ16s3efbDlfxSBze-x&tZmPzH4K8;Vh#3Sw*M9fxkLCjSIx+mS>NwCQf%`t*;;JpQ59V9(gI& zr1Mncp-54*L(jYp`(n~u0Z*NU@T-tu6McRO&AG_GBd>n;>>X@W18e%%R{-u*8iqPY z0R)LhX8k8H|5zqjAq@M~K!;Omo@msq#pa;kbE(>!2&u)X;2w^ku7+y+Mbhjl4RjyAo;_GKEc#bj=ac;^I#dnZNqzWZs# zIy@ZvQ!J3xz~a`u$1~bbb7WaryCcr5u25V=5^xmk^uft`kVMdsbrY*!