From 9ad5d8251dfa3cab2be494cd49f06678a6a84a4e Mon Sep 17 00:00:00 2001 From: Amol Lele <19983848+leleamol@users.noreply.github.com> Date: Tue, 8 Dec 2020 09:00:03 -0800 Subject: [PATCH] Introducing the profiling functionality. (#407) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * rotation policy * fix tests * fix write event call * add comments in code * add a test through hook * fix rotation * some fixes * delete file if empty * enable multi-process test * fix multi-process test * add pt distrib test * Revert "add pt distrib test" This reverts commit a8fc661a02ba29e6fdc49019006b2dafc3cbd67d. * enable write to s3 * address some review comments * address some more review comments * cleanup * some fixes * make timestamp mandatory * filename timestamp matches 1st event * more cleanup and fixes * consolidate classes * timestamp in UTC * address review comments * edit base_start_time * remove delete if empty * default queue size and flush secs * Add timestamp test * add abs and rel timestamp in record * save default values to constants file * Cached the names of parsed files to avoid parsing them everytime. * address review comments * lazy file creation * drop events if file creation fails * rename file to event end ts * correct s3 bucket name * test timestamp with file rotation check if timestamp of all events in a file are lesser than timestamp in file name * remove ref to s3 * remove changes to s3.py * add checks for healthy writer * test file open failure * Cleanup hook * Added the buffer for looking up trace file, removed the get_events_at_time function, updated the implementation of get_events to return the active events * make timestamp mandatory everywhere * fix mxnet test * Corrected the multiplier for microseconds * remove flush_secs * Updating the tests directory with new file format. * Simplify class structure * save base_start_time in record * Updated the test directories to the updated YYYYMMDDHR format * init env variables once * Renamed the function and added function comments * address some review comments * cleanup * Fixed the trace file look for start and end time events * Truncating the trace files and updating the test file. * fix pt test * fallback node ID * Removed the functionality to cap the upper_bound_timestamp * Optimize the refreshing the file list based on the last available timestamp in the datasource viz. local or S3 * Correctly named the file suffix. Truncated the horovod timeline file * Added the functionality to download the S3 files in parallel * Addressed the review comments * address review comments * Trace events writer - part 2 (#6) * ensure there's a dir for the new file * add .tmp * handle the case when events are far apart * fix a mistake in cur_hour * updated last_file_close_time to now Co-authored-by: Vikas-kum * Record step duration in keras hook (#8) * add step duration to keras hook Co-authored-by: Vikas-kum * test TF step time with timeline writer (#9) * Read node ID from Resource config (#10) * read host ID from resource config * use timeline writer directly (#11) * Added functionality to record node_id in the events (#7) * Added functionality to record node_id in the events * Added the test to verify node id from file * Moved the functions to extract node id and timestamp to utils directory. * Add profiler config parser (#12) * Timeline file name timestamp in us (#15) * file timestamp in us * Add comprehensive tests for detailed profiler config (#18) * adding comprehensive tests * refactoring fixtures * renaming vars * remove imports * remove extraneous fixture * PR changes * documenting test cases * documenting test cases * refactoring fixtures * Supporting efficiently downloding s3 files for distributed training (#14) * Supporting efficiently downloding s3 files for distributed training * updated op_name and args when recording step duration (#17) * fixes for right directory name(#20) * Fix folder name (#21) * fixes * change all variables to microsecs * Updating the files to fix the pre-commit failures (#23) * Change invalid file path (#25) * change invalid file path * fix other precommit errors * Add error handling for parsing profiler config (#27) * Fixing the tests for CI (#28) * Fixing the tests for CI * fix out_dir bug Co-authored-by: Neelesh Dodda * Default path for profiler has changed (#29) * Update and correct some documentation (#30) * Enabling TF profiler in smdebug (#5) * Enabling TF profiler in smdebug Co-authored-by: Neelesh Dodda * change variable name and folder path (#35) * change variable name and folder path * add tests to check rotation policy * Add ProfilerSystemMetricFileParser and basic tests (#16) * Add ProfilerSystemMetricFileParser and basic tests * Refactor MetricsReaderBase class * Fix timestamp to event files mapping for both MetricsReader and SystemMetricsReader * rename MetricsReader to AlgorithmMetricsReader * refactoring. Providing a way to avoid cache and hence going OOM (#38) * refactoring. Providing a way to avoid cache and hence going OOM * modifying test cases to have use_in_memory_cache param * Time annotations in PyTorch hook (#13) * modified pytorch hook to record time annotations Co-authored-by: Vikas Kumar * Pulling in changes from smdebug repo to private (#39) * latest commit from smdebug repo master is * Disable TB Testing (#275) with commit id b8661deaafc119ec6 Co-authored-by: Nihal Harish Co-authored-by: Vikas-kum * Reorganizing the profiler tests for PR CI build (#41) * Organized the profiler tests. * Updated the tests.sh for PR CI build * Updated the tests.sh for PR CI build * profiler dashboards (#4) * add files for profiler dashboards * updated dashboards to use timeline reader * fixed bug 2,5,6,7,9,10 from bugbash * fixed bug 1,3,4,8,16,17,19 from bugbash * linked x-axis of timeline charts * Creating a generic profiler dashboard & report (#42) * Creating a generic profiler dashboard which can take a training job name and region and execute the notebook. * review comments * Updated notebooks and added Pandas functionalities (#43) (#44) * updated notebook and added Pandas functionalities * minor fixes in profiler_generic_dashboard.ipynb Co-authored-by: Nathalie Rauschmayr * Enable file rotation for Horovod trace file (#33) * Hvd file reader and rotation of files Co-authored-by: Anirudh * Pytorch profiler new (#40) * adding profiling info to pytorch hook * imore changes * capturing forward and backward time from within pytorch hook Note that hook provides backward end time, so backward start time is approximated to end of last forward/backward or now So, forward times and backward end times should be accurate while backward start time is approximated. * irmeoved print statements * ran pre-commit and removed some log statements * pre commit run * Fixed the assert * Temporarily skipping the test on codebuild projects where pytorch is not installed. * Temporarily skipping the test on codebuild projects where pytorch is not installed. * Temporarily skipping the test on codebuild projects where pytorch is not installed. * Temporarily skipping the test on codebuild projects where pytorch is not installed. * Temporarily skipping the test on codebuild projects where pytorch is not installed. * reverted the temporary changes * Fixed the assert * FIxing the CI test failure * Fixed the code to include the last layer * Updated the tests and refactored the TraceEvent class. * Converted the rnn test to pytest variant * Fixed the assert for passing CI Co-authored-by: Vikas-Kum Co-authored-by: Vikas Kumar * Python profiler (#36) Co-authored-by: Neelesh Dodda * Changes to horovod file parser (#46) * TF2 profiler tests (#48) * test detailed step/time based profiling * Bug fixes for autograd profiler in Pytorch hook. (#50) * fixed pytorch hook * fixed merge conflict * fixed bug in hook * Adding action class (#285) (#54) * Adding action class Actions added: stop trianing job, email, sms Co-authored-by: Vikas-kum * Pull in changes from the sagemaker-debugger repository (#55) * Pull in changes from the sagemaker-debugger repository * Typecasting profiling parameters to int (#52) * Refactor analysis utils (#57) * Integration tests for profiler on sagemaker (#19) scripts and infrastructure code * Typecasting str profiling parameters to bool (#58) * Typecasting str profiling parameters to bool * Add pyinstrument for python profiling (#56) * Make DetailedProfilingConfig a string in profiler config (#67) * detailed profiling config now is string * install tf_datasets (#66) * Convert profiler data to pandas frame (#47) * add class to convert profiler data to pandas frame * fixed local reader * add notebook for pandas queries * added code to find workload balancing issues in multi GPU training * Adding more checks to integration tests (#73) * pytorch Added step event, mode and more details to detailed profiling (#78) * Added step event, mode and more details to detailed profiling * Changing op name string * Making op_name equivalent to TF * changing step num to mode_step * Adding phase to autograd events * Change timeline node_id for distributed workers (#80) * change timeline node_id for distributed workers * Add integration tests for detailed profiling and python profiling (#71) * Fixing a bug where step num was not correctly used when enabling detailed profiling Dumping the torch autograd profiler every step. If there are multiple steps then data builds up and can cause gpu memory build up. * Feature to profile for different step phases 2.Capturing profiling step phases for pytorch 3.Fix bug with path string which was always having cprofile in path even if pyinstrument profiler is used * Fix pre-commit * Fix call to stats_filename * Fixing PythonStepStats * auto commit * ifix x * iFix * fix * pre commit fix * fix bug * removed code * make profiling parameters case insensitive * docstring for case insensitive config * precommit * push profiler images to alpha and get tag from environment variable * push profiler images to alpha and get tag from environment variable * Add height param to HeatMap * specify registry ID as env variable, alpha by default * Some cleanup, adding total time in cprofile * Refactored metricsHistogram and stepHistogram and amde more modular * separate usepyinstrument * iFixes for metrics historgram * Fixing StepHistogram * removing pritn with logger * refactoring * changes in detailed profiling * remove imports * notebook fixes and histogram class fixes * Adding wheel lfile * running pre-commit * fix tests * Adding unique thread id , pid, for trace event parser In every event added event_phase, node_id * pre-commit * fixing notebook and other changes * fix check for event_Args None * Changing ntoebook * upload files to s3 during test * minor fix * create new s3 folder for stats * fix syntax errors * Some cleanup * Fix int typecast for rotatemaxfilesizebytes (#19) Co-authored-by: Vikas-kum * Pull in smdebug 145d43b (#38) * Pull in latest smdebug (0.9.1) (upto commit 145d43bc3699ea95744dc2f8c2cc6a83b1801ef6) * Reverting the change to GET_OBJECTS_MULTIPROCESSING_THRESHOLD in #14. * Adding metadata file for TF Profiler parser to include startitime (#4) * TF profiler event parser * fix can_start_prof bug * populate start time * handle tf trace json in reader * separate file for metadata * Reorder the writing of events so that events get correctly written according to their end timestamp. (#39) Co-authored-by: Vikas-kum * Enable profiling between steps for tensorflow (#2) * Dump HTML for each pyinstrument stats file (#16) * output html in python profiler * dump output html for pyinstrument * Add higher level analysis functions for cProfile python profiling (#6) * Updated preview notebooks (#8) * Valid trace file check (#41) * fix valid trace file check * change log level * Adding analysis utils and updating the analysis notebook (#9) * add pandas analysis utils * update profiler analysis notebook (#32) * Updated analysis utils (#34) * add python profiling to notebook (untested) Co-authored-by: NRauschmayr Co-authored-by: Neelesh Dodda * check record end time similar to c++ writer (#45) * remove flakiness offset from sm tests (#43) * Add example notebook fixes for python profiling (#46) * Refactored profiler dashboards (#42) * refactored dashboards to plot new system metrics * updated step timeline chart to plot train/eval/global step * bugfixes for analysis notebook (#44) * Bugfixes in analysis and notebooks (#49) * Followup to the PR on analysis utils (#50) * Prevent metrics reader from reading invalid files (#52) * Modify horovod tests to generate check for horovod timeline (#51) * Bugfixes (#57) * fix for dashboards * Add timeline image for bottlenecks notebook (#59) * Error handling for pyinstrument (#58) * Enable/disable python profiling after forward pass of pytorch hook instead of backward pass (#56) * Pytorch integration tests (#33) * Enabling integration tests for pytorch * Fixed the job index for codebuild project. * Fixed the job index for codebuild project. * Fixing the codebuild project to install smdebugger in docker * Fixing codebuild project * Adding cpu jobs * Adjusted the parameters for cpu jobs * PyTorch detailed profiler traces are not present in detailed_profiling directory. * Fixing the test yml file. * Fixing the test yml file. * Removed commented code. * Added test configuration for absent profiler. * Preloading the cifar10 dataset into source directory. * ENabled the assert for checking the timestamp * adjusted the tracefile counts * Fixed the job names, added tests for cprofile * Updated the job configs * Adjusted the expected trace file count. * Changed the order in which the trace events are written * Reduced the batch size for cpu tests. * Reduced the batch size for cpu tests. * Fixed the imports * Added capability to handle html file. * Adding horovod tests for integration * Adding horovod tests for integration * Fixed the assert for horovod trace file count * Valid trace file check (#41) * fix valid trace file check * change log level * Fixed the expected count of stats and trace files. * Fixed the profiler config name UsePyinstrument * Preloading mnist dataset to avoid downloading it from internet during training. * Bugfixes in analysis and notebooks (#49) * Added test scenario to test the file rotations. * Adding more test scenarios * Adding integration test for distributed training using distributed api * Adding horovod training with resnet50 and cifar10 * FIxing tehe launcher script for resnet50 with horovod. * Increased the batch size * Supporting res50 and cifar with horovod. * Fixed the validation for horovod tracefiles. * Update tests/sagemaker/test_profiler_pytorch.py Co-authored-by: Anirudh * Scheduling sagemaker jobs in parallel. * Fixed the config file path. Co-authored-by: Vandana Kannan Co-authored-by: Nathalie Rauschmayr Co-authored-by: Anirudh * Fix buildspec yaml file for TF integration tests (#66) * Merge latest changes from smdebug to smprofiler (#68) * Updating analysis utils (#63) * Modify step stats util to compute stats for multiproc data * Modify utils to handle multi-node data * Modify notebook utils to handle multi-node data Co-authored-by: Neelesh Dodda * Merge timeline for framework events (#5) * Fixing the CI failure caused by awscli (#72) * Add metrics config (#67) * Add API functions to python profiling analysis for correlation with framework metrics (#53) * Dataloader analysis for PyTorch (#64) * Adding the functions to get the dataloader events for pytorch * Adding the training script and notebook for dataloader analysis * Fixed the timeconversion from timestamp to UTC and fixed the local reader for system tracefiles. * Updating the dataloader analysis notebook * Updated the notebook with analysis for batch processing. * Updated notebook to display python profiler stats. * Updated the notebook with documenation and layout * Updated the notebook to have static contents * Updating the notebook to handle absence of traceevents * FIxed the tracevents as per the current format and added notebook for triggering the pytorch training jobs * Moved the analysis functions from notebook to a class * Updated the utility functions to retrieve the dataloader events * Added the test scripts for horovod and distributed training * Adding a script that uses dummy custom dataloader * Addressed the review comments * Updated the utility code and added a training script that uses custom datasets * Added hyper parameteres for custom dataset training. * Fix TF event file decompression issue (#73) * Fix bugs in keras hook (#75) * Reorder events in pytorch hook (#60) * Refactor metrics config (#76) * Perf benchmark (#31) * Fix for hvd reader issue and one more change (#74) * Fixing the batch time analysis in interactive notebook to not generate incorrect plot (#81) * Fixing the compuation of batchtime * Fixing the compuation of batchtime * retrigger CI * Attempting to fix PR CI * Attempting to fix PR CI for PyTorch * Attempting to fix PR CI for PyTorch * Merge timeline fixes (#82) * Merge timeline fixes 1) putting the node_ids as threads. 2) Providing right sort order for processes and threads 3) Fixing bugs * add check if gpu is available (#62) Co-authored-by: Vikas-kum * Performance benchmarking for PyTorch (#78) * Pytorch performance tests * Fixed the estimator * Fixed the training script for correct metrics generation * Added train duration metrics in the training script * Adjusted the alarm values * Adjusted the alarm values * Fixed the job name for no smdebug and no profiler * Optimized the training script and added comments in the driver script. * Updated the scripts for framework only training job * Removed the unenecessary code. * Updating the instance types. * Notebook for interactive analysis (#69) * Notebook for interactive analysis * add python profiling to interactive analysis notebook * Updated the interactive notebook with dataloader analysis for pytorch * updated the utility functions to retrieve the dataloader events * some changes to the nb * some fixes to the nb * fixes * reset index * editing nb content * fixes * nit fix * fixes after metricsconfig * update notebooks * add updated job notebooks * updated notebooks for bug bash * update TF notebook * rename notebooks * rename notebooks * updating notebooks with feedback * Renamed Profiler to EagleEye * minor edits * scripts * fix * Updated the interactive anlaysis notebook with minor fix. * Updated the instance type for rules to ml.m5.8xlarge' * Updated the rules instances to ml.r5.4xlarge' * miyoung's changes Co-authored-by: Neelesh Dodda Co-authored-by: Amol Lele <19983848+leleamol@users.noreply.github.com> Co-authored-by: Anirudh * Fixed the metrics names to have correct instance names. (#88) * Added empty name in an event during merge_timeline if it is missing (#87) * Add an empty name only for Horovod and Herring events if name is missing for E events. * Add ProfilerTrial class and profiler builtin rules (#54) * add files for gpu usage rule * adding rule to detect cpu bottlenecks * add rule to detect outliers in step duration * added node id to rule analysis * add rule for checking gpu memory increase * added rules for batch size and max initialization time * add rule to detect load balancing issues in multi GPU training * add dockerfiles to build rule container * applying changes from https://github.com/awslabs/sagemaker-profiler/commit/57dfe2bd960ae798610b6ff52f661a4f5475eded fixed output directory and label legends Co-authored-by: Vandana Kannan Co-authored-by: Vikas Kumar * Fixing the writing of first event in the tracefile that stores the start time from epoch (#85) * Fixing the writing of first event in the tracefile. * Added the master table to ensure that we always write the metaevent in the new traceevent file. * Fixing bugs in KerasHook and profiler utils (#89) * Change smdebug version in notebooks (#90) * change smdebug version * rename tf_python_stats_dir to python_stats_dir Co-authored-by: Neelesh Dodda * Dynamic ON/OFF Herring timeline for PyTorch framework (#80) * Fix pytest version (#91) * support mixed precision training (#96) * merging sys metrics and bottlenecks in the timeline (#93) * merging sys metrics and bottlenecks in the timeline * Fix hvd failures and add native TF training in TF integration tests (#97) * Reading rule stop signal file and stopping the rule if gracetime has … (#98) * Reading rule stop signal file and stopping the rule if gracetime(60s) has passed * [Sync] Sync smdebug with sagemaker-debugger master branch (#95) Co-authored-by: Vikas-kum Co-authored-by: Vandana Kannan Co-authored-by: Anirudh Co-authored-by: Miyoung Co-authored-by: Miyoung Choi Co-authored-by: Rahul Huilgol Co-authored-by: Amol Lele <19983848+leleamol@users.noreply.github.com> * add rule for framework metrics (#100) * add rule for framework metrics overview * update report * replaced matplolib figures with bokeh charts * fix pre-commit error * minor fixes in report notebook Co-authored-by: Connor Goggins * Update Profiler Trial and Rules to Generate Report on Every Invoke (#102) * [TRSL-1037] Emit RuleEvaluationConditionMet from ProfilerReport Rule (#105) * [TRSL-1037] Emit RuleEvaluationConditionMet from ProfilerReport Rule Update ProfilerReport rule to emit RuleEvaluationConditionMet if any subrule having rule evaluation confition met. * Update to emit RuleEvaluationConditionMet at the end of job * Fix comment * add unit test for ProfilerReport * remove scanel_interval passed in * Update unit tests * Fix incorrect comment on last step. * Update log message. * Sync with sagemaker-debugger master branch and fix issue with tensorflow_datasets version (#114) * Update sagemaker.md (#250) * Bumping version to 0.9.0 (#251) * Skip using standalone keras Py3.7+ (#253) * Gradtape zcc (#252) * Fix Incorrect Log Statement (#256) * Incorrect number of tensors saved with MirroredStrategy (#257) * Change Version to 0.8.1 (#258) * Save Scalars With Mirrored Strategy (#259) * skip flaky test (#262) * Don't export to collections for all workers with unsupported distrib training (#263) * version bump (#265) * Avoiding Basehook object pickling (#266) * handle eager tensors (#271) * TF 2.x: Support for keras to estimator (#268) * Revert "TF 2.x: Support for keras to estimator (#268)" (#273) This reverts commit 749bded360dc33ec44c8c68bf3c4138dda09ed2f. * Disable TB Testing (#275) * Support for TF 2 estimator (#274) * Adding a TF2 Hvd example and test (#279) * Moved end of training log from info to debug (#281) https://github.com/awslabs/sagemaker-debugger/issues/280 * Adding action class (#285) * Adding action class Actions added: stop trianing job, email, sms * Fix buildspec used for PR CI (#287) * Adding a test to check that PT model is saved without issues (#283) * test that model can be pickled without issues * Save Model Inputs, Model Outputs, Gradients, Custom Tensors, Layer Inputs, Layer Outputs (#282) * Pin pytest version (#293) * Load IRIS Dataset from S3 (#298) * Load dataset from s3 (#299) * remove problematic log (#300) * Change Enum (#301) * Doc update (#292) * rename enum (#305) * version bump to 0.9.1 (#304) * modify asserts (#307) * version compare (#306) * Support TF 2.3 Tests (#312) * Disable TB in ZCC for AWS TF 2.3.0 (#316) * Update Assert Statements For New TF 2.2.0 DLC (#320) * Version Bump (#319) * add a note for TF 2.2 limited support (#303) Co-authored-by: Miyoung Choi Co-authored-by: Nihal Harish * TF 2.2 documentation update (#322) * update TF 2.2 smdebug features * Update code samples/notes for new pySDK and smdebug/add and fix links * add 'New features' note Co-authored-by: Miyoung Choi * Adding pagination in list_training_jobs (#323) * Adding pagination in list_Training_jobs * Test Custom Step Usecase (#331) * save tf2 model (#333) * Add ability to only save shapes of tensors (#328) * Revert "Add ability to only save shapes of tensors (#328)" (#337) This reverts commit c9eb76984ab42bbfe17e6b72b25c1edb1bfdfbc2. * Function to Test If the hook has been configured with the Default hook config (#332) * Default hook config (#338) * version bump (#339) * TF ZCC limitation footnote (#342) * Ability to save shapes (#341) * WIP saveshape * Add shape writer * Add pytorch test * Add untested keras test * fix syntax * fix syntax * Import * Import * Add tests for TF * Simplify read code * Add read API and tests * Add mxnet test * Add s3 and json tests * lint * Fix payload * fix import * Handle different num tensors for losses * Fix exact equal condition * Fix mode bug * trigger CI * Add support for distributed training with writer map * Check that value throws exception * Fix tests to make them more resilient * Fix mxnet and pytorch tests * Remove tensor names * pre-commmit * Fix get_mode * Fix bug with old index files * Fix keras test with names of tensors * Set original name to None if tf_obj is None * Fix mirrored test for cpu * Add docs * trigger CI * Fix shape writer get * Simplify by removing shape writer * Cleanup * Fix name of writer * Addressed review comments * trigger ci * retrigger CI Co-authored-by: NihalHarish * Support Inputs and Labels in the dict format (#345) * 0.9.4 (#347) * Refactor Make Numpy Array (#329) * warn gradtape users about tf.function support (#348) * Support all tf types (#346) * Model Subclassing Test (#351) * Modify Should Save Tensor Test To Work on Any Version of TF (#352) * framework version updates (#360) * list training jobs improvements (#349) * Earlier list training job would make 50 attempts irrespective. This may be bad because of unnecessary traffic. * if there are training jobs found with prefix, we break * if there are exceptions caught more than 5 times we break. * Handle Deprecation Of experimental_ref api (#356) * check file exist before moving (#364) * check file exist before moving when closing the file. * Support Saving Tensors in Graph Mode with add_for_mode (#353) * Change layer name logic (#357) * Pass Variable Length Argument To Old Function Call (#366) * test concat layers (#367) * Update README.md (#371) * Pinning the version of tensorflow_datasets package so that it does not require updating TF (#373) Co-authored-by: NihalHarish * Bugfix: Debugger breaks if should_save_tensor is called before collections are prepared (#372) * Fixing the nightly build pipelines. Avoid force reinstall of rules package when not necessary (#374) * returning list instead of dict keys (#376) fix in reuturn of _get_sm_tj_jobs_with_prefix . This function should return list always. * Add support for mixed precision training (#378) * Modify Asserts to Work with TF 2.1.0 and TF 2.0.0 (#380) * pytorch tmp (#382) * extend zcc to 2.1.2 (#384) * disable pytorch (#386) * Removed the redundant installation of smdebug and smdebug-rules (#391) * Incrementing the version to 0.9.5 (#396) * pin tensorflow dataset in test config (#399) * add back test * revert some changes * unpin pytest version Co-authored-by: Nihal Harish Co-authored-by: Vikas-kum Co-authored-by: Vandana Kannan Co-authored-by: Anirudh Co-authored-by: Miyoung Co-authored-by: Miyoung Choi Co-authored-by: Rahul Huilgol Co-authored-by: Amol Lele <19983848+leleamol@users.noreply.github.com> * Changing the Herring user-facing API (#110) * [TRSL-998] Update Rule Test with Result Checking (#106) * [TRSL-998] Update Rule Test with Result Checking Update existing rule testing to assert against rule output. This will ensure rule are tested with its report result which should be deterministic thru CI. * Generate HTML Report at every ProfilerReport invoke (#112) This change adds HTML report generation at the end of every invoke of ProfilerReport rule. * Update RuleEvaluationConditionMet to indicate end of the rule (#115) * fix: Remove the hard code notebook file path (#117) * Run rules tests in CI (#116) * Log fix memory issue fix (#113) * Changed the Herring API and variable names (#118) * Removing the functionality to attach the backward hook to the module (#125) * Removing the functionality to attach the backward hook to the module * Updated the number of traceevents as the backward hook is no longer registered. * Herring TF2 Native Graident Tape SMDebugger support (#122) * Fix bug in base hook (#127) * Minor bugfixes/changes in rules (#126) * minor bugfixes for rules * Updating batch size rule (#123) * fix for batch size rule * Dataloader rule (#108) * added dataloader rule and updated profiler report * Redesign TF dataloader metrics collection (#92) * Update profiler config parser to match latest SDK changes (#120) * Replaced herringsinglenode command with smddpsinglenode (#129) * Updating the version for profiler GA release (#124) * Updating the version for profiler GA release * Trigger Build * Trigger Build * Trigger Build * Fix paths in profiler report (#131) * changed path in profiler report * fixed env variable (#132) * making info log to debug from trace event parser as it is very verbose (#134) * Only do detailed profiling for supported TF versions. (#135) * Update PT tests (#136) * Fix bug in parser (#137) * smdistributed.dataparallel should be invoked from mpi command (#138) * smdistributed.dataparallel should be invoked from mpi command * Added comments * Bugfix: Invalid Worker (#139) * smdistributed.dataparallel environment check (#140) * smdistributed.dataparallel environment check * addressed comments * Modified check_smdataparallel_env logic * Install rules packages in PR CI (#143) * Removed the files and folders that are not required in the public repository * Removed the integration tests. * FIxed the pre-commit checks Co-authored-by: Vandana Kannan Co-authored-by: Vikas-kum Co-authored-by: Vandana Kannan Co-authored-by: Nathalie Rauschmayr Co-authored-by: Neelesh Dodda Co-authored-by: Rajan Singh Co-authored-by: sife Co-authored-by: Anirudh Co-authored-by: Vikas Kumar Co-authored-by: Anirudh Co-authored-by: Karan Jariwala Co-authored-by: Nihal Harish Co-authored-by: Miyoung Co-authored-by: Miyoung Choi Co-authored-by: Rahul Huilgol Co-authored-by: Connor Goggins Co-authored-by: JC-Gu --- config/buildspec.yml | 4 +- config/buildspec_build_wheel.yml | 2 +- config/tests.sh | 11 +- docs/sagemaker.md | 1 - .../smdataparallel_mnist.py | 240 ++ .../scripts/smdataparallel_mnist_tf2.py | 78 + setup.py | 8 +- smdebug/_version.py | 2 +- smdebug/core/access_layer/__init__.py | 9 +- smdebug/core/access_layer/utils.py | 40 +- smdebug/core/config_constants.py | 1 + smdebug/core/hook.py | 47 + smdebug/core/json_config.py | 14 + smdebug/core/locations.py | 110 +- smdebug/core/tfevent/timeline_file_writer.py | 456 +++ smdebug/core/utils.py | 101 +- smdebug/exceptions.py | 14 +- smdebug/profiler/__init__.py | 15 +- smdebug/profiler/algorithm_metrics_reader.py | 245 ++ smdebug/profiler/analysis/__init__.py | 0 .../analysis/notebook_utils/__init__.py | 4 + .../analysis/notebook_utils/heatmap.py | 228 ++ .../notebook_utils/metrics_histogram.py | 167 + .../analysis/notebook_utils/step_histogram.py | 146 + .../notebook_utils/step_timeline_chart.py | 139 + .../notebook_utils/timeline_charts.py | 480 +++ .../analysis/notebook_utils/training_job.py | 97 + .../analysis/python_profile_analysis.py | 313 ++ .../profiler/analysis/python_stats_reader.py | 158 + smdebug/profiler/analysis/utils/__init__.py | 0 .../analysis/utils/merge_timelines.py | 355 ++ .../analysis/utils/pandas_data_analysis.py | 449 +++ .../analysis/utils/profiler_data_to_pandas.py | 559 +++ .../utils/python_profile_analysis_utils.py | 236 ++ .../utils/pytorch_dataloader_analysis.py | 166 + smdebug/profiler/hvd_trace_file_rotation.py | 180 + smdebug/profiler/metrics_reader_base.py | 295 ++ smdebug/profiler/profiler_config.py | 250 ++ smdebug/profiler/profiler_config_parser.py | 326 ++ smdebug/profiler/profiler_constants.py | 72 + smdebug/profiler/python_profile_utils.py | 94 + smdebug/profiler/python_profiler.py | 255 ++ smdebug/profiler/system_metrics_reader.py | 172 + .../profiler/system_profiler_file_parser.py | 111 + smdebug/profiler/tf_profiler_parser.py | 252 +- smdebug/profiler/trace_event_file_parser.py | 364 +- smdebug/profiler/utils.py | 302 ++ smdebug/pytorch/hook.py | 406 ++- smdebug/rules/rule.py | 7 + smdebug/rules/rule_invoker.py | 12 + smdebug/tensorflow/base_hook.py | 29 +- smdebug/tensorflow/keras.py | 185 +- smdebug/tensorflow/utils.py | 16 +- smdebug/trials/profiler_trial.py | 109 + smdebug/trials/utils.py | 7 +- tests/conftest.py | 32 + ...se_insensitive_profiler_config_parser.json | 10 + .../complete_profiler_config_parser.json | 9 + ...erval_rotation_profiler_config_parser.json | 8 + ...file_open_fail_profiler_config_parser.json | 10 + ..._size_rotation_profiler_config_parser.json | 8 + .../hvd_rotation_profiler_config_parser.json | 8 + .../invalid_profiler_config_parser.json | 1 + ...id_string_data_profiler_config_parser.json | 10 + .../new_step_profiler_config_parser.json | 7 + .../new_time_profiler_config_parser.json | 7 + .../old_step_profiler_config_parser.json | 7 + .../old_time_profiler_config_parser.json | 7 + .../simple_profiler_config_parser.json | 6 + .../string_data_profiler_config_parser.json | 10 + .../test_pytorch_profiler_config_parser.json | 7 + ...st_tf2_profiler_config_parser_by_step.json | 7 + ...st_tf2_profiler_config_parser_by_time.json | 7 + .../user_disabled_profile_config_parser.json | 3 + tests/core/test_timeline_writer.py | 377 ++ tests/core/test_utils.py | 56 +- tests/mxnet/test_hook.py | 36 + tests/profiler/__init__.py | 0 tests/profiler/core/__init__.py | 0 .../core/test_algorithm_metric_readers.py | 84 + .../core/test_horovodprofiler_events.py | 83 + tests/profiler/core/test_pandas_frames.py | 278 ++ .../core/test_profiler_config_parser.py | 512 +++ tests/profiler/core/test_python_profiler.py | 388 +++ .../profiler/core/test_smtfprofiler_events.py | 51 + .../core/test_system_metric_reader.py | 95 + .../core/test_system_profiler_file_parser.py | 62 + tests/profiler/core/test_tfprofiler_events.py | 62 + tests/profiler/core/test_utils.py | 218 ++ tests/profiler/ip-172-31-19-241.trace.json | 3034 ----------------- tests/profiler/pytorch/__init__.py | 0 tests/profiler/pytorch/scripts/pytorch_rnn.py | 67 + .../profiler/pytorch/test_pytorch_profiler.py | 88 + .../pytorch/test_pytorch_profiler_rnn.py | 80 + tests/profiler/pytorch/utils.py | 27 + ...1947000_1234-testhost_model_timeline.json} | 0 .../profiler/resources/1591160699.algo-1.json | 14 + tests/profiler/resources/__init__.py | 0 .../resources/horovod_timeline_small.json | 59 + ...6be41.ant.amazon.com_horovod_timeline.json | 5 + ...6be41.ant.amazon.com_horovod_timeline.json | 60 + ...15222_905-b88ab1bb8259_model_timeline.json | 7 + ...58411_905-b88ab1bb8259_model_timeline.json | 667 ++++ ...18694_905-b88ab1bb8259_pythontimeline.json | 48 + .../1590461127873222_0001_model_timeline.json | 62 + .../1590461139923994_0001_model_timeline.json | 7 + .../1590461139949971_0001_model_timeline.json | 62 + .../resources/profiler_config_parser_utils.py | 258 ++ ...930988000000_4487_0000_pythontimeline.json | 7 + ...930989000000_4487_0000_pythontimeline.json | 7 + ...930990000000_4487_0000_pythontimeline.json | 7 + ...930991000000_4487_0000_pythontimeline.json | 7 + .../system/incremental/1591160699.algo-1.json | 14 + .../system/incremental/1591748100.algo-1.json | 32 + .../system/incremental/1591748160.algo-1.json | 32 + .../ip-172-31-19-241.trace.json.gz | Bin 0 -> 3690 bytes .../python_stats | Bin 0 -> 420137 bytes .../python_stats | Bin 0 -> 38339 bytes .../python_stats | Bin 0 -> 30483 bytes ...1596659864545103_1596659864854168.metadata | 0 ....8c859046be41.ant.amazon.com.profile-empty | Bin 0 -> 40 bytes ...59046be41.ant.amazon.com.input_pipeline.pb | Bin 0 -> 3317 bytes ...c859046be41.ant.amazon.com.kernel_stats.pb | 0 ...859046be41.ant.amazon.com.overview_page.pb | Bin 0 -> 4137 bytes ...046be41.ant.amazon.com.tensorflow_stats.pb | Bin 0 -> 19616 bytes .../8c859046be41.ant.amazon.com.trace.json.gz | Bin 0 -> 7353 bytes .../profiler/tensorflow2/test_tf2_profiler.py | 97 + tests/profiler/test_smtfprofiler_events.py | 32 - tests/profiler/test_tfprofiler_events.py | 36 - tests/pytorch/test_distributed_training.py | 90 +- tests/tensorflow2/test_keras.py | 21 + .../horovod_tests/pytorch/test_hvd.py | 144 + .../tensorflow2/test_keras_fit.py | 100 + .../smdataparallel_tests/__init__.py | 0 .../smdataparallel_tests/constants.py | 6 + .../smdataparallel_tests/pytorch/__init__.py | 0 .../pytorch/test_smdataparallel.py | 148 + .../tensorflow2/__init__.py | 0 .../tensorflow2/test_tf2_smdataparallel.py | 135 + .../smdataparallel_tests/utils.py | 12 + tests/zero_code_change/tf_utils.py | 6 + 141 files changed, 13038 insertions(+), 3245 deletions(-) create mode 100644 examples/pytorch/zero_code_change_examples/smdataparallel_mnist.py create mode 100644 examples/tensorflow2/scripts/smdataparallel_mnist_tf2.py create mode 100644 smdebug/core/tfevent/timeline_file_writer.py create mode 100644 smdebug/profiler/algorithm_metrics_reader.py create mode 100644 smdebug/profiler/analysis/__init__.py create mode 100644 smdebug/profiler/analysis/notebook_utils/__init__.py create mode 100644 smdebug/profiler/analysis/notebook_utils/heatmap.py create mode 100644 smdebug/profiler/analysis/notebook_utils/metrics_histogram.py create mode 100644 smdebug/profiler/analysis/notebook_utils/step_histogram.py create mode 100644 smdebug/profiler/analysis/notebook_utils/step_timeline_chart.py create mode 100644 smdebug/profiler/analysis/notebook_utils/timeline_charts.py create mode 100644 smdebug/profiler/analysis/notebook_utils/training_job.py create mode 100644 smdebug/profiler/analysis/python_profile_analysis.py create mode 100644 smdebug/profiler/analysis/python_stats_reader.py create mode 100644 smdebug/profiler/analysis/utils/__init__.py create mode 100644 smdebug/profiler/analysis/utils/merge_timelines.py create mode 100644 smdebug/profiler/analysis/utils/pandas_data_analysis.py create mode 100644 smdebug/profiler/analysis/utils/profiler_data_to_pandas.py create mode 100644 smdebug/profiler/analysis/utils/python_profile_analysis_utils.py create mode 100644 smdebug/profiler/analysis/utils/pytorch_dataloader_analysis.py create mode 100644 smdebug/profiler/hvd_trace_file_rotation.py create mode 100644 smdebug/profiler/metrics_reader_base.py create mode 100644 smdebug/profiler/profiler_config.py create mode 100644 smdebug/profiler/profiler_config_parser.py create mode 100644 smdebug/profiler/profiler_constants.py create mode 100644 smdebug/profiler/python_profile_utils.py create mode 100644 smdebug/profiler/python_profiler.py create mode 100644 smdebug/profiler/system_metrics_reader.py create mode 100644 smdebug/profiler/system_profiler_file_parser.py create mode 100644 smdebug/profiler/utils.py create mode 100644 smdebug/trials/profiler_trial.py create mode 100644 tests/core/json_configs/case_insensitive_profiler_config_parser.json create mode 100644 tests/core/json_configs/complete_profiler_config_parser.json create mode 100644 tests/core/json_configs/file_interval_rotation_profiler_config_parser.json create mode 100644 tests/core/json_configs/file_open_fail_profiler_config_parser.json create mode 100644 tests/core/json_configs/file_size_rotation_profiler_config_parser.json create mode 100644 tests/core/json_configs/hvd_rotation_profiler_config_parser.json create mode 100644 tests/core/json_configs/invalid_profiler_config_parser.json create mode 100644 tests/core/json_configs/invalid_string_data_profiler_config_parser.json create mode 100644 tests/core/json_configs/new_step_profiler_config_parser.json create mode 100644 tests/core/json_configs/new_time_profiler_config_parser.json create mode 100644 tests/core/json_configs/old_step_profiler_config_parser.json create mode 100644 tests/core/json_configs/old_time_profiler_config_parser.json create mode 100644 tests/core/json_configs/simple_profiler_config_parser.json create mode 100644 tests/core/json_configs/string_data_profiler_config_parser.json create mode 100644 tests/core/json_configs/test_pytorch_profiler_config_parser.json create mode 100644 tests/core/json_configs/test_tf2_profiler_config_parser_by_step.json create mode 100644 tests/core/json_configs/test_tf2_profiler_config_parser_by_time.json create mode 100644 tests/core/json_configs/user_disabled_profile_config_parser.json create mode 100644 tests/core/test_timeline_writer.py create mode 100644 tests/profiler/__init__.py create mode 100644 tests/profiler/core/__init__.py create mode 100644 tests/profiler/core/test_algorithm_metric_readers.py create mode 100644 tests/profiler/core/test_horovodprofiler_events.py create mode 100644 tests/profiler/core/test_pandas_frames.py create mode 100644 tests/profiler/core/test_profiler_config_parser.py create mode 100644 tests/profiler/core/test_python_profiler.py create mode 100644 tests/profiler/core/test_smtfprofiler_events.py create mode 100644 tests/profiler/core/test_system_metric_reader.py create mode 100644 tests/profiler/core/test_system_profiler_file_parser.py create mode 100644 tests/profiler/core/test_tfprofiler_events.py create mode 100644 tests/profiler/core/test_utils.py delete mode 100644 tests/profiler/ip-172-31-19-241.trace.json create mode 100644 tests/profiler/pytorch/__init__.py create mode 100644 tests/profiler/pytorch/scripts/pytorch_rnn.py create mode 100644 tests/profiler/pytorch/test_pytorch_profiler.py create mode 100644 tests/profiler/pytorch/test_pytorch_profiler_rnn.py create mode 100644 tests/profiler/pytorch/utils.py rename tests/profiler/{smtf_profiler_trace.json => resources/1589314018481947000_1234-testhost_model_timeline.json} (100%) create mode 100644 tests/profiler/resources/1591160699.algo-1.json create mode 100644 tests/profiler/resources/__init__.py create mode 100644 tests/profiler/resources/horovod_timeline_small.json create mode 100644 tests/profiler/resources/horovod_timeline_traces/framework/pevents/2020070206/1593673051472361_88359-8c859046be41.ant.amazon.com_horovod_timeline.json create mode 100644 tests/profiler/resources/horovod_timeline_traces/framework/pevents/2020070206/1593673051473228_88359-8c859046be41.ant.amazon.com_horovod_timeline.json create mode 100644 tests/profiler/resources/merge_traces/framework/pevents/2020071317/1594662614815222_905-b88ab1bb8259_model_timeline.json create mode 100644 tests/profiler/resources/merge_traces/framework/pevents/2020071317/1594662624558411_905-b88ab1bb8259_model_timeline.json create mode 100644 tests/profiler/resources/merge_traces/framework/pevents/2020071317/1594662624718694_905-b88ab1bb8259_pythontimeline.json create mode 100644 tests/profiler/resources/model_timeline_traces/framework/pevents/2020052602/1590461127873222_0001_model_timeline.json create mode 100644 tests/profiler/resources/model_timeline_traces/framework/pevents/2020052602/1590461139923994_0001_model_timeline.json create mode 100644 tests/profiler/resources/model_timeline_traces/framework/pevents/2020052602/1590461139949971_0001_model_timeline.json create mode 100644 tests/profiler/resources/profiler_config_parser_utils.py create mode 100644 tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930988000000_4487_0000_pythontimeline.json create mode 100644 tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930989000000_4487_0000_pythontimeline.json create mode 100644 tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930990000000_4487_0000_pythontimeline.json create mode 100644 tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930991000000_4487_0000_pythontimeline.json create mode 100644 tests/profiler/resources/test_traces/system/incremental/1591160699.algo-1.json create mode 100644 tests/profiler/resources/test_traces/system/incremental/1591748100.algo-1.json create mode 100644 tests/profiler/resources/test_traces/system/incremental/1591748160.algo-1.json create mode 100644 tests/profiler/resources/tfprofiler_local_missing_metadata/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/ip-172-31-19-241.trace.json.gz create mode 100644 tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/cProfile/1596585363775342.0_1596585373504789.0_151-algo-1_-1/python_stats create mode 100644 tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/cProfile/1596585374963238.0_1596585376008161.0_151-algo-1_2/python_stats create mode 100644 tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/cProfile/1596585376012363.8_1596585376017246.5_151-algo-1_3/python_stats create mode 100644 tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/detailed_profiling/2020080513/000000000/80807-8c859046be41.ant.amazon.com_1596659864545103_1596659864854168.metadata create mode 100644 tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/detailed_profiling/2020080513/000000000/events.out.tfevents.1596659864.8c859046be41.ant.amazon.com.profile-empty create mode 100644 tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/8c859046be41.ant.amazon.com.input_pipeline.pb create mode 100644 tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/8c859046be41.ant.amazon.com.kernel_stats.pb create mode 100644 tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/8c859046be41.ant.amazon.com.overview_page.pb create mode 100644 tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/8c859046be41.ant.amazon.com.tensorflow_stats.pb create mode 100644 tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/8c859046be41.ant.amazon.com.trace.json.gz create mode 100644 tests/profiler/tensorflow2/test_tf2_profiler.py delete mode 100644 tests/profiler/test_smtfprofiler_events.py delete mode 100644 tests/profiler/test_tfprofiler_events.py create mode 100644 tests/zero_code_change/smdataparallel_tests/__init__.py create mode 100644 tests/zero_code_change/smdataparallel_tests/constants.py create mode 100644 tests/zero_code_change/smdataparallel_tests/pytorch/__init__.py create mode 100644 tests/zero_code_change/smdataparallel_tests/pytorch/test_smdataparallel.py create mode 100644 tests/zero_code_change/smdataparallel_tests/tensorflow2/__init__.py create mode 100644 tests/zero_code_change/smdataparallel_tests/tensorflow2/test_tf2_smdataparallel.py create mode 100644 tests/zero_code_change/smdataparallel_tests/utils.py diff --git a/config/buildspec.yml b/config/buildspec.yml index a4a93e81a4..fdc9230faf 100755 --- a/config/buildspec.yml +++ b/config/buildspec.yml @@ -10,7 +10,7 @@ env: run_pytest_tensorflow: "disable" run_pytest_tensorflow2: "disable" run_pytest_xgboost: "disable" - run_pytest_profiler: "disable" + run_pytest_profiler: "enable" run_integration_pytest_pytorch: "disable" run_integration_pytest_mxnet: "disable" run_integration_pytest_tensorflow: "disable" @@ -41,6 +41,8 @@ phases: build: commands: - cd $CODEBUILD_SRC_DIR && python setup.py bdist_wheel --universal + # We do not need to force install smdebug-rules. The container used for PR builds do not have smdebug rules binary. + # Force installing rules binary attempts to re-install ipython-genutils which fails on PyTorch Ubuntu 16.04 containers. - cd $RULES_CODEBUILD_SRC_DIR && python setup.py bdist_wheel --universal - if [ "$run_pytest_xgboost" = "enable" ]; then pip install --force-reinstall $RULES_CODEBUILD_SRC_DIR/dist/*.whl; else pip install $RULES_CODEBUILD_SRC_DIR/dist/*.whl; fi - cd $CODEBUILD_SRC_DIR && pip install --force-reinstall dist/*.whl && cd .. diff --git a/config/buildspec_build_wheel.yml b/config/buildspec_build_wheel.yml index 421ce86191..6d7320ae92 100644 --- a/config/buildspec_build_wheel.yml +++ b/config/buildspec_build_wheel.yml @@ -16,7 +16,7 @@ phases: - sudo apt-get install unzip -qq -o=Dpkg::Use-Pty=0 - cd $CODEBUILD_SRC_DIR && chmod +x config/protoc_downloader.sh && ./config/protoc_downloader.sh - pip install --upgrade pip==19.3.1 - - pip install -q pytest wheel pyYaml pytest-html pre-commit pytest-cov + - pip install -q pytest==5.3.3 wheel pyYaml pytest-html pre-commit pytest-cov - pip uninstall -y boto3 && pip uninstall -y aiobotocore && pip uninstall -y botocore build: diff --git a/config/tests.sh b/config/tests.sh index 9185840d5e..310c04f4af 100644 --- a/config/tests.sh +++ b/config/tests.sh @@ -43,6 +43,14 @@ run_for_framework() { fi } +# Functionto invoke the profiler tests. The core tests in profiler are run for all the frameworks +run_profiler_test() { + # Running the core tests in profiler. + python -m pytest ${code_coverage_smdebug:+--cov=./ --cov-append} --durations=50 --html=$REPORT_DIR/report_profiler_core.html -v -s --self-contained-html tests/profiler/core + + # Running the framework specific profiler tests + python -m pytest ${code_coverage_smdebug:+--cov=./ --cov-append} --durations=50 --html=$REPORT_DIR/report_profiler_$1.html -v -s --self-contained-html tests/profiler/$1 +} export TF_CPP_MIN_LOG_LEVEL=1 export SMDEBUG_LOG_LEVEL=info #export BLOCK_STDOUT=TRUE @@ -53,7 +61,6 @@ export REPORT_DIR=$OUT_DIR/pytest_reports python -m pytest ${code_coverage_smdebug:+--cov=./ --cov-append} -v -W=ignore --durations=50 --html=$REPORT_DIR/report_analysis.html --self-contained-html tests/analysis run_for_framework core -run_for_framework profiler if [ "$run_pytest_xgboost" = "enable" ] ; then run_for_framework xgboost @@ -68,6 +75,7 @@ fi if [ "$run_pytest_tensorflow2" = "enable" ] ; then pip install tensorflow_datasets==4.0.1 run_for_framework tensorflow2 + run_profiler_test tensorflow2 fi if [ "$run_pytest_mxnet" = "enable" ] ; then @@ -76,6 +84,7 @@ fi if [ "$run_pytest_pytorch" = "enable" ] ; then run_for_framework pytorch + run_profiler_test pytorch fi check_logs $REPORT_DIR/* diff --git a/docs/sagemaker.md b/docs/sagemaker.md index 22a5ea00ce..e8b4db36c0 100644 --- a/docs/sagemaker.md +++ b/docs/sagemaker.md @@ -14,7 +14,6 @@ - [TensorBoard Visualization](#tensorboard-visualization) - [Example Notebooks](#example-notebooks) - ## Configuring SageMaker Debugger Regardless of which of the two above ways you have enabled SageMaker Debugger, you can configure it using the SageMaker python SDK. There are two aspects to this configuration. diff --git a/examples/pytorch/zero_code_change_examples/smdataparallel_mnist.py b/examples/pytorch/zero_code_change_examples/smdataparallel_mnist.py new file mode 100644 index 0000000000..f989dffe2b --- /dev/null +++ b/examples/pytorch/zero_code_change_examples/smdataparallel_mnist.py @@ -0,0 +1,240 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and limitations under the License. + +# Future +from __future__ import print_function + +# Standard Library +import argparse +import time + +# Third Party +import smdistributed.dataparallel.torch.distributed as smdataparallel +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from smdistributed.dataparallel.torch.parallel.distributed import DistributedDataParallel as DDP +from torch.optim.lr_scheduler import StepLR +from torchvision import datasets, transforms + +smdataparallel.init_process_group() + + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + self.conv1 = nn.Conv2d(1, 32, 3, 1) + self.conv2 = nn.Conv2d(32, 64, 3, 1) + self.dropout1 = nn.Dropout2d(0.25) + self.dropout2 = nn.Dropout2d(0.5) + self.fc1 = nn.Linear(9216, 128) + self.fc2 = nn.Linear(128, 10) + + def forward(self, x): + x = self.conv1(x) + x = F.relu(x) + x = self.conv2(x) + x = F.relu(x) + x = F.max_pool2d(x, 2) + x = self.dropout1(x) + x = torch.flatten(x, 1) + x = self.fc1(x) + x = F.relu(x) + x = self.dropout2(x) + x = self.fc2(x) + output = F.log_softmax(x, dim=1) + return output + + +def train(args, model, device, train_loader, optimizer, epoch): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = F.nll_loss(output, target) + loss.backward() + optimizer.step() + if batch_idx % args.log_interval == 0 and args.rank == 0: + print( + "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( + epoch, + batch_idx * len(data) * args.world_size, + len(train_loader.dataset), + 100.0 * batch_idx / len(train_loader), + loss.item(), + ) + ) + if args.verbose: + print("Batch", batch_idx, "from rank", args.rank) + + +def test(model, device, test_loader): + model.eval() + test_loss = 0 + correct = 0 + with torch.no_grad(): + for data, target in test_loader: + data, target = data.to(device), target.to(device) + output = model(data) + test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss + pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability + correct += pred.eq(target.view_as(pred)).sum().item() + + test_loss /= len(test_loader.dataset) + + print( + "\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format( + test_loss, correct, len(test_loader.dataset), 100.0 * correct / len(test_loader.dataset) + ) + ) + + +def main(): + # Training settings + parser = argparse.ArgumentParser(description="PyTorch MNIST Example") + parser.add_argument( + "--batch-size", + type=int, + default=64, + metavar="N", + help="input batch size for training (default: 64)", + ) + parser.add_argument( + "--test-batch-size", + type=int, + default=1000, + metavar="N", + help="input batch size for testing (default: 1000)", + ) + parser.add_argument( + "--epochs", + type=int, + default=14, + metavar="N", + help="number of epochs to train (default: 14)", + ) + parser.add_argument( + "--lr", type=float, default=1.0, metavar="LR", help="learning rate (default: 1.0)" + ) + parser.add_argument( + "--gamma", + type=float, + default=0.7, + metavar="M", + help="Learning rate step gamma (default: 0.7)", + ) + parser.add_argument("--seed", type=int, default=1, metavar="S", help="random seed (default: 1)") + parser.add_argument( + "--log-interval", + type=int, + default=10, + metavar="N", + help="how many batches to wait before logging training status", + ) + parser.add_argument( + "--save-model", action="store_true", default=False, help="For Saving the current Model" + ) + parser.add_argument( + "--verbose", + action="store_true", + default=False, + help="For displaying SMDataParallel-specific logs", + ) + parser.add_argument( + "--data-path", + type=str, + default="/tmp/data", + help="Path for downloading " "the MNIST dataset", + ) + + args = parser.parse_args() + args.world_size = smdataparallel.get_world_size() + args.rank = rank = smdataparallel.get_rank() + args.local_rank = local_rank = smdataparallel.get_local_rank() + args.lr = 1.0 + args.batch_size //= args.world_size // 8 + args.batch_size = max(args.batch_size, 1) + data_path = args.data_path + + if args.verbose: + print( + "Hello from rank {} of local_rank {} in world size of {}".format( + rank, local_rank, args.world_size + ) + ) + + if not torch.cuda.is_available(): + raise Exception("Must run SMDataParallel MNIST example on CUDA-capable devices.") + + torch.manual_seed(args.seed) + + device = torch.device("cuda") + + if local_rank == 0: + train_dataset = datasets.MNIST( + data_path, + train=True, + download=True, + transform=transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] + ), + ) + else: + time.sleep(8) + train_dataset = datasets.MNIST( + data_path, + train=True, + download=False, + transform=transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] + ), + ) + + train_sampler = torch.utils.data.distributed.DistributedSampler( + train_dataset, num_replicas=args.world_size, rank=rank + ) + train_loader = torch.utils.data.DataLoader( + train_dataset, + batch_size=args.batch_size, + shuffle=False, + num_workers=0, + pin_memory=True, + sampler=train_sampler, + ) + if rank == 0: + test_loader = torch.utils.data.DataLoader( + datasets.MNIST( + data_path, + train=False, + transform=transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] + ), + ), + batch_size=args.test_batch_size, + shuffle=True, + ) + + model = DDP(Net().to(device)) + torch.cuda.set_device(local_rank) + model.cuda(local_rank) + optimizer = optim.Adadelta(model.parameters(), lr=args.lr) + scheduler = StepLR(optimizer, step_size=1, gamma=args.gamma) + for epoch in range(1, args.epochs + 1): + train(args, model, device, train_loader, optimizer, epoch) + if rank == 0: + test(model, device, test_loader) + scheduler.step() + + if args.save_model: + torch.save(model.state_dict(), "mnist_cnn.pt") + + +if __name__ == "__main__": + main() diff --git a/examples/tensorflow2/scripts/smdataparallel_mnist_tf2.py b/examples/tensorflow2/scripts/smdataparallel_mnist_tf2.py new file mode 100644 index 0000000000..2cefae9cc4 --- /dev/null +++ b/examples/tensorflow2/scripts/smdataparallel_mnist_tf2.py @@ -0,0 +1,78 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "LICENSE.txt" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions and limitations under the License. + +# Third Party +import smdistributed.dataparallel.tensorflow as smdataparallel +import tensorflow as tf + +# Register smdataparallel shutdown hook +smdataparallel.init() + +gpus = tf.config.experimental.list_physical_devices("GPU") +for gpu in gpus: + tf.config.experimental.set_memory_growth(gpu, True) +if gpus: + tf.config.experimental.set_visible_devices(gpus[smdataparallel.local_rank()], "GPU") + +(mnist_images, mnist_labels), _ = tf.keras.datasets.mnist.load_data( + path="mnist-%d.npz" % smdataparallel.rank() +) + +dataset = tf.data.Dataset.from_tensor_slices( + (tf.cast(mnist_images[..., tf.newaxis] / 255.0, tf.float32), tf.cast(mnist_labels, tf.int64)) +) +dataset = dataset.repeat().shuffle(10000).batch(128) + +mnist_model = tf.keras.Sequential( + [ + tf.keras.layers.Conv2D(32, [3, 3], activation="relu"), + tf.keras.layers.Conv2D(64, [3, 3], activation="relu"), + tf.keras.layers.MaxPooling2D(pool_size=(2, 2)), + tf.keras.layers.Dropout(0.25), + tf.keras.layers.Flatten(), + tf.keras.layers.Dense(128, activation="relu"), + tf.keras.layers.Dropout(0.5), + tf.keras.layers.Dense(10, activation="softmax"), + ] +) +loss = tf.losses.SparseCategoricalCrossentropy() + +opt = tf.optimizers.Adam(0.001 * smdataparallel.size()) + +checkpoint_dir = "/tmp/checkpoints" +checkpoint = tf.train.Checkpoint(model=mnist_model, optimizer=opt) + + +def training_step(images, labels, first_batch): + with tf.GradientTape() as tape: + probs = mnist_model(images, training=True) + loss_value = loss(labels, probs) + + # Create a new DistributedGradientTape, which uses TensorFlow’s GradientTape under the hood, + # using an AllReduce to combine gradient values before applying gradients to model weights. + tape = smdataparallel.DistributedGradientTape(tape) + + grads = tape.gradient(loss_value, mnist_model.trainable_variables) + opt.apply_gradients(zip(grads, mnist_model.trainable_variables)) + + # Broadcast model and optimizer variable are first forward pass for sync + if first_batch: + smdataparallel.broadcast_variables(mnist_model.variables, root_rank=0) + smdataparallel.broadcast_variables(opt.variables(), root_rank=0) + + return loss_value + + +for batch, (images, labels) in enumerate(dataset.take(1000 // smdataparallel.size())): + loss_value = training_step(images, labels, batch == 0) + + if batch % 10 == 0 and smdataparallel.local_rank() == 0: + print("Step #%d\tLoss: %.6f" % (batch, loss_value)) + +if smdataparallel.rank() == 0: + checkpoint.save(checkpoint_dir) diff --git a/setup.py b/setup.py index c8c95e309e..262203fafb 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,13 @@ DOCLINES = (__doc__ or "").split("\n") FRAMEWORKS = ["tensorflow", "pytorch", "mxnet", "xgboost"] TESTS_PACKAGES = ["pytest", "torchvision", "pandas"] -INSTALL_REQUIRES = ["protobuf>=3.6.0", "numpy>1.16.0,<2.0.0", "packaging", "boto3>=1.10.32"] +INSTALL_REQUIRES = [ + "protobuf>=3.6.0", + "numpy>1.16.0,<2.0.0", + "packaging", + "boto3>=1.10.32", + "pyinstrument>=3.1.3", +] def compile_summary_protobuf(): diff --git a/smdebug/_version.py b/smdebug/_version.py index f8c6ac7fea..5becc17c04 100644 --- a/smdebug/_version.py +++ b/smdebug/_version.py @@ -1 +1 @@ -__version__ = "0.9.5" +__version__ = "1.0.0" diff --git a/smdebug/core/access_layer/__init__.py b/smdebug/core/access_layer/__init__.py index 651ff62b13..e60e052b43 100644 --- a/smdebug/core/access_layer/__init__.py +++ b/smdebug/core/access_layer/__init__.py @@ -1,4 +1,11 @@ # Local from .file import TSAccessFile from .s3 import TSAccessS3 -from .utils import check_dir_exists, has_training_ended, training_has_ended +from .utils import ( + DEFAULT_GRACETIME_FOR_RULE_STOP_SEC, + ENV_RULE_STOP_SIGNAL_FILENAME, + check_dir_exists, + has_training_ended, + is_rule_signalled_gracetime_passed, + training_has_ended, +) diff --git a/smdebug/core/access_layer/utils.py b/smdebug/core/access_layer/utils.py index 7187fc7c93..958af6ba39 100644 --- a/smdebug/core/access_layer/utils.py +++ b/smdebug/core/access_layer/utils.py @@ -1,5 +1,6 @@ # Standard Library import os +import time # Third Party from botocore.exceptions import ClientError @@ -15,7 +16,15 @@ from .s3 import TSAccessS3 END_OF_JOB_FILENAME = "training_job_end.ts" +ENV_RULE_STOP_SIGNAL_FILENAME = "SAGEMAKER_ENV_RULE_STOP_SIGNAL_FILE" +DEFAULT_GRACETIME_FOR_RULE_STOP_SEC = 60 +RULE_JOB_STOP_SIGNAL_FILENAME = os.getenv(ENV_RULE_STOP_SIGNAL_FILENAME, default=None) +RULESTOP_GRACETIME_SECONDS = os.getenv( + "rule_stop_grace_time_secs", default=DEFAULT_GRACETIME_FOR_RULE_STOP_SEC +) + logger = get_logger() +logger.info(f"RULE_JOB_STOP_SIGNAL_FILENAME: {RULE_JOB_STOP_SIGNAL_FILENAME}") def training_has_ended(trial_prefix): @@ -48,8 +57,7 @@ def training_has_ended(trial_prefix): """ -def has_training_ended(trial_prefix): - file_path = os.path.join(trial_prefix, END_OF_JOB_FILENAME) +def file_exists(file_path): s3, bucket_name, key_name = is_s3(file_path) if s3: try: @@ -70,6 +78,34 @@ def has_training_ended(trial_prefix): return os.path.exists(file_path) +def has_training_ended(trial_prefix): + file_path = os.path.join(trial_prefix, END_OF_JOB_FILENAME) + return file_exists(file_path) + + +def is_rule_signalled_gracetime_passed(trial_prefix): + RULE_JOB_STOP_SIGNAL_FILENAME = os.getenv(ENV_RULE_STOP_SIGNAL_FILENAME, default=None) + if RULE_JOB_STOP_SIGNAL_FILENAME is None: + return False + file_path = os.path.join(trial_prefix, RULE_JOB_STOP_SIGNAL_FILENAME) + if file_exists(file_path): + try: + # check if gracetime passed + with open(file_path, "r") as f: + rulestop_timestamp_sec_since_epoch = int(f.read()) + if time.time() > rulestop_timestamp_sec_since_epoch + RULESTOP_GRACETIME_SECONDS: + logger.info( + f"Got rule signal file:{file_path} . time in file is:{rulestop_timestamp_sec_since_epoch} Gracetime:{RULESTOP_GRACETIME_SECONDS} has passed. Returning true." + ) + return True + except Exception as ex: + logger.info( + f"Got exception while reading from rule_stop_signal file. Exception is :{ex} . Returning true." + ) + return True + return False + + def delete_s3_prefixes(bucket, keys): if not isinstance(keys, list): keys = [keys] diff --git a/smdebug/core/config_constants.py b/smdebug/core/config_constants.py index d8a212f6b0..11f3fff9af 100644 --- a/smdebug/core/config_constants.py +++ b/smdebug/core/config_constants.py @@ -42,4 +42,5 @@ CALLABLE_CACHE_ENV_VAR = "SMDEBUG_KERAS_CALLABLE_CACHE_TYPE" DEFAULT_CALLABLE_CACHE = "CACHE_PER_MODE" +DEFAULT_RESOURCE_CONFIG_FILE = "/opt/ml/input/config/resourceconfig.json" DEFAULT_SAVED_COLLECTIONS = ["losses"] diff --git a/smdebug/core/hook.py b/smdebug/core/hook.py index 8c352d1105..04fceb5852 100644 --- a/smdebug/core/hook.py +++ b/smdebug/core/hook.py @@ -38,6 +38,7 @@ from smdebug.core.sagemaker_utils import is_sagemaker_job from smdebug.core.save_config import SaveConfig, SaveConfigMode from smdebug.core.state_store import StateStore +from smdebug.core.tfevent.timeline_file_writer import TimelineFileWriter from smdebug.core.utils import ( flatten, get_tb_worker, @@ -49,6 +50,7 @@ ) from smdebug.core.writer import FileWriter from smdebug.exceptions import InvalidCollectionConfiguration +from smdebug.profiler.profiler_config_parser import ProfilerConfigParser try: from smexperiments.metrics import SageMakerFileMetricsWriter @@ -103,6 +105,7 @@ def __init__( include_collections: Optional[List[str]] = None, save_all: bool = False, include_workers: str = "one", + profiler_config_parser: Optional[ProfilerConfigParser] = None, ): """ A class used to represent the hook which gets attached to the @@ -146,6 +149,9 @@ def __init__( they are all saved in the collection `all` include_workers: str makes the hook save data from all workers + + profiler_config_parser: ProfilerConfigParser object + if passed, use this profiler configuration. by default, set up a new profiler configuration here. """ self.out_dir = verify_and_get_out_dir(out_dir) self.tensorboard_dir = get_tensorboard_dir( @@ -224,6 +230,15 @@ def __init__( self.mode_steps = {ModeKeys.GLOBAL: init_step} self.writer = None + if profiler_config_parser is None: + profiler_config_parser = ProfilerConfigParser() + profiler_config_parser.load_config() + self.profiler_config_parser = profiler_config_parser + + self.timeline_writer = TimelineFileWriter(profiler_config_parser=profiler_config_parser) + self.hvd_reader = None + self.is_smdataparallel_profiling = False + if is_sagemaker_job() and SageMakerFileMetricsWriter is not None: self.metrics_writer = SageMakerFileMetricsWriter() else: @@ -524,11 +539,20 @@ def _close_tb_writer(self): def close(self): self._cleanup() + def log_outstanding_timeline_metrics(self): + pass + def _cleanup(self): self._close_writers() if self.metrics_writer: self.metrics_writer.close() + self.log_outstanding_timeline_metrics() + self.timeline_writer.close() + + # close the Horovod file reader thread if it has been enabled + if self.hvd_reader and self.hvd_reader.enabled: + self.hvd_reader.close() training_has_ended(self.out_dir) if self.first_process is True: @@ -699,6 +723,27 @@ def _write_histogram_summary(self, tensor_name, tensor_value, save_collections): ) break + def record_trace_events( + self, timestamp, training_phase="", op_name="", phase="X", duration=1, **kwargs + ): + """ + Write trace events to the timeline. + :param training_phase: strings like, data_iterating, forward, backward, operations etc + :param op_name: more details about phase like whether dataset or iterator + :param phase: this is defaulted to 'X' + :param timestamp: start_time for the event (in seconds) + :param duration: any duration manually computed (in seconds) + :param kwargs: can be process id and thread id + """ + self.timeline_writer.write_trace_events( + training_phase=training_phase, + op_name=op_name, + phase=phase, + timestamp=timestamp, + duration=duration, + **kwargs, + ) + def _write_scalars(self): """ This function writes all the scalar values saved in the scalar_cache to file. @@ -946,6 +991,7 @@ def __init__( include_collections: Optional[List[str]] = None, save_all: bool = False, include_workers: str = "one", + profiler_config_parser=None, ): super().__init__( collection_manager=collection_manager, @@ -961,6 +1007,7 @@ def __init__( include_collections=include_collections, save_all=save_all, include_workers=include_workers, + profiler_config_parser=profiler_config_parser, ) self.exported_collections = False self.data_type_name = data_type_name diff --git a/smdebug/core/json_config.py b/smdebug/core/json_config.py index aa9b088d5f..7ed43d4356 100644 --- a/smdebug/core/json_config.py +++ b/smdebug/core/json_config.py @@ -63,6 +63,7 @@ CONFIG_SAVE_ALL_KEY, CONFIG_SAVE_CONFIGS_KEY, DEFAULT_CONFIG_FILE_PATH, + DEFAULT_RESOURCE_CONFIG_FILE, DEFAULT_SAGEMAKER_OUTDIR, DEFAULT_SAGEMAKER_TENSORBOARD_PATH, DEFAULT_WORKER_NAME, @@ -298,3 +299,16 @@ def parse_save_config_dict(params, mode=None) -> Dict: if "end_step" in params: ret["end_step"] = params["end_step"] return ret + + +def get_node_id_from_resource_config(): + """ Expects resource config to be present at /opt/ml/input/config/resourceconfig.json + Returns the current host ID in that json file, or None if not exists. + """ + path = Path(DEFAULT_RESOURCE_CONFIG_FILE) + if path.is_file(): + my_dict = json.loads(path.read_text()) + current_host_id = my_dict.get("current_host") + return current_host_id + else: + return None diff --git a/smdebug/core/locations.py b/smdebug/core/locations.py index c50cdda208..0ee17b7cfc 100644 --- a/smdebug/core/locations.py +++ b/smdebug/core/locations.py @@ -1,11 +1,22 @@ # Standard Library import os import re +import time from abc import ABC, abstractmethod +from datetime import datetime + +# First Party +from smdebug.profiler.profiler_constants import ( + CONVERT_TO_MICROSECS, + DEFAULT_PREFIX, + MERGEDTIMELINE_SUFFIX, + PYTHONTIMELINE_SUFFIX, + TRACE_DIRECTORY_FORMAT, +) # Local from .logger import get_logger -from .utils import get_immediate_subdirectories +from .utils import ensure_dir, get_immediate_subdirectories, get_node_id logger = get_logger() @@ -117,6 +128,103 @@ def get_file_location(self, base_dir=""): return os.path.join(event_key_prefix, self.get_filename()) +class TraceFileLocation: + # File path generated based on + # $ENV_BASE_FOLDER/framework/pevents/$START_TIME_YYYYMMDDHR/ + # $FILEEVENTENDTIMEUTCINEPOCH_{$ENV_NODE_ID}_model_timeline.json + @staticmethod + def get_file_location(timestamp, base_dir, suffix=PYTHONTIMELINE_SUFFIX): + env_base_location = base_dir + date_hour = time.strftime( + TRACE_DIRECTORY_FORMAT, time.gmtime(timestamp / CONVERT_TO_MICROSECS) + ) + timestamp = int(round(timestamp)) + worker_id = get_node_id() + file_path = os.path.join( + env_base_location, + DEFAULT_PREFIX + + "/" + + date_hour + + "/" + + str(timestamp) + + "_" + + worker_id + + "_" + + suffix, + ) + return file_path + + @staticmethod + def get_detailed_profiling_log_dir(base_folder, framework, current_step): + current_time = datetime.today().strftime("%Y%m%d%H") + padded_start_step = str(current_step).zfill(9) + log_dir = os.path.join( + base_folder, + "framework", + framework, + "detailed_profiling", + current_time, + padded_start_step, + ) + ensure_dir(log_dir, is_file=False) + return log_dir + + @staticmethod + def get_tf_profiling_metadata_file(base_folder, start_time_us, end_time_us): + metadata_file = os.path.join( + base_folder, + get_node_id() + + "_" + + str(int(round(start_time_us))) + + "_" + + str(int(round(end_time_us))) + + ".metadata", + ) + ensure_dir(metadata_file, is_file=True) + return metadata_file + + @staticmethod + def get_python_profiling_stats_dir( + base_folder, + profiler_name, + framework, + start_mode, + start_step, + start_phase, + start_time_since_epoch_in_micros, + end_mode, + end_step, + end_phase, + end_time_since_epoch_in_micros, + ): + node_id = get_node_id() + stats_dir = "{0}-{1}-{2}-{3}_{4}-{5}-{6}-{7}".format( + start_mode, + start_step, + start_phase, + start_time_since_epoch_in_micros, + end_mode, + end_step, + end_phase, + end_time_since_epoch_in_micros, + ) + stats_dir_path = os.path.join( + base_folder, "framework", framework, profiler_name, node_id, stats_dir + ) + ensure_dir(stats_dir_path, is_file=False) + return stats_dir_path + + @staticmethod + def get_merged_trace_file_location(base_dir, timestamp_in_us): + env_base_location = base_dir + timestamp = int(round(timestamp_in_us)) + worker_id = get_node_id() + file_path = os.path.join( + env_base_location, str(timestamp) + "_" + worker_id + "_" + MERGEDTIMELINE_SUFFIX + ) + return file_path + + class IndexFileLocationUtils: # These functions are common to index reader and index writer MAX_INDEX_FILE_NUM_IN_INDEX_PREFIX = 1000 diff --git a/smdebug/core/tfevent/timeline_file_writer.py b/smdebug/core/tfevent/timeline_file_writer.py new file mode 100644 index 0000000000..c622ee7bca --- /dev/null +++ b/smdebug/core/tfevent/timeline_file_writer.py @@ -0,0 +1,456 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Writes trace events to disk in trial dir or user-specified dir.""" + +# Standard Library +import collections +import json +import os +import threading +import time +from datetime import datetime + +# Third Party +import six + +# First Party +from smdebug.core.access_layer.file import SMDEBUG_TEMP_PATH_SUFFIX +from smdebug.core.locations import TraceFileLocation +from smdebug.core.logger import get_logger +from smdebug.core.utils import ensure_dir, get_node_id +from smdebug.profiler.profiler_constants import CONVERT_TO_MICROSECS, PYTHONTIMELINE_SUFFIX + +logger = get_logger() + + +def _get_sentinel_event(base_start_time): + """Generate a sentinel trace event for terminating worker.""" + return TimelineRecord(timestamp=time.time(), base_start_time=base_start_time) + + +""" +TimelineRecord represents one trace event that ill be written into a trace event JSON file. +""" + + +class TimelineRecord: + def __init__( + self, + timestamp, + base_start_time, + training_phase="", + phase="X", + operator_name="", + args=None, + duration=0, + ): + """ + :param timestamp: Mandatory field. Absolute start_time for the event in seconds. + :param training_phase: strings like, train_step_time , test_step_time etc + :param phase: Trace event phases. Example "X", "B", "E", "M" + :param operator_name: more details about phase like whether dataset or iterator + :param args: other information to be added as args + :param duration: any duration manually computed (in seconds) + """ + self.training_phase = training_phase + self.phase = phase + self.op_name = operator_name + self.args = args + self.base_start_time = base_start_time + abs_ts_micros = int(timestamp * CONVERT_TO_MICROSECS) + self.rel_ts_micros = abs_ts_micros - self.base_start_time + self.duration = ( + duration + if duration is not None + else int(round(time.time() * CONVERT_TO_MICROSECS) - abs_ts_micros) + ) + self.event_end_ts_micros = abs_ts_micros + self.duration + self.pid = 0 + self.tid = 0 + + def to_json(self): + json_dict = { + "name": self.op_name, + "pid": self.pid, + "ph": self.phase, + "ts": self.rel_ts_micros, + } + + # handle Instant event + if self.phase == "i": + if self.args: + # Instant events have a field unique to them called scope. + # scope can be "g" - global, "p" - process, "t" - thread. + # parsing this value that is being passed as args. + s = self.args["s"] if "s" in self.args else "t" + json_dict.update({"s": s}) + if "s" in self.args: + self.args.pop("s") + elif self.phase == "X": + json_dict.update({"dur": self.duration}) + + if self.args: + json_dict["args"] = self.args + + return json.dumps(json_dict) + + +class TimelineFileWriter: + """This class is adapted from EventFileWriter in Tensorflow: + https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/summary/writer/event_file_writer.py + Writes TimelineRecord to a JSON trace file. + The `TimelineFileWriter` class creates a timeline JSON file in the specified directory, + and asynchronously writes TimelineRecord to the file. + """ + + def __init__(self, profiler_config_parser, max_queue=100, suffix=PYTHONTIMELINE_SUFFIX): + """Creates a `TimelineFileWriter` and a trace event file to write to. + This event file will contain TimelineRecord as JSON strings, which are written to + disk via the write_record method. + If the profiler is not enabled, trace events will not be written to the file. + The other arguments to the constructor control the asynchronous writes to + the event file: + """ + self.start_time_since_epoch_in_micros = int(round(time.time() * CONVERT_TO_MICROSECS)) + self._profiler_config_parser = profiler_config_parser + self._event_queue = six.moves.queue.Queue(max_queue) + self._sentinel_event = _get_sentinel_event(self.start_time_since_epoch_in_micros) + self._worker = _TimelineLoggerThread( + queue=self._event_queue, + sentinel_event=self._sentinel_event, + base_start_time_in_us=self.start_time_since_epoch_in_micros, + profiler_config_parser=self._profiler_config_parser, + suffix=suffix, + ) + self._worker.start() + + def _update_base_start_time(self, base_start_time_in_us): + """ + Some trace files such as the Horovod trace file may start before this timeline + writer is initialized. In such case, use this function to update the start time + since epoch in micros. + """ + if base_start_time_in_us != self.start_time_since_epoch_in_micros: + self.start_time_since_epoch_in_micros = base_start_time_in_us + self._worker._update_base_start_time(base_start_time_in_us) + + def write_trace_events( + self, timestamp, training_phase="", op_name="", phase="X", duration=0, **kwargs + ): + """ + Creates TimelineRecord from the details passed as parameters, and enqueues an event for write. + :param timestamp:start_time for the event (in seconds) + :param training_phase: strings like, data_iteration, forward, backward, operations etc + :param op_name: more details about phase like whether dataset or iterator + :param phase: phase of trace event. default is 'X' + :param duration: any duration manually computed (in seconds) + :param kwargs: other params. can be process id and thread id + """ + if not self._worker._healthy or not self._profiler_config_parser.profiling_enabled: + return + duration_in_us = int(duration * CONVERT_TO_MICROSECS) # convert to micro seconds + args = {**kwargs} + event = TimelineRecord( + training_phase=training_phase, + operator_name=op_name, + phase=phase, + timestamp=timestamp, + args=args, + duration=duration_in_us, + base_start_time=self.start_time_since_epoch_in_micros, + ) + self.write_event(event) + + def write_event(self, event): + """Adds a trace event to the JSON file.""" + self._event_queue.put(event) + + def flush(self): + """Flushes the trace event file to disk. + Call this method to make sure that all pending events have been written to disk. + """ + self._event_queue.join() + self._worker.flush() + + def close(self): + """Flushes the trace event file to disk and close the file. + """ + self.write_event(self._sentinel_event) + self._worker.join() + self._worker.close() + + +class _TimelineLoggerThread(threading.Thread): + """Thread that logs events. Copied from + https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/summary/writer/event_file_writer.py#L133""" + + def __init__( + self, + queue, + sentinel_event, + base_start_time_in_us, + profiler_config_parser, + verbose=False, + suffix=PYTHONTIMELINE_SUFFIX, + ): + """Creates a _TimelineLoggerThread.""" + threading.Thread.__init__(self) + self.daemon = True + self._queue = queue + self._sentinel_event = sentinel_event + self._num_outstanding_events = 0 + self._writer = None + self.verbose = verbose + # This is a master dictionary that keeps the track of training_phase to pid globally. The dictionary will not + # get + # reset for a new file. This ensures that we keep unique pid for the training phases. + self.training_phase_to_pid = collections.defaultdict(int) + # This table keeps track of training_phase to pid for a given file. It will be reset for every new file. For + # a given training phase, if we don't find entry in this table, we will write metaevent for that training + # phase. + self.tensor_table = collections.defaultdict(int) + self.continuous_fail_count = 0 + self.is_first = True + self._update_base_start_time(base_start_time_in_us) + self._healthy = True + self._profiler_config_parser = profiler_config_parser + self.node_id = get_node_id() + self.suffix = suffix + + def _update_base_start_time(self, base_start_time_in_us): + """ + Some trace files such as the Horovod trace file may start before this timeline + writer is initialized. In such case, use this function to update the start time + since epoch in micros. + """ + self.last_event_end_time_in_us = int(round(base_start_time_in_us)) + self.last_file_close_time_in_us = self.last_event_end_time_in_us + self.cur_hour = datetime.utcfromtimestamp( + self.last_file_close_time_in_us / CONVERT_TO_MICROSECS + ).hour + + def run(self): + while True: + # if there is long interval between 2 events, just keep checking if + # the file is still open. if it is open for too long and a new event + # has not occurred, close the open file based on rotation policy. + if self._writer and self._should_rotate_now(time.time() * CONVERT_TO_MICROSECS): + self.close() + + event = self._queue.get() + + if ( + not self._healthy + or not self._profiler_config_parser.profiling_enabled + or event is self._sentinel_event + ): + self._queue.task_done() + break + + try: + # write event + _ = self.write_event(event) + finally: + self._queue.task_done() + time.sleep(0) + + def open(self, path, cur_event_end_time): + """ + Open the trace event file either from init or when closing and opening a file based on rotation policy + """ + try: + ensure_dir(path) + self._writer = open(path, "a+") + except (OSError, IOError) as err: + logger.debug(f"Sagemaker-Debugger: failed to open {path}: {str(err)}") + self.continuous_fail_count += 1 + return False + self.tensor_table = collections.defaultdict(int) + self.is_first = True + self._writer.write("[\n") + self._healthy = True + self.cur_hour = datetime.utcfromtimestamp(cur_event_end_time / CONVERT_TO_MICROSECS).hour + return True + + def _get_rotation_info(self, now_in_us): + file_size = self.file_size() + now = now_in_us / CONVERT_TO_MICROSECS # convert to seconds + + # find the difference between the now and last file closed time (in seconds) + diff_in_seconds = int(round(now - (self.last_file_close_time_in_us / CONVERT_TO_MICROSECS))) + + now_datehour = datetime.utcfromtimestamp(now) + + # check if the flush is going to happen in the next hour, if so, + # close the file, create a new directory for the next hour and write to file there + diff_in_hours = abs(now_datehour.hour - self.cur_hour) + + return file_size, diff_in_seconds, diff_in_hours + + def _should_rotate_now(self, now_in_us): + file_size, diff_in_seconds, diff_in_hours = self._get_rotation_info(now_in_us) + rotation_policy = self._profiler_config_parser.config.trace_file.rotation_policy + + if diff_in_hours != 0: + return True + + if diff_in_seconds > rotation_policy.file_close_interval: + return True + + if file_size > rotation_policy.file_max_size: + return True + + return False + + def write_event(self, record): + """Appends trace event to the file.""" + # Check if event is of type TimelineRecord. + if not isinstance(record, TimelineRecord): + raise TypeError("expected a TimelineRecord, " " but got %s" % type(record)) + self._num_outstanding_events += 1 + + """ + Rotation policy: + Close file if file size exceeds $ENV_MAX_FILE_SIZE or folder was created more than + $ENV_CLOSE_FILE_INTERVAL time duration. + """ + end_time_for_event_in_us = record.event_end_ts_micros + + # check if any of the rotation policies have been satisfied. close the existing + # trace file and open a new one + # policy 1: if file size exceeds specified max_size + # policy 2: if same file has been written to for close_interval time + # policy 3: if a write is being made in the next hour, create a new directory + if self._writer and self._should_rotate_now(end_time_for_event_in_us): + self.close() + + # if file has not been created yet, create now + if not self._writer: + file_opened = self.open(path=self.name(), cur_event_end_time=end_time_for_event_in_us) + if not file_opened: + file_open_fail_threshold = ( + self._profiler_config_parser.config.trace_file.file_open_fail_threshold + ) + if self.continuous_fail_count >= file_open_fail_threshold: + logger.warning( + "Encountered {} number of continuous failures while trying to open the file. " + "Marking the writer unhealthy. All future events will be dropped.".format( + str(file_open_fail_threshold) + ) + ) + self._healthy = False + return + + # First writing a metadata event + if self.is_first: + args = {"start_time_since_epoch_in_micros": record.base_start_time} + json_dict = {"name": "process_name", "ph": "M", "pid": 0, "args": args} + self._writer.write(json.dumps(json_dict) + ",\n") + + args = {"sort_index": 0} + json_dict = {"name": "process_sort_index", "ph": "M", "pid": 0, "args": args} + self._writer.write(json.dumps(json_dict) + ",\n") + self.is_first = False + + if self.tensor_table[record.training_phase] == 0: + # Get the tensor_idx from master table if not create one and append it to master table. + if record.training_phase in self.training_phase_to_pid: + tensor_idx = self.training_phase_to_pid[record.training_phase] + else: + tensor_idx = len(self.training_phase_to_pid) + self.training_phase_to_pid[record.training_phase] = tensor_idx + + self.tensor_table[record.training_phase] = tensor_idx + + # Instant events don't have a training phase + if record.phase != "i": + args = {"name": record.training_phase} + json_dict = {"name": "process_name", "ph": "M", "pid": tensor_idx, "args": args} + self._writer.write(json.dumps(json_dict) + ",\n") + + args = {"sort_index": tensor_idx} + json_dict = { + "name": "process_sort_index", + "ph": "M", + "pid": tensor_idx, + "args": args, + } + self._writer.write(json.dumps(json_dict) + ",\n") + + record.pid = self.tensor_table[record.training_phase] + + # write the trace event record + position_and_length_of_record = self._writer.write(record.to_json() + ",\n") + self.flush() + if record.event_end_ts_micros > self.last_event_end_time_in_us: + self.last_event_end_time_in_us = record.event_end_ts_micros + return position_and_length_of_record + + def flush(self): + """Flushes the trace event file to disk.""" + if self._num_outstanding_events == 0: + return + if self._writer is not None: + self._writer.flush() + if self.verbose and logger is not None: + logger.debug( + "wrote %d %s to disk", + self._num_outstanding_events, + "event" if self._num_outstanding_events == 1 else "events", + ) + self._num_outstanding_events = 0 + + def close(self): + """Flushes the pending events and closes the writer after it is done.""" + if self._writer is not None: + # seeking the last ',' and replacing with ']' to mark EOF + file_seek_pos = self._writer.tell() + self._writer.seek(file_seek_pos - 2) + self._writer.truncate() + + if file_seek_pos > 2: + self._writer.write("\n]") + + self.flush() + self._writer.close() + + if self._profiler_config_parser.profiling_enabled: + # ensure that there's a directory for the new file name + new_file_name = TraceFileLocation().get_file_location( + base_dir=self._profiler_config_parser.config.local_path, + timestamp=self.last_event_end_time_in_us, + suffix=self.suffix, + ) + ensure_dir(new_file_name) + os.rename(self.name(), new_file_name) + + self._writer = None + self.last_file_close_time_in_us = time.time() * CONVERT_TO_MICROSECS + + def name(self): + return ( + self._profiler_config_parser.config.local_path + + "/framework/" + + self.node_id + + "_" + + self.suffix + + SMDEBUG_TEMP_PATH_SUFFIX + ) + + def file_size(self): + return os.path.getsize(self.name()) # in bytes diff --git a/smdebug/core/utils.py b/smdebug/core/utils.py index a2487f049c..5081ab6804 100644 --- a/smdebug/core/utils.py +++ b/smdebug/core/utils.py @@ -22,6 +22,7 @@ from smdebug.core.logger import get_logger from smdebug.exceptions import IndexReaderException +_is_invoked_via_smddp = None logger = get_logger() @@ -103,7 +104,7 @@ def is_s3(path): return False, None, None -def is_first_process(path): +def is_first_process(path, is_dir=True): """ This function is used to determine the caller of the process is the first process to do so. @@ -134,10 +135,15 @@ def is_first_process(path): ) return True # Cannot Implement This Functionality for S3 else: - ensure_dir(path, is_file=False) - filename = os.path.join(path, CLAIM_FILENAME) + if is_dir: + ensure_dir(path, is_file=False) + filename = os.path.join(path, CLAIM_FILENAME) + else: + ensure_dir(path) + filename = path try: fd = os.open(filename, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + os.fsync(fd) os.close(fd) return True except FileExistsError: @@ -310,6 +316,69 @@ def get_tb_worker(): return f"{os.getpid()}_{socket.gethostname()}" +def get_distributed_worker(): + """Get the rank for horovod or torch distributed. If none of them are being used, + return None""" + rank = None + try: + import torch.distributed as dist + except (ImportError, ModuleNotFoundError): + dist = None + rank = None + if dist and hasattr(dist, "is_initialized") and dist.is_initialized(): + rank = dist.get_rank() + else: + try: + import horovod.torch as hvd + + if hvd.size(): + rank = hvd.rank() + except (ModuleNotFoundError, ValueError, ImportError): + pass + + try: + import horovod.tensorflow as hvd + + if hvd.size(): + rank = hvd.rank() + except (ModuleNotFoundError, ValueError, ImportError): + pass + + # smdistributed.dataparallel should be invoked via `mpirun`. + # It supports EC2 machines with 8 GPUs per machine. + if check_smdataparallel_env(): + try: + import smdistributed.dataparallel.torch.distributed as smdataparallel + + if smdataparallel.get_world_size(): + return smdataparallel.get_rank() + except (ModuleNotFoundError, ValueError, ImportError): + pass + + try: + import smdistributed.dataparallel.tensorflow as smdataparallel + + if smdataparallel.size(): + return smdataparallel.rank() + except (ModuleNotFoundError, ValueError, ImportError): + pass + return rank + + +def get_node_id(): + """Gets current host ID from SM config and set node ID as pid-hostID. + If config is not available, use pid-hostname. + """ + from smdebug.core.json_config import get_node_id_from_resource_config # prevent circular import + + rank = get_distributed_worker() + + node_id = get_node_id_from_resource_config() + rank = rank if rank is not None else os.getpid() + node_id = f"{rank}-{node_id}" if node_id else f"{rank}_{socket.gethostname()}" + return node_id.replace("_", "-") + + def remove_file_if_exists(file_path): if os.path.exists(file_path): os.remove(file_path) @@ -402,3 +471,29 @@ def __exit__(self, *args): shutil.rmtree(self.out_dir, ignore_errors=True) if self.tensorboard_dir: shutil.rmtree(self.tensorboard_dir, ignore_errors=True) + + +def check_smdataparallel_env(): + # Check to ensure it is invoked by mpi and the SM distribution is `dataparallel` + global _is_invoked_via_smddp + if _is_invoked_via_smddp is None: + _is_invoked_via_mpi = ( + os.getenv("OMPI_COMM_WORLD_SIZE") is not None + and int(os.getenv("OMPI_COMM_WORLD_SIZE")) >= 8 + ) + if os.getenv("SM_FRAMEWORK_PARAMS") is None: + _is_invoked_via_smddp = False + else: + try: + smddp_flag = json.loads(os.getenv("SM_FRAMEWORK_PARAMS")) + except: + _is_invoked_via_smddp = False + return _is_invoked_via_smddp + if ( + smddp_flag.get("sagemaker_distributed_dataparallel_enabled", False) + and _is_invoked_via_mpi + ): + _is_invoked_via_smddp = True + else: + _is_invoked_via_smddp = False + return _is_invoked_via_smddp diff --git a/smdebug/exceptions.py b/smdebug/exceptions.py index e2ed43da91..27d2c821e2 100644 --- a/smdebug/exceptions.py +++ b/smdebug/exceptions.py @@ -99,6 +99,15 @@ def __str__(self): return "Invalid Worker: {}".format(self.worker) +class NoMoreProfilerData(Exception): + def __init__(self, timestamp): + self.timestamp = timestamp + self.msg = "Looking for timestamp {} and reached " "end of training.".format(timestamp) + + def __str__(self): + return self.msg + + class NoMoreData(Exception): def __init__(self, step, mode, last_step): self.step = step @@ -117,9 +126,10 @@ def __str__(self): class RuleEvaluationConditionMet(Exception): - def __init__(self, rule_name, step): + def __init__(self, rule_name, step, end_of_rule=False): self.rule_name = rule_name self.step = step + self.end_of_rule = end_of_rule def __str__(self): return "Evaluation of the rule {} at step {} resulted in the condition being met".format( @@ -130,7 +140,7 @@ def __str__(self): class InsufficientInformationForRuleInvocation(Exception): def __init__(self, rule_name, message): self.rule_name = rule_name - self.message = mesage + self.message = message def __str__(self): return "Insufficient information to invoke rule {}: {}".format(self.rule_name, self.message) diff --git a/smdebug/profiler/__init__.py b/smdebug/profiler/__init__.py index b0eb219a68..b075236b11 100644 --- a/smdebug/profiler/__init__.py +++ b/smdebug/profiler/__init__.py @@ -1,3 +1,16 @@ # Local -from .tf_profiler_parser import SMTFProfilerEvents, TFProfilerEvents +from .algorithm_metrics_reader import LocalAlgorithmMetricsReader, S3AlgorithmMetricsReader +from .metrics_reader_base import MetricsReaderBase +from .system_metrics_reader import ( + LocalSystemMetricsReader, + ProfilerSystemEvents, + S3SystemMetricsReader, +) +from .system_profiler_file_parser import SystemProfilerEventParser +from .tf_profiler_parser import ( + HorovodProfilerEvents, + SMDataParallelProfilerEvents, + SMProfilerEvents, + TensorboardProfilerEvents, +) from .trace_event_file_parser import TraceEvent, TraceEventParser diff --git a/smdebug/profiler/algorithm_metrics_reader.py b/smdebug/profiler/algorithm_metrics_reader.py new file mode 100644 index 0000000000..303b9d4a8c --- /dev/null +++ b/smdebug/profiler/algorithm_metrics_reader.py @@ -0,0 +1,245 @@ +# Standard Library + +import bisect +import json +import os + +# First Party +from smdebug.core.access_layer.s3handler import ListRequest, ReadObjectRequest, S3Handler, is_s3 +from smdebug.profiler.metrics_reader_base import MetricsReaderBase +from smdebug.profiler.profiler_constants import ( + DEFAULT_PREFIX, + ENV_TIME_BUFFER, + HOROVODTIMELINE_SUFFIX, + MODELTIMELINE_SUFFIX, + PYTHONTIMELINE_SUFFIX, + SMDATAPARALLELTIMELINE_SUFFIX, + TENSORBOARDTIMELINE_SUFFIX, + TIME_BUFFER_DEFAULT, +) +from smdebug.profiler.tf_profiler_parser import ( + HorovodProfilerEvents, + SMDataParallelProfilerEvents, + SMProfilerEvents, + TensorboardProfilerEvents, +) +from smdebug.profiler.utils import ( + get_node_id_from_tracefilename, + get_timestamp_from_tracefilename, + is_valid_tfprof_tracefilename, + is_valid_tracefilename, +) + + +class AlgorithmMetricsReader(MetricsReaderBase): + # cache the fetched events in memory + # if you have enough available memory subsequent fetch will be faster + # if there is not enough memory, use use_in_memory_cache to use S3 or disk as cache + def __init__(self, use_in_memory_cache=False): + super().__init__(use_in_memory_cache) + self.prefix = "framework" + self._SMEventsParser = SMProfilerEvents() + self._PythontimelineEventsParser = SMProfilerEvents() + self._DetailedframeworkEventsParser = SMProfilerEvents(type="DetailedframeworkMetrics") + self._TBEventsParser = TensorboardProfilerEvents() + self._HorovordEventsParser = HorovodProfilerEvents() + self._SMdataparallelEventsParser = SMDataParallelProfilerEvents() + self._event_parsers = [ + self._PythontimelineEventsParser, + self._DetailedframeworkEventsParser, + self._TBEventsParser, + self._HorovordEventsParser, + self._SMdataparallelEventsParser, + ] + + """ + The following function returns the time range for which the tracefiles are currently available in S3 or local + directory. Users can still query for events for the window greater than this range. In that case, the reader will + query S3 or local directory to check the tracefiles are available. + """ + + def get_current_time_range_for_event_query(self): + timestamps = self._timestamp_to_filename.keys() + return (timestamps[0], timestamps[-1]) if len(timestamps) > 0 else (0, 0) + + """ + Return the tracefiles that were written during the given range. If use_buffer is True, we will consider adding a + buffer of TIME_BUFFER_DEFAULT microseconds to increase the time range. This is done because the events are written to the + file after they end. It is possible that an event would have stared within the window of start and end, however it + did not complete at or before 'end' time. Hence the event will not appear in the tracefile that corresponds to + 'end' timestamp. It will appear in the future event file. + We will also add a buffer for the 'start' i.e. we will look for tracefiles that were written prior to 'start'. + Those files might contain 'B' type events that had started prior to 'start' + """ + + def _get_event_files_in_the_range( + self, start_time_microseconds, end_time_microseconds, use_buffer=True + ): + # increase the time range using TIME_BUFFER_DEFAULT + if use_buffer: + time_buffer = os.getenv(ENV_TIME_BUFFER, TIME_BUFFER_DEFAULT) + start_time_microseconds = start_time_microseconds - time_buffer + end_time_microseconds = end_time_microseconds + time_buffer + + """ + We need to intelligently detect whether we need to refresh the list of available event files. + Approach 1: Keep the start prefix for S3, 'x' minutes (say 5) lagging behind the last available timestamp. + This will cover for the case where a node or writer is not efficient enough to upload the files to S3 + immediately. For local mode we may have to walk the directory every time. + This is currently implemented by computing the start prefix and TRAILING DURATION. + TODO: + Approach 2: If we can know the expected number of files per node and per writer, we can intelligently wait + for that type of file for certain amount of time. + """ + + """ + In case of S3, we will refresh the event file list if the requested end timestamp is less than the timestamp + of _startAfterPrefix. + In case of local mode, the event file list will be refreshed if the end timestamp is not less than the last + available timestamp + """ + + if end_time_microseconds >= self.get_timestamp_of_latest_available_file() or end_time_microseconds >= get_timestamp_from_tracefilename( + self._startAfter_prefix + ): + self.refresh_event_file_list() + + timestamps = sorted(self._timestamp_to_filename.keys()) + + # Find the timestamp that is greater than or equal start_time_microseconds. The tracefile corresponding to + # that timestamp will contain events that are active during start_time_microseconds + lower_bound_timestamp_index = bisect.bisect_left(timestamps, start_time_microseconds) + + # Find the timestamp that is immediate right to the end_time_microseconds. The tracefile corresponding to + # that timestamp will contain events that are active during end_time_microseconds. + upper_bound_timestamp_index = bisect.bisect_left(timestamps, end_time_microseconds) + + event_files = list() + for index in timestamps[lower_bound_timestamp_index : upper_bound_timestamp_index + 1]: + event_files.extend(self._timestamp_to_filename[index]) + self.logger.debug(f"event files to be fetched:{event_files}") + return event_files + + """ + The function returns the right event parser for given file name + 1. For Filename containing 'pythontimeline.json' -> SMEventsParser + 2. For Filename containing 'model_timeline.json' -> SMEventsParser + 3. For Filename containing 'tensorboard' (TBD) -> TensorboardProfilerEvents + 4. For Filename containing 'horovod_timeline.json' -> 'HorovodProfilerEvents + 5. For Filename containing 'smdataparallel_timeline.json' -> 'SMDataParallelProfilerEvents + """ + + def _get_event_parser(self, filename): + if PYTHONTIMELINE_SUFFIX in filename: + return self._PythontimelineEventsParser + if MODELTIMELINE_SUFFIX in filename: + return self._DetailedframeworkEventsParser + if TENSORBOARDTIMELINE_SUFFIX in filename: + return self._TBEventsParser + if HOROVODTIMELINE_SUFFIX in filename: + return self._HorovordEventsParser + if SMDATAPARALLELTIMELINE_SUFFIX in filename: + return self._SMdataparallelEventsParser + + def _get_timestamp_from_filename(self, event_file): + return get_timestamp_from_tracefilename(event_file) + + def _get_event_file_regex(self): + return r"(.+)\.(json|csv|json.gz)$" + + +class LocalAlgorithmMetricsReader(AlgorithmMetricsReader): + """ + The metrics reader is created with root folder in which the tracefiles are stored. + """ + + def __init__(self, trace_root_folder, use_in_memory_cache=False): + self.trace_root_folder = trace_root_folder + super().__init__(use_in_memory_cache) + # Pre-build the file list so that user can query get_timestamp_of_latest_available_file() and get_current_time_range_for_event_query + self.refresh_event_file_list() + + """ + Create a map of timestamp to filename + """ + + def refresh_event_file_list(self): + self.logger.debug(f"Refreshing framework metrics from {self.trace_root_folder}") + self._refresh_event_file_list_local_mode(self.trace_root_folder) + + def parse_event_files(self, event_files): + self._parse_event_files_local_mode(event_files) + + +class S3AlgorithmMetricsReader(AlgorithmMetricsReader): + """ + The s3_trial_path points to a s3 folder in which the tracefiles are stored. e.g. + s3://my_bucket/experiment_base_folder + """ + + def __init__(self, s3_trial_path, use_in_memory_cache=False): + super().__init__(use_in_memory_cache) + s3, bucket_name, base_folder = is_s3(s3_trial_path) + if not s3: + self.logger.error( + "The trial path is expected to be S3 path e.g. s3://bucket_name/trial_folder" + ) + else: + self.bucket_name = bucket_name + self.base_folder = base_folder + self.prefix = os.path.join(self.base_folder, self.prefix, "") + self.logger.info( + f"S3AlgorithmMetricsReader created with bucket:{bucket_name} and prefix:{self.prefix}" + ) + # Pre-build the file list so that user can query get_timestamp_of_latest_available_file() and get_current_time_range_for_event_query + self.refresh_event_file_list() + + """ + The function opens and reads the event files if they are not already parsed. + For S3 metrics reader, we are currently downloading the entire event file. Currently, we will add this file to a + _parsed_file set assuming that the file is complete and it won't be updated on S3 later. However it is + possible that the event file has not reached it's maximum size and will be updated later. + TODO: Check the downloaded size and add the file to _parsed_files only if the file with maximum size is + downloaded and parsed. + """ + + def parse_event_files(self, event_files): + file_read_requests = [] + event_files_to_read = [] + + for event_file in event_files: + if event_file not in self._parsed_files: + self.logger.debug(f"Will request s3 object {event_file}") + event_files_to_read.append(event_file) + file_read_requests.append(ReadObjectRequest(path=event_file)) + + event_data_list = S3Handler.get_objects(file_read_requests) + self.logger.debug(f"Got results back from s3 for {event_files}") + for event_data, event_file in zip(event_data_list, event_files_to_read): + self.logger.debug(f"Will parse events in event file:{event_file}") + if event_file.endswith("json.gz") and is_valid_tfprof_tracefilename(event_file): + self._get_event_parser(event_file).read_events_from_file(event_file) + self._parsed_files.add(event_file) + else: + if is_valid_tracefilename(event_file): + event_string = event_data.decode("utf-8") + json_data = json.loads(event_string) + node_id = get_node_id_from_tracefilename(event_file) + self._get_event_parser(event_file).read_events_from_json_data( + json_data, node_id + ) + self._parsed_files.add(event_file) + else: + self.logger.info(f"Invalid tracefilename:{event_file} . Skipping.") + + """ + Create a map of timestamp to filename + """ + + def refresh_event_file_list(self): + start_after = self._startAfter_prefix if self._startAfter_prefix else self.prefix + self.logger.debug( + f"Making listreq with bucket:{self.bucket_name} prefix:{self.prefix} startAfter:{start_after}" + ) + list_dir = ListRequest(Bucket=self.bucket_name, Prefix=self.prefix, StartAfter=start_after) + self._refresh_event_file_list_s3_mode(list_dir) diff --git a/smdebug/profiler/analysis/__init__.py b/smdebug/profiler/analysis/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/smdebug/profiler/analysis/notebook_utils/__init__.py b/smdebug/profiler/analysis/notebook_utils/__init__.py new file mode 100644 index 0000000000..2290aa0690 --- /dev/null +++ b/smdebug/profiler/analysis/notebook_utils/__init__.py @@ -0,0 +1,4 @@ +# Local +from .metrics_histogram import MetricsHistogram +from .step_histogram import StepHistogram +from .training_job import TrainingJob diff --git a/smdebug/profiler/analysis/notebook_utils/heatmap.py b/smdebug/profiler/analysis/notebook_utils/heatmap.py new file mode 100644 index 0000000000..f63c36c6b9 --- /dev/null +++ b/smdebug/profiler/analysis/notebook_utils/heatmap.py @@ -0,0 +1,228 @@ +# Standard Library +import re + +# Third Party +import bokeh +import numpy as np +from bokeh.io import output_notebook, push_notebook, show +from bokeh.models import ColumnDataSource, HoverTool +from bokeh.models.glyphs import Image +from bokeh.models.tickers import FixedTicker +from bokeh.plotting import figure, show + +output_notebook(hide_banner=True) + + +class Heatmap: + def __init__( + self, + metrics_reader, + select_metrics=[], + starttime=0, + endtime=None, + select_dimensions=[".*CPU", ".*GPU", ".*Memory"], + select_events=[".*"], + plot_height=350, + show_workers=True, + ): + + self.select_dimensions = select_dimensions + self.select_events = select_events + self.show_workers = show_workers + self.metrics_reader = metrics_reader + self.available_dimensions = [] + self.available_events = [] + self.start = 0 # replace with system_metrics_reader.get_first_available_timestamp()/1000000 + + if endtime == None: + # get timestamp of latest file and events + self.last_timestamp_system_metrics = ( + self.metrics_reader.get_timestamp_of_latest_available_file() + ) + else: + self.last_timestamp_system_metrics = endtime + events = self.metrics_reader.get_events(starttime, self.last_timestamp_system_metrics) + + self.plot_height = plot_height + + # get timestamp of latest file and events + self.last_timestamp = self.metrics_reader.get_timestamp_of_latest_available_file() + events = self.metrics_reader.get_events(0, self.last_timestamp) + + self.system_metrics = self.preprocess_system_metrics(events, system_metrics={}) + self.create_plot() + + def preprocess_system_metrics(self, events, system_metrics): + + # read all available system metric events and store them in dict + for event in events: + if self.show_workers is True: + event_unique_id = f"{event.dimension}-nodeid:{str(event.node_id)}" + else: + event_unique_id = event.dimension + if event_unique_id not in system_metrics: + system_metrics[event_unique_id] = {} + self.available_dimensions.append(event_unique_id) + if event.name not in system_metrics[event_unique_id]: + system_metrics[event_unique_id][event.name] = [] + self.available_events.append(event.name) + system_metrics[event_unique_id][event.name].append([event.timestamp, event.value]) + + for dimension in system_metrics: + for event in system_metrics[dimension]: + # convert to numpy + system_metrics[dimension][event] = np.array(system_metrics[dimension][event]) + + # subtract first timestamp + system_metrics[dimension][event][:, 0] = ( + system_metrics[dimension][event][:, 0] - self.start + ) + + # compute total utilization per event dimension + for event_dimension in system_metrics: + n = len(system_metrics[event_dimension]) + total = [sum(x) for x in zip(*system_metrics[event_dimension].values())] + system_metrics[event_dimension]["total"] = np.array(total) / n + self.available_events.append("total") + + self.filtered_events = [] + print(f"select events:{self.select_events}") + self.filtered_dimensions = [] + print(f"select dimensions:{self.select_dimensions}") + for metric in self.select_events: + r = re.compile(r".*" + metric) + self.filtered_events.extend(list(filter(r.search, self.available_events))) + self.filtered_events = set(self.filtered_events) + print(f"filtered_events:{self.filtered_events}") + for metric in self.select_dimensions: + r = re.compile(metric) # + r".*") + self.filtered_dimensions.extend(list(filter(r.search, self.available_dimensions))) + self.filtered_dimensions = set(self.filtered_dimensions) + print(f"filtered_dimensions:{self.filtered_dimensions}") + + return system_metrics + + def create_plot(self): + + # define list of metric names (needed for tooltip) + tmp = [] + metric_names = [] + yaxis = {} + + # number of datapoints + max_width = 0 + for key in self.system_metrics.keys(): + if key.startswith("CPUUtilization"): + width = self.system_metrics[key]["total"].shape[0] + if width >= max_width: + max_width = width + + self.width = max_width + + for dimension in self.filtered_dimensions: + for event in self.filtered_events: + if event in self.system_metrics[dimension]: + values = self.system_metrics[dimension][event][: self.width, 1] + tmp.append(values) + metric_names.append(dimension + "_" + event), + timestamps = self.system_metrics[dimension][event][: self.width, 0] + yaxis[len(tmp)] = dimension + "_" + event + + ymax = len(tmp) + 1 + yaxis[ymax] = "" + + # define figure + start = 0 + if self.width > 1000: + start = self.width - 1000 + self.plot = figure( + plot_height=self.plot_height, + x_range=(start, self.width), + y_range=(0, ymax), + plot_width=1000, + tools="crosshair,reset,xwheel_zoom, box_edit", + ) + self.plot.xaxis.axis_label = "Indices" + # tooltip + hover = HoverTool( + tooltips=[("usage", "@image"), ("metric", "@metric"), ("index", "$x{10}")] + ) + + color_mapper = bokeh.models.LinearColorMapper(bokeh.palettes.viridis(100)) + color_mapper.high = 100 + color_mapper.low = 0 + + tmp = np.array(tmp) + self.source = ColumnDataSource( + data=dict( + image=[np.array(tmp[i]).reshape(1, -1) for i in range(len(tmp))], + x=[0] * ymax, + y=[i for i in range(ymax)], + dw=[self.width] * (ymax), + dh=[1.3] * (ymax), + metric=[i for i in metric_names], + ) + ) + + images = Image(image="image", x="x", y="y", dw="dw", dh="dh", color_mapper=color_mapper) + + # plot + self.plot.add_glyph(self.source, images) + self.plot.add_tools(hover) + self.plot.xgrid.visible = False + self.plot.ygrid.visible = False + self.plot.yaxis.ticker = FixedTicker(ticks=np.arange(0, ymax).tolist()) + self.plot.yaxis.major_label_text_font_size = "5pt" + self.plot.yaxis.major_label_overrides = yaxis + + self.target = show(self.plot, notebook_handle=True) + + def update_data(self, current_timestamp): + + # get all events from last to current timestamp + events = self.metrics_reader.get_events(self.last_timestamp, current_timestamp) + self.last_timestamp = current_timestamp + + if len(events) > 0: + new_system_metrics = self.preprocess_system_metrics(events, system_metrics={}) + + # append numpy arrays to previous numpy arrays + for dimension in self.filtered_dimensions: + for event in self.filtered_events: + if event in self.system_metrics[dimension]: + self.system_metrics[dimension][event] = np.vstack( + [ + self.system_metrics[dimension][event], + new_system_metrics[dimension][event], + ] + ) + max_width = 0 + for key in self.system_metrics.keys(): + if key.startswith("CPUUtilization"): + width = self.system_metrics[key]["cpu0"].shape[0] + if width >= max_width: + max_width = width + + self.width = max_width + + tmp = [] + metric_names = [] + + for dimension in self.filtered_dimensions: + for event in self.filtered_events: + if event in self.system_metrics[dimension]: + values = self.system_metrics[dimension][event][: self.width, 1] + tmp.append(values) + metric_names.append(dimension + "_" + event) + timestamps = self.system_metrics[dimension][event][: self.width, 0] + + # update heatmap + images = [np.array(tmp[i]).reshape(1, -1) for i in range(len(tmp))] + self.source.data["image"] = images + self.source.data["dw"] = [self.width] * (len(tmp) + 1) + self.source.data["metric"] = metric_names + + if self.width > 1000: + self.plot.x_range.start = self.width - 1000 + self.plot.x_range.end = self.width + push_notebook() diff --git a/smdebug/profiler/analysis/notebook_utils/metrics_histogram.py b/smdebug/profiler/analysis/notebook_utils/metrics_histogram.py new file mode 100644 index 0000000000..ebe473d7c7 --- /dev/null +++ b/smdebug/profiler/analysis/notebook_utils/metrics_histogram.py @@ -0,0 +1,167 @@ +# Standard Library +import re + +# Third Party +import numpy as np +from bokeh.io import output_notebook, push_notebook, show +from bokeh.layouts import gridplot +from bokeh.models import ColumnDataSource +from bokeh.plotting import figure, show + +output_notebook(hide_banner=True) + + +class MetricsHistogram: + def __init__(self, metrics_reader): + + self.metrics_reader = metrics_reader + self.system_metrics = {} + self.select_dimensions = [] + self.select_events = [] + self.sources = {} + self.target = None + self.available_dimensions = [] + self.available_events = [] + + """ + @param starttime is starttime_since_epoch_in_micros. Default value 0, which means start + @param endtime is endtime_since_epoch_in_micros. Default value is MetricsHistogram.last_timestamp , i.e., last_timestamp seen by system_metrics_reader + @param select_metrics is array of metrics to be selected, Default ["cpu", "gpu"] + """ + + def plot( + self, + starttime=0, + endtime=None, + select_dimensions=[".*"], + select_events=[".*"], + show_workers=True, + ): + if endtime == None: + endtime = self.metrics_reader.get_timestamp_of_latest_available_file() + all_events = self.metrics_reader.get_events(starttime, endtime) + print( + f"Found {len(all_events)} system metrics events from timestamp_in_us:{starttime} to timestamp_in_us:{endtime}" + ) + self.last_timestamp = endtime + self.select_dimensions = select_dimensions + self.select_events = select_events + self.show_workers = show_workers + self.system_metrics = self.preprocess_system_metrics( + all_events=all_events, system_metrics={} + ) + self.create_plot() + + def clear(self): + self.system_metrics = {} + self.sources = {} + + def preprocess_system_metrics(self, all_events=[], system_metrics={}): + + # read all available system metric events and store them in dict + for event in all_events: + if self.show_workers is True: + event_unique_id = f"{event.dimension}-nodeid:{str(event.node_id)}" + else: + event_unique_id = event.dimension + if event_unique_id not in system_metrics: + system_metrics[event_unique_id] = {} + self.available_dimensions.append(event_unique_id) + if event.name not in system_metrics[event_unique_id]: + system_metrics[event_unique_id][event.name] = [] + self.available_events.append(event.name) + system_metrics[event_unique_id][event.name].append(event.value) + + # compute total utilization per event dimension + for event_dimension in system_metrics: + n = len(system_metrics[event_dimension]) + total = [sum(x) for x in zip(*system_metrics[event_dimension].values())] + system_metrics[event_dimension]["total"] = np.array(total) / n + self.available_events.append("total") + + # add user defined metrics to the list + self.filtered_events = [] + print(f"select events:{self.select_events}") + self.filtered_dimensions = [] + print(f"select dimensions:{self.select_dimensions}") + for metric in self.select_events: + r = re.compile(r".*" + metric + r".*") + self.filtered_events.extend(list(filter(r.search, self.available_events))) + self.filtered_events = set(self.filtered_events) + print(f"filtered_events:{self.filtered_events}") + for metric in self.select_dimensions: + r = re.compile(metric) # + r".*") + self.filtered_dimensions.extend(list(filter(r.search, self.available_dimensions))) + self.filtered_dimensions = set(self.filtered_dimensions) + print(f"filtered_dimensions:{self.filtered_dimensions}") + + return system_metrics + + def _get_probs_binedges(self, values): + # create histogram bins + bins = np.arange(0, 100, 2) + probs, binedges = np.histogram(values, bins=bins) + bincenters = 0.5 * (binedges[1:] + binedges[:-1]) + return probs, binedges + + def create_plot(self): + figures = [] + + # create a histogram per dimension and event + for dimension in self.filtered_dimensions: + self.sources[dimension] = {} + for event in self.filtered_events: + if event in self.system_metrics[dimension]: + p = figure(plot_height=250, plot_width=250) + probs, binedges = self._get_probs_binedges( + self.system_metrics[dimension][event] + ) + # set data + source = ColumnDataSource( + data=dict(top=probs, left=binedges[:-1], right=binedges[1:]) + ) + self.sources[dimension][event] = source + p.quad( + top="top", + bottom=0, + left="left", + right="right", + source=source, + fill_color="navy", + line_color="white", + fill_alpha=0.5, + ) + + # set plot + p.y_range.start = 0 + p.xaxis.axis_label = dimension + "_" + event + p.yaxis.axis_label = "Occurences" + p.grid.grid_line_color = "white" + figures.append(p) + + p = gridplot(figures, ncols=4) + self.target = show(p, notebook_handle=True) + print(f"filtered_dimensions:{self.filtered_dimensions}") + + def update_data(self, current_timestamp): + # get all events from last to current timestamp + events = self.metrics_reader.get_events(self.last_timestamp, current_timestamp) + self.last_timestamp = current_timestamp + + self.system_metrics = self.preprocess_system_metrics(events, self.system_metrics) + + # create a histogram per dimension and event + for dimension in self.filtered_dimensions: + for event in self.filtered_events: + if event in self.system_metrics[dimension]: + values = self.system_metrics[dimension][event] + + # create new histogram bins + probs, binedges = self._get_probs_binedges( + self.system_metrics[dimension][event] + ) + # update data + self.sources[dimension][event].data["top"] = probs + self.sources[dimension][event].data["left"] = binedges[:-1] + self.sources[dimension][event].data["right"] = binedges[1:] + push_notebook() diff --git a/smdebug/profiler/analysis/notebook_utils/step_histogram.py b/smdebug/profiler/analysis/notebook_utils/step_histogram.py new file mode 100644 index 0000000000..216f8a421d --- /dev/null +++ b/smdebug/profiler/analysis/notebook_utils/step_histogram.py @@ -0,0 +1,146 @@ +# Third Party +# Standard Library +import re + +import numpy as np +from bokeh.io import output_notebook, push_notebook, show +from bokeh.layouts import gridplot +from bokeh.models import ColumnDataSource +from bokeh.plotting import figure, show + +output_notebook(hide_banner=True) + +MICROS = 1000000.0 + + +class StepHistogram: + def __init__(self, metrics_reader, width=500): + + self.metrics_reader = metrics_reader + + # get timestamp of latest file and events + self.last_timestamp = self.metrics_reader.get_timestamp_of_latest_available_file() + print(f"StepHistogram created, last_timestamp found:{self.last_timestamp}") + + self.step_metrics = {} + self.sources = {} + + # number of datapoints to plot + self.width = width + + def _get_filtered_list(self, step_metrics): + filtered_metrics = [] + available_metrics = list(step_metrics.keys()) + print(f"Select metrics:{self.select_metrics}") + print(f"Available_metrics: {available_metrics}") + for metric in self.select_metrics: + r = re.compile(metric) + filtered_metrics.extend(list(filter(r.search, available_metrics))) + print(f"Filtered metrics:{filtered_metrics}") + # delete the keys which needs to be filtered out + for key in available_metrics: + if key not in filtered_metrics: + del step_metrics[key] + return step_metrics + + def _create_step_metrics(self, all_events, step_metrics={}): + for event in all_events: + if self.show_workers is True: + event_unique_id = f"{event.event_phase}-nodeid:{str(event.node_id)}" + else: + event_unique_id = event.event_phase + if event_unique_id not in step_metrics: + step_metrics[event_unique_id] = [] + step_metrics[event_unique_id].append(event.duration / MICROS) + step_metrics = self._get_filtered_list(step_metrics) + return step_metrics + + """ + @param starttime: starttime_since_epoch_in_micros . Default vlaue is 0, which means that since start of training + @param endtime: endtime_since_epoch_in_micros. Default value is metrics_reader.last_timestamp, i.e., latest timestamp seen by metrics_reader + @param select_metrics: specifies list of metrics regexes that will be shown + @param show_workers: if this is True, every metrics will be suffixed by node:$WORKER_ID, and for every metrics, graphs will be shown for each worker + """ + + def plot( + self, + starttime=0, + endtime=None, + select_metrics=["Step:ModeKeys", "Forward-node", "Backward\(post-forward\)-node"], + show_workers=True, + ): + if endtime is None: + endtime = self.metrics_reader.get_timestamp_of_latest_available_file() + print(f"stephistogram getting events from {starttime} to {endtime}") + all_events = self.metrics_reader.get_events(starttime, endtime) + print(f"Total events fetched:{len(all_events)}") + self.last_timestamp = endtime + self.select_metrics = select_metrics + self.show_workers = show_workers + self.step_metrics = self._create_step_metrics(all_events) + + self.create_plot(step_metrics=self.step_metrics) + + def clear(): + self.step_metrics = {} + self.sources = {} + + def _get_probs_binedges(self, metrics_arr): + steps_np = np.array(metrics_arr) + values = steps_np[:-1] + min_value = np.min(values) + max_value = np.median(values) + 2 * np.std(values) + + # define histogram bins + bins = np.arange(min_value, max_value, (max_value - min_value) / 50.0) + probs, binedges = np.histogram(steps_np[:-1], bins=bins) + bincenters = 0.5 * (binedges[1:] + binedges[:-1]) + return probs, binedges + + def create_plot(self, step_metrics={}): + metrics = list(step_metrics.keys()) + figures = [] + self.sources = {} + + # create a histogram per metric + for index, metric in enumerate(metrics): + p = figure(plot_height=350, plot_width=450) + # creates bins + probs, binedges = self._get_probs_binedges(step_metrics[metric]) + source = ColumnDataSource(data=dict(top=probs, left=binedges[:-1], right=binedges[1:])) + self.sources[metric] = source + p.quad( + top="top", + bottom=0, + left="left", + right="right", + source=source, + fill_color="navy", + line_color="white", + fill_alpha=0.5, + ) + + # create plot + p.y_range.start = 0 + p.xaxis.axis_label = metric + " step time in ms" + p.yaxis.axis_label = "Occurences" + p.grid.grid_line_color = "white" + figures.append(p) + + p = gridplot(figures, ncols=4) + self.target = show(p, notebook_handle=True) + + def update_data(self, current_timestamp): + # get new events + events = self.metrics_reader.get_events(self.last_timestamp, current_timestamp) + self.last_timestamp = current_timestamp + self.step_metrics = self._create_step_metrics(events, step_metrics=self.step_metrics) + # update histograms + for index, metric in enumerate(self.step_metrics): + if metric in self.sources: + probs, binedges = self._get_probs_binedges(self.step_metrics[metric]) + # update data + self.sources[metric].data["top"] = probs + self.sources[metric].data["left"] = binedges[:-1] + self.sources[metric].data["right"] = binedges[1:] + push_notebook(handle=self.target) diff --git a/smdebug/profiler/analysis/notebook_utils/step_timeline_chart.py b/smdebug/profiler/analysis/notebook_utils/step_timeline_chart.py new file mode 100644 index 0000000000..49092dec74 --- /dev/null +++ b/smdebug/profiler/analysis/notebook_utils/step_timeline_chart.py @@ -0,0 +1,139 @@ +# Third Party +import numpy as np +from bokeh.io import push_notebook, show +from bokeh.models import ColumnDataSource, HoverTool +from bokeh.models.glyphs import Line +from bokeh.plotting import figure, show + + +class StepTimelineChart: + def __init__(self, metrics_reader=None, width=1000): + + self.metrics_reader = metrics_reader + + # get last timestamp + self.last_timestamp = self.metrics_reader.get_timestamp_of_latest_available_file() + + # read timeline + self.metrics_reader.get_events(0, self.last_timestamp) + + self.steps = [] + self.metric_names = [] + # read events for given timerange + events = self.metrics_reader.get_events(0, self.last_timestamp) + + # process events + for event in events: + if event.event_name.startswith("Step"): + self.metric_names.append(event.event_name) + self.steps.append( + [ + event.start_time / 1000000.0, + event.duration / 1000000.0, + int(event.event_args["step_num"]), + ] + ) + # convert to numpy array + self.steps_np = np.array(self.steps) + self.steps_np = self.steps_np[1:, :] + self.start = self.steps_np[0, 0] + self.steps_np = self.steps_np[self.steps_np[:, 0].argsort()] + + # number of datapoints to plot + self.width = width + + # create plot + self.create_plot() + + def create_plot(self): + + # plot either last 500 or last x datapoints + if self.steps_np.shape[0] >= self.width: + start_index = self.width + else: + start_index = self.steps_np.shape[0] - 1 + + # create figure: set the xrange to only show last start_index datapoints + self.plot = figure( + plot_height=250, + plot_width=1000, + x_range=( + self.steps_np[-start_index, 0] - self.start, + self.steps_np[-1, 0] - self.start, + ), + tools="crosshair,pan,reset,save,wheel_zoom", + ) + + # create line chart for step duration + self.source = ColumnDataSource( + data=dict( + x=self.steps_np[:, 0] - self.start, + y=self.steps_np[:, 1], + step=self.steps_np[:, 2], + metric=self.metric_names, + ) + ) + line = Line(x="x", y="y", line_color="blue") + + # tooltip + hover = HoverTool( + tooltips=[ + ("index", "$index"), + ("step", "@step"), + ("metric", "@metric"), + ("(x,y)", "($x, $y)"), + ] + ) + + # create plot + self.plot.add_tools(hover) + self.plot.add_glyph(self.source, line) + self.plot.xaxis.axis_label = "Time in ms" + self.plot.yaxis.axis_label = "Step duration in ms" + self.plot.xgrid.visible = False + self.plot.ygrid.visible = False + + # show + self.target = show(self.plot, notebook_handle=True) + + def update_data(self, current_timestamp): + + # get new events + events = self.metrics_reader.get_events(self.last_timestamp, current_timestamp) + self.last_timestamp = current_timestamp + + if len(events) > 0: + # process new events and append to list + for event in events: + if event.event_name.startswith("Step"): + self.metric_names.append(event.event_name) + self.steps.append( + [ + event.start_time / 1000000.0, + event.duration / 1000000.0, + int(event.event_args["step_num"]), + ] + ) + # convert to numpy + self.steps_np = np.array(self.steps) + self.steps_np = self.steps_np[self.steps_np[:, 0].argsort()] + + # check how many datapoints can be plotted + if self.steps_np.shape[0] > self.width: + start_index = self.width + self.plot.x_range.start = self.steps_np[-self.width, 0] - self.start + self.plot.y_range.end = np.max(self.steps_np[-self.width, 1]) + else: + start_index = self.steps_np.shape[0] - 1 + self.plot.x_range.start = self.steps_np[-start_index, 0] - self.start + self.plot.y_range.end = np.max(self.steps_np[-start_index:, 1]) + + # set new datarange + self.plot.x_range.end = self.steps_np[-1, 0] - self.start + + # update line chart + self.source.data["x"] = self.steps_np[:, 0] - self.start + self.source.data["y"] = self.steps_np[:, 1] + self.source.data["step"] = self.steps_np[:, 2] + self.source.data["metric"] = self.metric_names + push_notebook() diff --git a/smdebug/profiler/analysis/notebook_utils/timeline_charts.py b/smdebug/profiler/analysis/notebook_utils/timeline_charts.py new file mode 100644 index 0000000000..a52112fff1 --- /dev/null +++ b/smdebug/profiler/analysis/notebook_utils/timeline_charts.py @@ -0,0 +1,480 @@ +# Standard Library +import re + +# Third Party +import numpy as np +from bokeh.io import output_notebook, push_notebook, show +from bokeh.layouts import column, row +from bokeh.models import ColumnDataSource, CustomJS, Div, HoverTool +from bokeh.models.glyphs import Circle, Line, Quad +from bokeh.plotting import figure, show + +# First Party +from smdebug.profiler.utils import TimeUnits + +output_notebook(hide_banner=True) + + +class TimelineCharts: + def __init__( + self, + system_metrics_reader, + framework_metrics_reader, + starttime=0, + endtime=None, + select_dimensions=[".*"], + select_events=[".*"], + x=1000, + show_workers=True, + ): + + self.select_dimensions = select_dimensions + self.select_events = select_events + self.show_workers = show_workers + + # placeholder + self.sources = {} + self.available_dimensions = [] + self.available_events = [] + + self.system_metrics_reader = system_metrics_reader + self.framework_metrics_reader = framework_metrics_reader + + if endtime == None: + # get timestamp of latest file and events + self.last_timestamp_system_metrics = ( + self.system_metrics_reader.get_timestamp_of_latest_available_file() + ) + else: + self.last_timestamp_system_metrics = endtime + events = self.system_metrics_reader.get_events( + starttime, self.last_timestamp_system_metrics + ) + # first timestamp + self.start = 0 # replace with system_metrics_reader.get_first_available_timestamp()/1000000 + self.system_metrics = self.preprocess_system_metrics(events, system_metrics={}) + + min_width = float("inf") + for key in self.system_metrics.keys(): + if key.startswith("CPUUtilization"): + width = self.system_metrics[key]["total"].shape[0] + if width <= min_width: + min_width = width + if x < min_width: + self.width = x + else: + self.width = min_width - 1 + + # create plot + self.create_plot() + + def preprocess_system_metrics(self, events, system_metrics): + + # read all available system metric events and store them in dict + for event in events: + if self.show_workers is True: + event_unique_id = f"{event.dimension}-nodeid:{str(event.node_id)}" + else: + event_unique_id = event.dimension + if event_unique_id not in system_metrics: + system_metrics[event_unique_id] = {} + self.available_dimensions.append(event_unique_id) + if event.name not in system_metrics[event_unique_id]: + system_metrics[event_unique_id][event.name] = [] + self.available_events.append(event.name) + system_metrics[event_unique_id][event.name].append([event.timestamp, event.value]) + + for dimension in system_metrics: + for event in system_metrics[dimension]: + # convert to numpy + system_metrics[dimension][event] = np.array(system_metrics[dimension][event]) + + # subtract first timestamp + system_metrics[dimension][event][:, 0] = ( + system_metrics[dimension][event][:, 0] - self.start + ) + + # compute total utilization per event dimension + for event_dimension in system_metrics: + n = len(system_metrics[event_dimension]) + total = [sum(x) for x in zip(*system_metrics[event_dimension].values())] + system_metrics[event_dimension]["total"] = np.array(total) / n + self.available_events.append("total") + + self.filtered_events = [] + print(f"select events:{self.select_events}") + self.filtered_dimensions = [] + print(f"select dimensions:{self.select_dimensions}") + for metric in self.select_events: + r = re.compile(r".*" + metric) + self.filtered_events.extend(list(filter(r.search, self.available_events))) + self.filtered_events = set(self.filtered_events) + print(f"filtered_events:{self.filtered_events}") + for metric in self.select_dimensions: + r = re.compile(metric) # + r".*") + self.filtered_dimensions.extend(list(filter(r.search, self.available_dimensions))) + self.filtered_dimensions = set(self.filtered_dimensions) + print(f"filtered_dimensions:{self.filtered_dimensions}") + return system_metrics + + def plot_system_metrics(self): + + self.figures = [] + x_range = None + # iterate over metrics e.g. cpu usage, gpu usage, i/o reads and writes etc + # create a histogram per dimension and event + for dimension in self.filtered_dimensions: + self.sources[dimension] = {} + for event in self.filtered_events: + if event in self.system_metrics[dimension]: + + values = self.system_metrics[dimension][event] + values = values[values[:, 0].argsort()] + # set y ranges for cpu and gpu which are measured in percent + if "Utilization" in dimension or "Memory" in dimension: + y_range = (0, 102) + else: + y_range = ( + np.min(values[-self.width :, 1]), + np.max(values[-self.width :, 1]), + ) + + # create figure: each system metric has its own figure + + if x_range == None: + plot = figure( + plot_height=200, + plot_width=1000, + x_range=(values[-self.width, 0], values[-1, 0]), + y_range=y_range, + tools="crosshair,xbox_select,pan,reset,save,xwheel_zoom", + ) + x_range = plot.x_range + else: + plot = figure( + plot_height=200, + plot_width=1000, + x_range=x_range, + y_range=y_range, + tools="crosshair,xbox_select,pan,reset,save,xwheel_zoom", + ) + + plot.xgrid.visible = False + plot.ygrid.visible = False + + # create line chart for system metric + source = ColumnDataSource(data=dict(x=values[:, 0], y=values[:, 1])) + + callback = CustomJS( + args=dict(s1=source, div=self.div), + code=""" + console.log('Running CustomJS callback now.'); + var inds = s1.selected.indices; + console.log(inds); + var line = " Selected index range: [" + Math.min.apply(Math,inds) + "," + Math.max.apply(Math,inds) + "]\\n"; + console.log(line) + var text = div.text.concat(line); + var lines = text.split("\\n") + if (lines.length > 35) + lines.shift(); + div.text = lines.join("\\n");""", + ) + + plot.js_on_event("selectiongeometry", callback) + + line = Line(x="x", y="y", line_color="blue") + circle = Circle(x="x", y="y", fill_alpha=0, line_width=0) + p = plot.add_glyph(source, line) + p = plot.add_glyph(source, circle) + + # create tooltip for hover tool + hover = HoverTool( + renderers=[p], tooltips=[("index", "$index"), ("(x,y)", "($x, $y)")] + ) + + plot.xaxis.axis_label = "Time in ms" + plot.yaxis.axis_label = dimension + "_" + event + plot.add_tools(hover) + + # store figure and datasource + self.figures.append(plot) + self.sources[dimension][event] = source + + return self.figures + + def create_plot(self): + + self.div = Div(width=250, height=100, height_policy="fixed") + figures = self.plot_system_metrics() + p = column(figures) + self.target = show(row(p, self.div), notebook_handle=True) + + def find_time_annotations(self, indexes): + + if len(indexes) > 0: + cpu_util = None + for key in self.system_metrics.keys(): + if key.startswith("CPUUtilization"): + width = self.system_metrics[key]["total"].shape[0] + if cpu_util is None or np.min(indexes) <= width <= np.max(indexes): + cpu_util = self.system_metrics[key] + + begin_timestamp = cpu_util["total"][np.min(indexes), 0] + end_timestamp = cpu_util["total"][np.max(indexes), 0] + total_time = end_timestamp - begin_timestamp + print( + f"Selected timerange: {begin_timestamp + self.start} to {end_timestamp + self.start}" + ) + + cumulative_time = {} + events = self.framework_metrics_reader.get_events( + begin_timestamp + self.start, end_timestamp + self.start, unit=TimeUnits.SECONDS + ) + for event in events: + if event.event_args is not None and "step_num" in event.event_args: + key = event.event_name + "_" + str(event.event_args["step_num"]) + else: + key = event.event_name + if key not in cumulative_time: + cumulative_time[key] = 0 + + cumulative_time[key] += event.duration / 1000000000.0 + + for event_name in cumulative_time: + print(f"Spent {cumulative_time[event_name]} ms (cumulative time) in {event_name}") + else: + print("No selection made") + + def plot_framework_events(self, events, begin_timestamp, end_timestamp): + framework_events = {} + yaxis = {} + counter = 0 + for index, event in enumerate(events): + if event.event_args is not None: + if "bytes_fetched" in event.event_args or "worker_id" in event.event_args: + continue + + if event.event_phase not in framework_events: + framework_events[event.event_phase] = [] + yaxis[event.event_phase] = counter + counter += 1 + + framework_events[event.event_phase].append( + [ + int(event.start_time / 1000.0), + int(event.end_time / 1000.0), + yaxis[event.event_phase], + ] + ) + if index > 500: + print( + """Reached more than 500 datapoints. + Will only plot first 500 datapoints for the given timerange""" + ) + break + return framework_events + + def plot_dataloaders(self, events, begin_timestamp, end_timestamp): + # read all available system metric events and store them in dict + dataloaders = {} + tids = {} + + for index, event in enumerate(events): + if event.event_args is None: + continue + if "bytes_fetched" in event.event_args: + if event.event_name not in dataloaders: + dataloaders[event.event_name] = [] + if event.tid not in tids: + tids[event.tid] = len(tids.keys()) + dataloaders[event.event_name].append( + [int(event.start_time / 1000.0), int(event.end_time / 1000.0), tids[event.tid]] + ) + elif "worker_id" in event.event_args: + if event.event_name not in dataloaders: + dataloaders[event.event_name] = [] + worker_id = event.event_args["worker_id"] + if worker_id not in tids: + tids[worker_id] = len(tids.keys()) + dataloaders[event.event_name].append( + [ + int(event.start_time / 1000.0), + int(event.end_time / 1000.0), + tids[event.event_args["worker_id"]], + ] + ) + + if index > 500: + print("Reached more than 500 datapoints. Will stop plotting.") + break + + return dataloaders + + def plot_detailed_profiler_data(self, indexes): + + if len(indexes) > 0: + cpu_util = None + for key in self.system_metrics.keys(): + if key.startswith("CPUUtilization"): + width = self.system_metrics[key]["cpu0"].shape[0] + if cpu_util is None or np.min(indexes) <= width <= np.max(indexes): + cpu_util = self.system_metrics[key] + + begin_timestamp = cpu_util["cpu0"][np.min(indexes), 0] + end_timestamp = cpu_util["cpu0"][np.max(indexes), 0] + print( + f"Selected timerange: {begin_timestamp + self.start} to {end_timestamp + self.start}" + ) + events = self.framework_metrics_reader.get_events( + begin_timestamp + self.start, end_timestamp + self.start, unit=TimeUnits.SECONDS + ) + + dataloaders = self.plot_dataloaders( + events, begin_timestamp + self.start, end_timestamp + self.start + ) + framework_events = self.plot_framework_events( + events, begin_timestamp + self.start, end_timestamp + self.start + ) + + # define figure + plot_dataloaders = figure( + plot_height=250, + plot_width=1000, + tools="crosshair,xbox_select,pan,reset,save,xwheel_zoom", + ) + plot_dataloaders.xaxis.axis_label = "Time in ms" + plot_dataloaders.yaxis.axis_label = "Thread ID" + # tooltip + hover = HoverTool(tooltips=[("metric", "@metric"), ("index", "$x{10}")]) + + for event in dataloaders.keys(): + for entry in range(len(dataloaders[event])): + # create source that contains time annotations + source = ColumnDataSource( + data=dict( + top=[dataloaders[event][entry][2]], + bottom=[dataloaders[event][entry][2] - 1], + left=[dataloaders[event][entry][0]], + right=[dataloaders[event][entry][1]], + metric=[event], + ) + ) + + # vertical bars + quad = Quad( + top="top", + bottom="bottom", + left="left", + right="right", + fill_color="black", + line_color=None, + fill_alpha=0.2, + ) + + # plot + plot_dataloaders.add_glyph(source, quad) + plot_dataloaders.add_tools(hover) + + plot_framework_events = figure( + plot_height=250, + plot_width=1000, + tools="crosshair,xbox_select,pan,reset,save,xwheel_zoom", + ) + plot_framework_events.xaxis.axis_label = "Time in ms" + plot_framework_events.yaxis.axis_label = "Framework metric" + # tooltip + hover = HoverTool(tooltips=[("metric", "@metric"), ("index", "$x{10}")]) + + for event in framework_events.keys(): + for entry in range(len(framework_events[event])): + # create source that contains time annotations + source = ColumnDataSource( + data=dict( + top=[framework_events[event][entry][2]], + bottom=[framework_events[event][entry][2] - 1], + left=[framework_events[event][entry][0]], + right=[framework_events[event][entry][1]], + metric=[event], + ) + ) + + # vertical bars + quad = Quad( + top="top", + bottom="bottom", + left="left", + right="right", + fill_color="blue", + fill_alpha=0.2, + line_color=None, + ) + + # plot + plot_framework_events.add_glyph(source, quad) + plot_framework_events.add_tools(hover) + + p = column([plot_dataloaders, plot_framework_events]) + self.target = show(p, notebook_handle=True) + else: + print("No selection made") + + def update_data(self, current_timestamp): + + # get all events from last to current timestamp + events = self.system_metrics_reader.get_events( + self.last_timestamp_system_metrics, current_timestamp + ) + print( + f"Found {len(events)} new system metrics events from timestamp_in_us:{self.last_timestamp_system_metrics} to timestamp_in_us:{current_timestamp}" + ) + if len(events) > 0: + new_system_metrics = self.preprocess_system_metrics(events, system_metrics={}) + + # append numpy arrays to previous numpy arrays + for dimension in self.filtered_dimensions: + for event in self.filtered_events: + if event in self.system_metrics[dimension]: + new_system_metrics[dimension][event] = new_system_metrics[dimension][event][ + new_system_metrics[dimension][event][:, 0].argsort() + ] + self.system_metrics[dimension][event] = np.vstack( + [ + self.system_metrics[dimension][event], + new_system_metrics[dimension][event], + ] + ) + self.system_metrics[dimension][event] = self.system_metrics[dimension][ + event + ][self.system_metrics[dimension][event][:, 0].argsort()] + + max_width = 0 + cpu_util = None + for key in self.system_metrics.keys(): + if key.startswith("CPUUtilization"): + width = self.system_metrics[key]["total"].shape[0] + if cpu_util is None or width >= max_width: + max_width = width + cpu_util = self.system_metrics[key] + + self.width = max_width - 1 + + if self.width > 1000: + min_value = cpu_util["total"][-1000, 0] + else: + min_value = cpu_util["total"][-self.width, 0] + max_value = cpu_util["total"][-1, 0] + + for figure in self.figures: + figure.x_range.start = int(min_value) + figure.x_range.end = int(max_value) + + # update line charts with system metrics + for dimension in self.filtered_dimensions: + for event in self.filtered_events: + if event in self.system_metrics[dimension]: + values = np.array(self.system_metrics[dimension][event]) + self.sources[dimension][event].data["x"] = values[:, 0] + self.sources[dimension][event].data["y"] = values[:, 1] + + self.last_timestamp_system_metrics = current_timestamp + push_notebook() diff --git a/smdebug/profiler/analysis/notebook_utils/training_job.py b/smdebug/profiler/analysis/notebook_utils/training_job.py new file mode 100644 index 0000000000..5638a96d9d --- /dev/null +++ b/smdebug/profiler/analysis/notebook_utils/training_job.py @@ -0,0 +1,97 @@ +# Standard Library +import os +import time + +# Third Party +import boto3 + +# First Party +from smdebug.profiler.algorithm_metrics_reader import S3AlgorithmMetricsReader +from smdebug.profiler.system_metrics_reader import S3SystemMetricsReader +from smdebug.profiler.utils import us_since_epoch_to_human_readable_time + + +class TrainingJob: + def __init__(self, training_job_name, region=None): + self.name = training_job_name + self.profiler_config = None + self.profiler_s3_output_path = None + if region is None: + self.sm_client = boto3.client("sagemaker") + else: + self.sm_client = boto3.client("sagemaker", region_name=region) + self.profiler_config, self.profiler_s3_output_path = ( + self.get_config_and_profiler_s3_output_path() + ) + self.system_metrics_reader = None + self.framework_metrics_reader = None + + def get_systems_metrics_reader(self): + return self.system_metrics_reader + + def get_framework_metrics_reader(self): + return self.framework_metrics_reader + + def get_sm_client(self): + return self.sm_client + + def get_config_and_profiler_s3_output_path(self): + if self.profiler_config is None: + + self.ds = self.get_sm_client().describe_training_job(TrainingJobName=self.name) + attempt = 0 + while attempt < 60: + if "ProfilerConfig" in self.ds: + pc = self.ds["ProfilerConfig"] + if "S3OutputPath" in pc: + self.profiler_config = pc + self.profiler_s3_output_path = os.path.join( + pc["S3OutputPath"], self.name, "profiler-output" + ) + break + attempt += 1 + print(f"ProfilerConfig:{self.profiler_config}") + print(f"s3 path:{self.profiler_s3_output_path}") + return self.profiler_config, self.profiler_s3_output_path + + def wait_for_sys_profiling_data_to_be_available(self): + self.system_metrics_reader = S3SystemMetricsReader(self.profiler_s3_output_path) + last_job_status = "" + last_secondary_status = "" + while self.system_metrics_reader.get_timestamp_of_latest_available_file() == 0: + print("Profiler data from system not available yet") + self.system_metrics_reader.refresh_event_file_list() + p = self.describe_training_job() + + if "TrainingJobStatus" in p: + status = p["TrainingJobStatus"] + if "SecondaryStatus" in p: + secondary_status = p["SecondaryStatus"] + if last_job_status != status or last_secondary_status != secondary_status: + print( + f"time: {time.time()} TrainingJobStatus:{status} TrainingJobSecondaryStatus:{secondary_status}" + ) + last_job_status = status + last_secondary_status = secondary_status + time.sleep(10) + + print("\n\nProfiler data from system is available") + + def wait_for_framework_profiling_data_to_be_available(self): + self.framework_metrics_reader = S3AlgorithmMetricsReader(self.profiler_s3_output_path) + + events = [] + while self.framework_metrics_reader.get_timestamp_of_latest_available_file() == 0: + print("Profiler data from framework not available yet") + self.framework_metrics_reader.refresh_event_file_list() + time.sleep(10) + + print("\n\n Profiler data from framework is available") + last_timestamp = self.framework_metrics_reader.get_timestamp_of_latest_available_file() + print( + f"Found recorded framework annotations. Latest available timestamp microsseconds_since_epoch is:{last_timestamp} , human_readable_timestamp in utc:", + us_since_epoch_to_human_readable_time(last_timestamp), + ) + + def describe_training_job(self): + return self.get_sm_client().describe_training_job(TrainingJobName=self.name) diff --git a/smdebug/profiler/analysis/python_profile_analysis.py b/smdebug/profiler/analysis/python_profile_analysis.py new file mode 100644 index 0000000000..215da77212 --- /dev/null +++ b/smdebug/profiler/analysis/python_profile_analysis.py @@ -0,0 +1,313 @@ +# Standard Library +import json +import os +import pstats +from collections import defaultdict + +# Third Party +import pandas as pd + +# First Party +from smdebug.core.logger import get_logger +from smdebug.profiler.analysis.python_stats_reader import ( + LocalPythonStatsReader, + S3PythonStatsReader, +) +from smdebug.profiler.analysis.utils.python_profile_analysis_utils import ( + PyinstrumentStepStats, + StepPhase, + StepPythonProfileStats, + cProfileStats, +) +from smdebug.profiler.profiler_constants import ( + CONVERT_TO_MICROSECS, + CPROFILE_NAME, + PYINSTRUMENT_NAME, +) +from smdebug.profiler.python_profile_utils import PythonProfileModes, python_profile_mode_to_str + + +class PythonProfileAnalysis: + name = "" # placeholder + + def __init__(self, local_profile_dir="/tmp/python_stats", s3_path=None): + """Analysis class that takes in path to the profile directory, and sets up the python stats reader, which + fetches metadata of the python profiling done for each step. Also provides functions for analysis on this + profiling, such as fetching stats by a specific step or time interval. + + If s3_path is provided, the S3PythonStatsReader is used and local_profile_dir will represent the local + directory path that the reader will create the stats directory and then download the stats to. + Otherwise, LocalPythonStatsReader is used and local_profile_dir represents the path to the stats directory, + which already holds the stats. + + ... + + Attributes + ---------- + python_stats_reader: PythonStatsReader + The reader to use for loading the python stats. + python_profile_stats: list of StepPythonProfileStats + List of stats for each step profiled. + """ + self.python_stats_reader = ( + S3PythonStatsReader(local_profile_dir, s3_path) + if s3_path + else LocalPythonStatsReader(local_profile_dir) + ) + self._refresh_python_profile_stats(True) + + def _refresh_python_profile_stats(self, refresh_stats): + """Helper function to load in the most recent python stats via the python stats reader. + """ + if refresh_stats: + get_logger().info("Refreshing python profile stats.") + self.python_profile_stats = self.python_stats_reader.load_python_profile_stats() + + def _fetch_profile_stats_by_node_id(self, node_id): + """Helper function to filter profile stats by node ID. If no specific node ID is provided, pick the first + stats object's node ID. + """ + if len(self.python_profile_stats) == 0: + return [] + if node_id == "any": + node_id = self.python_profile_stats[0].node_id + return [ + step_stats for step_stats in self.python_profile_stats if step_stats.node_id == node_id + ] + + def _aggregate_stats(self, requested_stats): + """ + Helper function to the requested stats by the user. To be overriden in the subclass as this is profiler + dependent. + """ + return requested_stats # placeholder + + def fetch_profile_stats_by_step( + self, + start_step, + end_step=None, + mode=PythonProfileModes.TRAIN, + start_phase=StepPhase.STEP_START, + end_phase=StepPhase.STEP_END, + node_id="any", + refresh_stats=True, + ): + """API function to fetch stats based on step interval. + """ + self._refresh_python_profile_stats(refresh_stats) + + if end_step is None: + end_step = start_step + + requested_stats = [ + step_stats + for step_stats in self._fetch_profile_stats_by_node_id(node_id) + if step_stats.in_step_interval(start_step, end_step, start_phase, end_phase) + and step_stats.has_start_and_end_mode(mode, mode) + ] + return self._aggregate_stats(requested_stats) + + def fetch_profile_stats_by_time( + self, + start_time_since_epoch_in_secs, + end_time_since_epoch_in_secs, + node_id="any", + refresh_stats=True, + ): + """API function to fetch stats based on time interval. + """ + self._refresh_python_profile_stats(refresh_stats) + start_time_since_epoch_in_micros = start_time_since_epoch_in_secs * CONVERT_TO_MICROSECS + end_time_since_epoch_in_micros = end_time_since_epoch_in_secs * CONVERT_TO_MICROSECS + requested_stats = [ + step_stats + for step_stats in self._fetch_profile_stats_by_node_id(node_id) + if step_stats.in_time_interval( + start_time_since_epoch_in_micros, end_time_since_epoch_in_micros + ) + ] + return self._aggregate_stats(requested_stats) + + def fetch_profile_stats_between_modes( + self, start_mode, end_mode, node_id="any", refresh_stats=True + ): + """API function that fetches stats with the provided start and end mode. + """ + self._refresh_python_profile_stats(refresh_stats) + requested_stats = [ + step_stats + for step_stats in self._fetch_profile_stats_by_node_id(node_id) + if step_stats.has_start_and_end_mode(start_mode, end_mode) + ] + return self._aggregate_stats(requested_stats) + + def fetch_pre_step_zero_profile_stats(self, node_id="any", refresh_stats=True): + """API function that fetches stats from profiling until step 0. + """ + self._refresh_python_profile_stats(refresh_stats) + requested_stats = [ + step_stats + for step_stats in self._fetch_profile_stats_by_node_id(node_id) + if step_stats.has_pre_step_zero_profile_stats() + ] + return self._aggregate_stats(requested_stats) + + def fetch_post_hook_close_profile_stats(self, node_id="any", refresh_stats=True): + """API function that fetches stats from profiling after the hook is closed. + """ + self._refresh_python_profile_stats(refresh_stats) + requested_stats = [ + step_stats + for step_stats in self._fetch_profile_stats_by_node_id(node_id) + if step_stats.has_post_hook_close_profile_stats() + ] + return self._aggregate_stats(requested_stats) + + def list_profile_stats(self, refresh_stats=True): + """API function that returns a DataFrame of the python profile stats, where each row holds the metadata for + each instance of profiling and the corresponding stats file (one per step). + + The columns of this DataFrame include: + - profiler_name: The name of the profiler used to generate this stats file, cProfile or pyinstrument + - framework: The machine learning framework used in training. + - start_time_since_epoch_in_micros: The UTC time (in microseconds) at which profiling started for this step. + - end_time_since_epoch_in_micros: The UTC time (in microseconds) at which profiling finished for this step. + = node_id The node ID of the node used in the session. + - start_phase The phase at which python profiling was started. + - start_step: The step at which python profiling was started. -1 if before step 0. + - end_phase The phase at which python profiling was stopped. + - end_step: The step at which python profiling was stopped. + - stats_path The path to the dumped python stats resulting from profiling this step. + """ + self._refresh_python_profile_stats(refresh_stats) + if len(self.python_profile_stats) == 0: + print("No stats were found!") + return pd.DataFrame() + df = pd.DataFrame(list(map(lambda x: vars(x), self.python_profile_stats))) + df["start_mode"] = df["start_mode"].map(python_profile_mode_to_str) + df["end_mode"] = df["end_mode"].map(python_profile_mode_to_str) + return df + + def list_available_node_ids(self, refresh_stats=True): + """API function to list the available node IDs we have python profiling stats for. + """ + self._refresh_python_profile_stats(refresh_stats) + all_node_ids = map(lambda x: x.node_id, self.python_profile_stats) + unique_node_ids = list(set(all_node_ids)) + unique_node_ids.sort() + return unique_node_ids + + +class cProfileAnalysis(PythonProfileAnalysis): + """Analysis class used specifically for python profiling with cProfile + """ + + name = CPROFILE_NAME + + def _refresh_python_profile_stats(self, refresh_stats): + """Helper function to load in the most recent python stats via the python stats reader. + Filters out any stats not generated by cProfile. + """ + super()._refresh_python_profile_stats(refresh_stats) + self.python_profile_stats = list( + filter(lambda x: x.profiler_name == CPROFILE_NAME, self.python_profile_stats) + ) + + def _aggregate_stats(self, stats, refresh_stats=True): + """Aggregate the stats files into one pStats.Stats object corresponding to the requested interval. + Then returns a `cProfileStats` object (which holds the pStats.Stats object and parsed stats for each called + function in these steps). + """ + ps = pstats.Stats() + + if len(stats) == 0: + print("No stats were found for the requested interval!") + return None + + for step_stats in stats: + ps.add(step_stats.stats_path) + return cProfileStats(ps) + + def fetch_profile_stats_by_training_phase(self, node_id="any", refresh_stats=True): + """API function that fetches and aggregates stats for every possible combination of start and end mode. + + For example, if training and validation are done while detailed profiling is enabled, the combinations are: + - (PRE_STEP_ZERO, TRAIN) + - (TRAIN, TRAIN) + - (TRAIN, EVAL) + - (EVAL, EVAL) + - (EVAL, POST_HOOK_CLOSE) + + All stats files within each of these combinations are aggregated. + """ + self._refresh_python_profile_stats(refresh_stats) + training_phase_stats = {} + for start_mode in PythonProfileModes: + for end_mode in PythonProfileModes: + step_stats = self.fetch_profile_stats_between_modes( + start_mode, end_mode, node_id=node_id, refresh_stats=False + ) + if step_stats is not None: + training_phase_stats[(start_mode, end_mode)] = step_stats + return training_phase_stats + + def fetch_profile_stats_by_job_phase(self, node_id="any", refresh_stats=True): + """API function that fetches and aggregates stats by job phase. + + The job phases are: + - initialization: profiling until step 0 (pre-step zero profiling) + - training loop: training and validation + - finalization: profiling after the hook is closed (post-hook-close profiling) + """ + self._refresh_python_profile_stats(refresh_stats) + training_phase_stats = self.fetch_profile_stats_by_training_phase( + node_id=node_id, refresh_stats=False + ) + job_phase_stats = {} + ps = pstats.Stats() + for (start_mode, end_mode), step_stats in training_phase_stats.items(): + if start_mode == PythonProfileModes.PRE_STEP_ZERO: + job_phase_stats["initialization"] = step_stats + elif end_mode == PythonProfileModes.POST_HOOK_CLOSE: + job_phase_stats["finalization"] = step_stats + else: + ps.add(step_stats.ps) + job_phase_stats["training_loop"] = cProfileStats(ps) + return job_phase_stats + + +class PyinstrumentAnalysis(PythonProfileAnalysis): + """Analysis class used specifically for python profiling with pyinstrument. + """ + + name = PYINSTRUMENT_NAME + + def _refresh_python_profile_stats(self, refresh_stats): + """Helper function to load in the most recent python stats via the python stats reader. + Filters out any stats not generated by pyinstrument. + """ + super()._refresh_python_profile_stats(refresh_stats) + self.python_profile_stats = list( + filter(lambda x: x.profiler_name == PYINSTRUMENT_NAME, self.python_profile_stats) + ) + + def _aggregate_stats(self, stats): + """Load and return a list of dictionaries corresponding to each step's stats file. + """ + if len(stats) == 0: + print("No stats were found for the requested interval!") + return None + + aggregated_stats_dict = defaultdict(lambda: []) + for step_stats in stats: + aggregated_stats_dict[step_stats.start_time_since_epoch_in_micros].append(step_stats) + + aggregated_stats = [] + for aggregated_step_stats in aggregated_stats_dict.values(): + aggregated_step_stats.sort(key=lambda x: os.path.basename(x.stats_path)) + html_step_stats, json_step_stats = aggregated_step_stats + with open(json_step_stats.stats_path, "r") as json_data: + json_stats = json.load(json_data) + aggregated_stats.append(PyinstrumentStepStats(html_step_stats.stats_path, json_stats)) + + return aggregated_stats diff --git a/smdebug/profiler/analysis/python_stats_reader.py b/smdebug/profiler/analysis/python_stats_reader.py new file mode 100644 index 0000000000..ca5dda9f0c --- /dev/null +++ b/smdebug/profiler/analysis/python_stats_reader.py @@ -0,0 +1,158 @@ +# Standard Library +import os +import shutil + +# First Party +from smdebug.core.access_layer.s3handler import ListRequest, ReadObjectRequest, S3Handler, is_s3 +from smdebug.core.logger import get_logger +from smdebug.profiler.analysis.utils.python_profile_analysis_utils import StepPythonProfileStats +from smdebug.profiler.profiler_constants import ( + CPROFILE_NAME, + CPROFILE_STATS_FILENAME, + PYINSTRUMENT_HTML_FILENAME, + PYINSTRUMENT_JSON_FILENAME, + PYINSTRUMENT_NAME, +) + + +class PythonStatsReader: + """Basic framework for stats reader to retrieve stats from python profiling + """ + + def __init__(self, profile_dir): + """ + :param profile_dir: The path to the directory where the python profile stats are. + """ + self.profile_dir = profile_dir + + def load_python_profile_stats(self): + """Load the python profile stats. To be implemented in subclass. + """ + + +class S3PythonStatsReader(PythonStatsReader): + """Higher level stats reader to download python stats from s3. + """ + + def __init__(self, profile_dir, s3_path): + """ + :param profile_dir: The path to the directory where the profile directory is created. The stats will then + be downloaded to this newly created directory. + :param s3_path: The path in s3 to the base folder of the logs. + """ + assert os.path.isdir(profile_dir), "The provided profile directory does not exist!" + super().__init__(os.path.join(profile_dir, "python_stats")) + self._validate_s3_path(s3_path) + + def _set_up_profile_dir(self): + """Recreate the profile directory, clearing any files that were in it. + """ + shutil.rmtree(self.profile_dir, ignore_errors=True) + os.makedirs(self.profile_dir) + + def _validate_s3_path(self, s3_path): + """Validate the provided s3 path and set the bucket name and prefix. + :param s3_path: The path in s3 to the base folder of the logs. + """ + s3, bucket_name, base_folder = is_s3(s3_path) + assert s3, "The provided s3 path should have the following format: s3://bucket_name/..." + self.bucket_name = bucket_name + self.prefix = os.path.join(base_folder, "framework") + + def load_python_profile_stats(self): + """Load the stats in by creating the profile directory, downloading each stats directory from s3 to the + profile directory, parsing the metadata from each stats directory name and creating a StepPythonProfileStats + entry corresponding to the stats file in the stats directory. + + For cProfile, the stats file name is `python_stats`. + For pyinstrument, the stats file name `python_stats.json`. + """ + python_profile_stats = [] + + self._set_up_profile_dir() + + list_request = ListRequest(Bucket=self.bucket_name, Prefix=self.prefix) + s3_filepaths = S3Handler.list_prefix(list_request) + object_requests = [ + ReadObjectRequest(os.path.join("s3://", self.bucket_name, s3_filepath)) + for s3_filepath in s3_filepaths + ] + objects = S3Handler.get_objects(object_requests) + + for full_s3_filepath, object_data in zip(s3_filepaths, objects): + if os.path.basename(full_s3_filepath) not in ( + CPROFILE_STATS_FILENAME, + PYINSTRUMENT_JSON_FILENAME, + PYINSTRUMENT_HTML_FILENAME, + ): + get_logger().info(f"Unknown file {full_s3_filepath} found, skipping...") + continue + + path_components = full_s3_filepath.split("/") + framework, profiler_name, node_id, stats_dir, stats_file = path_components[-5:] + + stats_dir_path = os.path.join(self.profile_dir, node_id, stats_dir) + os.makedirs(stats_dir_path, exist_ok=True) + stats_file_path = os.path.join(stats_dir_path, stats_file) + + with open(stats_file_path, "wb") as f: + f.write(object_data) + + python_profile_stats.append( + StepPythonProfileStats( + framework, profiler_name, node_id, stats_dir, stats_file_path + ) + ) + python_profile_stats.sort( + key=lambda x: (x.start_time_since_epoch_in_micros, x.node_id) + ) # sort each step's stats by the step number, then node ID. + return python_profile_stats + + +class LocalPythonStatsReader(PythonStatsReader): + """Higher level stats reader to load the python stats locally. + """ + + def __init__(self, profile_dir): + """ + :param profile_dir: The path to the directory where the python profile stats are. + """ + assert os.path.isdir(profile_dir), "The provided stats directory does not exist!" + super().__init__(profile_dir) + + def load_python_profile_stats(self): + """Load the stats in by scanning each stats directory in the profile directory, parsing the metadata from the + stats directory name and creating a StepPythonProfileStats entry corresponding to the stats file in the + stats directory. + + For cProfile, the stats file name is `python_stats`. + For pyinstrument, the stats file name `python_stats.json` or `python_stats.html`. + """ + python_profile_stats = [] + framework = os.path.basename(os.path.dirname(self.profile_dir)) + for node_id in os.listdir(self.profile_dir): + node_dir_path = os.path.join(self.profile_dir, node_id) + for stats_dir in os.listdir(node_dir_path): + stats_dir_path = os.path.join(node_dir_path, stats_dir) + for filename in os.listdir(stats_dir_path): + if filename == CPROFILE_STATS_FILENAME: + profiler_name = CPROFILE_NAME + stats_file_path = os.path.join(stats_dir_path, CPROFILE_STATS_FILENAME) + elif filename == PYINSTRUMENT_JSON_FILENAME: + profiler_name = PYINSTRUMENT_NAME + stats_file_path = os.path.join(stats_dir_path, PYINSTRUMENT_JSON_FILENAME) + elif filename == PYINSTRUMENT_HTML_FILENAME: + profiler_name = PYINSTRUMENT_NAME + stats_file_path = os.path.join(stats_dir_path, PYINSTRUMENT_HTML_FILENAME) + else: + get_logger().info(f"Unknown file {filename} found, skipping...") + continue + python_profile_stats.append( + StepPythonProfileStats( + framework, profiler_name, node_id, stats_dir, stats_file_path + ) + ) + python_profile_stats.sort( + key=lambda x: (x.start_time_since_epoch_in_micros, x.node_id) + ) # sort each step's stats by the step number, then node ID. + return python_profile_stats diff --git a/smdebug/profiler/analysis/utils/__init__.py b/smdebug/profiler/analysis/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/smdebug/profiler/analysis/utils/merge_timelines.py b/smdebug/profiler/analysis/utils/merge_timelines.py new file mode 100644 index 0000000000..34f00b1acd --- /dev/null +++ b/smdebug/profiler/analysis/utils/merge_timelines.py @@ -0,0 +1,355 @@ +# Standard Library +import collections +import json +from enum import Enum + +# First Party +from smdebug.core.access_layer.file import TSAccessFile +from smdebug.core.access_layer.s3 import TSAccessS3 +from smdebug.core.locations import TraceFileLocation +from smdebug.core.logger import get_logger +from smdebug.core.utils import is_s3 +from smdebug.profiler.algorithm_metrics_reader import ( + LocalAlgorithmMetricsReader, + S3AlgorithmMetricsReader, +) +from smdebug.profiler.analysis.utils.profiler_data_to_pandas import PandasFrame +from smdebug.profiler.trace_event_file_parser import TraceEvent + +logger = get_logger() + + +class MergeUnit(Enum): + """ + Enum to get Merge Unit - time or step. + """ + + # merge by time interval + TIME = "time" + + # merge by step interval + STEP = "step" + + +class MergedTimeline: + def __init__(self, path, file_suffix_filter=None, output_directory=None): + """ + :param path: trace root folder that contains framework and system folders + :param file_suffix_filter: list of file suffix + PYTHONTIMELINE_SUFFIX = "pythontimeline.json" + MODELTIMELINE_SUFFIX = "model_timeline.json" + TENSORBOARDTIMELINE_SUFFIX = "trace.json.gz" + HOROVODTIMELINE_SUFFIX = "horovod_timeline.json" + SMDATAPARALLELTIMELINE_SUFFIX = "smdataparallel_timeline.json". + Default: None (all files will be merged) + :param output_directory: Path where merged file should be saved + Default: None (writes to the same location as the 'path' argument. + """ + self.path = path + self.out_dir = output_directory if output_directory is not None else self.path + self.file_suffix_filter = file_suffix_filter + + self.bytes_written = 0 + + # Reader for system and framework metrics + if path.startswith("s3"): + self.framework_metrics_reader = S3AlgorithmMetricsReader(self.path) + else: + self.framework_metrics_reader = LocalAlgorithmMetricsReader(self.path) + + self._writer = None + self.tensor_table = collections.defaultdict(int) + # eventphase_starting_ids contains indexes for each of these phases, + # The value of these is used to put pid for timeline. This makes sure that we will have timeline produced + # which will be in order Step, Forward, Backward, DataIterator, Dataset, /device, /host, horovod, others.. + # Note that we can have 10000 event phase of each type + self.eventphase_starting_ids = { + "Step:": 1, + "Forward": 10000, + "Backward": 20000, + "DataIterator": 30000, + "Dataset": 40000, + "/device": 50000, + "/host": 60000, + "gpu_functions": 70000, + "cpu_functions": 80000, + "Horovod": 90000, + "other": 1000000, + } + + def open(self, file_path): + """ + Open the trace event file + """ + s3, bucket_name, key_name = is_s3(file_path) + try: + if s3: + self._writer = TSAccessS3(bucket_name, key_name, binary=False) + else: + self._writer = TSAccessFile(file_path, "w") + except (OSError, IOError) as err: + logger.debug(f"Sagemaker-Debugger: failed to open {file_path}: {str(err)}") + start, length = self._writer.write("[\n") + self.bytes_written += length + + def file_name(self, end_timestamp_in_us): + """ + Since this util will be used from a notebook or local directory, we directly write + to the merged file + """ + return TraceFileLocation().get_merged_trace_file_location( + base_dir=self.out_dir, timestamp_in_us=end_timestamp_in_us + ) + + def merge_timeline(self, start, end, unit=MergeUnit.TIME, sys_metrics_filter={"lowgpu": ()}): + """ + Get all trace files captured and merge them for viewing in the browser + """ + if unit == MergeUnit.STEP or unit == "step": + start_timestamp_in_us, end_timestamp_in_us = self.framework_metrics_reader._get_time_interval_for_step( + start, end + ) + else: + start_timestamp_in_us, end_timestamp_in_us = start, end + + filename = self.file_name(end_timestamp_in_us) + if self._writer is None: + self.open(filename) + + # get framework metrics + self.framework_metrics_reader.refresh_event_file_list() + framework_events = self.framework_metrics_reader.get_events( + start_timestamp_in_us, end_timestamp_in_us, file_suffix_filter=self.file_suffix_filter + ) + print("Got framework events") + if sys_metrics_filter is not None and len(sys_metrics_filter) > 0: + print(f"Appending system metrics with filter:{sys_metrics_filter}") + system_metrics_df = PandasFrame(self.path) + filtered_sys_metrics = system_metrics_df.get_profiler_data_by_time( + start_timestamp_in_us, end_timestamp_in_us, get_framework_metrics=False + ) + # 1st element of tuple is sys metrics df + filtered_sys_metrics = filtered_sys_metrics[0] + filtered_sys_metrics["timestamp_us"] = ( + filtered_sys_metrics["timestamp_us"] + system_metrics_df.start_time + ) + filtered_sys_metrics = filtered_sys_metrics.sort_values(by="timestamp_us") + + gpu_rows = filtered_sys_metrics[filtered_sys_metrics["dimension"] == "GPUUtilization"] + selected_sys = gpu_rows[gpu_rows["value"] < 85] + + # convert timestamp_us to actual epoch time + #'timestamp', 'timestamp_us', 'value', 'system_metric', 'dimension', + #'nodeID', 'type' + selected_sys = selected_sys.sort_values(by="timestamp_us") + print(len(selected_sys)) + prev_timestamp = None + prev_args = {} + added_sys_bottleneck_events_count = 0 + added_counter_events_count = 0 + current_iloc_selected_sys = 0 + # iterate over each row, if prev_timestamp is Not None and this timestamp != prev_timestamp: create trace event with args) + for index, row in filtered_sys_metrics.iterrows(): + if current_iloc_selected_sys >= len(selected_sys): + break + timestamp = selected_sys.iloc[current_iloc_selected_sys]["timestamp_us"] + print(f"selected sys timestamp:{timestamp} row timestamp:{row['timestamp_us']}") + + if (timestamp - 1000 * 20) <= (row["timestamp_us"]) <= (1000 * 20 + timestamp): + prev_timestamp = timestamp + prev_args[ + row["system_metric"] + "_" + row["dimension"] + "_" + row["nodeID"] + ] = row["value"] + else: + if prev_timestamp is not None: + # Make this instant marker global + # prev_args["s"] = "g" + t_event = TraceEvent( + ts=int(prev_timestamp), + name="sys_metrics_bottlenecks", + dur=0, + phase_pid=0, + phase_tid=0, + event_args=prev_args, + node_id=0, + phase="i", + file_type="system_metrics", + event_phase="sys_metrics_bottlenecks", + pid=0, + tid=0, + process_info=None, + ) + framework_events.append(t_event) + added_sys_bottleneck_events_count += 1 + prev_timestamp = None + prev_args = {} + if row["timestamp_us"] > timestamp: + current_iloc_selected_sys += 1 + if current_iloc_selected_sys >= len(selected_sys): + break + timestamp = selected_sys.iloc[current_iloc_selected_sys]["timestamp_us"] + if (timestamp - 1000 * 20) <= row["timestamp_us"] <= (1000 * 20 + timestamp): + prev_timestamp = timestamp + prev_args[ + row["system_metric"] + "_" + row["dimension"] + row["nodeID"] + ] = row["value"] + # {"name": "process_name", "ph": "M", "pid": 1000001, "args": {"name": "counter"}}, + # {"name": "process_sort_index", "ph": "M", "pid": 1000001, "args": {"sort_index": 1000001}}, + + # {"name": "ctr", "pid": 1000001, "ph": "C", "ts": 1602564436444665, "args": {"cats": 0}}, + # create a counter event + t_event = TraceEvent( + ts=int(row["timestamp_us"]), + name=row["system_metric"] + "_" + row["dimension"] + "_" + row["nodeID"], + dur=0, + phase_pid=0, + phase_tid=0, + event_args={"value": row["value"]}, + node_id=0, + phase="C", + file_type="system_metrics", + event_phase="sys_metrics", + pid=0, + tid=0, + process_info=None, + ) + added_counter_events_count += 1 + framework_events.append(t_event) + # prev_timestamp = row['timestamp_us'] + # prev_args[row['system_metric'] + "_" + row['dimension'] + row['nodeID']] = row['value'] + + if len(prev_args) > 0 and prev_timestamp is not None: + t_event = TraceEvent( + ts=int(prev_timestamp), + name="sys_metrics_bottlenecks", + dur=0, + phase_pid=0, + phase_tid=0, + event_args=prev_args, + node_id=0, + phase="i", + file_type="system_metrics", + event_phase="sys_metrics_bottlenecks", + pid=0, + tid=0, + process_info=None, + ) + framework_events.append(t_event) + added_sys_bottleneck_events_count += 1 + print( + f"Added {added_sys_bottleneck_events_count} sys events and count_counter_events:{added_counter_events_count}" + ) + + framework_events.sort(key=lambda x: x.start_time) + + seen_phasepid_tids = {} + print("Rewriting events") + for event in framework_events: + # print(str(event.tid) + "\n") + if self.tensor_table[event.event_phase] == 0: + # We will create tensor_idx based on what event_phase is there + # tensor idx would be generated to show timeline in order of Step(0), Forward(1)/Backward(2), DataIterator(3), Dataset(4), + # TF(/device, /host), PT detailed(cpu_functions/gpu_functions), horovod/SMDataParallel + # tensor_idx = len(self.tensor_table) + + found = False + for key in self.eventphase_starting_ids.keys(): + if key in event.event_phase: + tensor_idx = self.eventphase_starting_ids[key] + self.eventphase_starting_ids[key] += 1 + found = True + break + if not found: + tensor_idx = self.eventphase_starting_ids["other"] + self.eventphase_starting_ids["other"] += 1 + + self.tensor_table[event.event_phase] = tensor_idx + + # Instant events don't have a training phase + # TODO check with cycle marker + if event.phase != "i": + args = {"name": event.event_phase} + json_dict = {"name": "process_name", "ph": "M", "pid": tensor_idx, "args": args} + _, length = self._writer.write(json.dumps(json_dict) + ",\n") + self.bytes_written += length + + args = {"sort_index": tensor_idx} + json_dict = { + "name": "process_sort_index", + "ph": "M", + "pid": tensor_idx, + "args": args, + } + _, length = self._writer.write(json.dumps(json_dict) + ",\n") + self.bytes_written += length + # below we are modeling system metrics as instant events + elif event.file_type == "system_metrics": + # {"name": “LowGpu”, "ph": "i", "ts": 1234523.3, "pid": 2343, "tid": 2347, "s": "g"} + args = {"name": event.event_name} + json_dict = {"name": "process_name", "ph": "M", "pid": tensor_idx, "args": args} + event.phase_pid = event.phase_tid = tensor_idx + _, length = self._writer.write(json.dumps(json_dict) + ",\n") + self.bytes_written += length + + args = {"sort_index": tensor_idx} + json_dict = { + "name": "process_sort_index", + "ph": "M", + "pid": tensor_idx, + "args": args, + } + _, length = self._writer.write(json.dumps(json_dict) + ",\n") + self.bytes_written += length + + tensor_idx = self.tensor_table[event.event_phase] + # event.tid is not written. write it + is_event_seen = False + if ( + event.phase_pid in seen_phasepid_tids + and event.tid in seen_phasepid_tids[event.phase_pid] + ): + is_event_seen = True + # if thread id for this event pid is not yet seen, write the metadata for this thread + if event.process_info is not None and not is_event_seen: + phase_tid = event.phase_tid + thread_info = event.process_info._threads[phase_tid] + args = {"name": thread_info.thread_name} + json_dict = { + "name": "thread_name", + "ph": "M", + "pid": tensor_idx, + # + "tid": thread_info.tid, + "args": args, + } + _, length = self._writer.write(json.dumps(json_dict) + ",\n") + self.bytes_written += length + if event.phase_pid not in seen_phasepid_tids: + seen_phasepid_tids[event.phase_pid] = {} + seen_phasepid_tids[event.phase_pid][event.tid] = 1 + # change event pid back tensor idx before writing it. + event.pid = tensor_idx + # {"name": “LowGpu”, "ph": "i", "ts": 1234523.3, "pid": 2343, "tid": 2347, "s": "g"} + + _, length = self._writer.write(event.to_json() + ",\n") + self.bytes_written += length + + self.close() + + get_logger().info(f"Merged timeline saved at: {filename}") + return filename + + def close(self): + file_seek_pos = self.bytes_written - 2 + if isinstance(self._writer, TSAccessFile): + self._writer._accessor.seek(file_seek_pos) + self._writer._accessor.truncate() + else: + self._writer.data = self._writer.data[:file_seek_pos] + + if file_seek_pos > 2: + self._writer.write("\n]") + + self._writer.flush() + self._writer.close() + self._writer = None diff --git a/smdebug/profiler/analysis/utils/pandas_data_analysis.py b/smdebug/profiler/analysis/utils/pandas_data_analysis.py new file mode 100644 index 0000000000..9c839ebb76 --- /dev/null +++ b/smdebug/profiler/analysis/utils/pandas_data_analysis.py @@ -0,0 +1,449 @@ +# Standard Library +from enum import Enum + +# Third Party +import pandas as pd + +# First Party +from smdebug.core.logger import get_logger + + +class StatsBy(Enum): + """ + Enum to get stats by different categories. + """ + + # training phase such as TRAIN/EVAL/GLOBAL. + TRAINING_PHASE = "training_phase" + + # framework metrics such as function names/ operator names + FRAMEWORK_METRICS = "framework_metric" + + # event phase name as retrieved from events + PROCESS = "process" + + +class Resource(Enum): + """ + Enum to specify the device/resource specified in system metrics + """ + + CPU = "cpu" + + GPU = "gpu" + + IO = "i/o" + + NETWORK = "network" + + MEMORY = "memory" + + +# Container class for job stats +class JobStats(dict): + def __setitem__(self, key, item): + self.__dict__[key] = item + + def __getitem__(self, key): + return self.__dict__[key] + + def __repr__(self): + return repr(pd.DataFrame.from_dict(self.__dict__).T) + + +class PandasFrameAnalysis: + """ + This class contains some of the common utils that can be used with + the system metrics and framework metrics DataFrames. + The functions here only query the DataFrame and return results. The results + will then have to be plotted/visualized by the user or other utils. + """ + + def __init__(self, system_df, framework_df): + self.sys_metrics_df = system_df + self.framework_metrics_df = framework_df + + self.framework_metrics_df["duration_us"] = ( + self.framework_metrics_df["end_time_us"] - self.framework_metrics_df["start_time_us"] + ) + self._get_step_numbers() + + def _get_step_time_mapping(self): + phase_metrics_df = self.framework_metrics_df[ + self.framework_metrics_df["framework_metric"].str.contains("Step:ModeKeys") + ] + + # multi-processing + phase_metrics_df = phase_metrics_df[phase_metrics_df["step"] != -1] + phase_metrics_df = ( + phase_metrics_df.groupby(["step"]) + .agg({"start_time_us": "min", "end_time_us": "max"}) + .reset_index() + ) + self.step_time_df = phase_metrics_df + + def _get_step_numbers(self): + self._get_step_time_mapping() + + def helper(x): + if x["step"] != -1: + return x["step"] + result = self.step_time_df[ + (self.step_time_df["start_time_us"] < x["start_time_us"]) + & (self.step_time_df["end_time_us"] > x["end_time_us"]) + ] + return result["step"].iloc[0] + + self.framework_metrics_df["step"] = self.framework_metrics_df.apply( + lambda x: helper(x), axis=1 + ) + + def get_job_statistics(self): + """ + Returns a Dictionary with information about runtime of training job, initilization, training loop and finalization. + """ + job_statistics = JobStats() + job_statistics["start_time"] = min(self.sys_metrics_df["timestamp"]) + job_statistics["end_time"] = max(self.sys_metrics_df["timestamp"]) + job_statistics["job_duration"] = max(self.sys_metrics_df["timestamp_us"]) - min( + self.sys_metrics_df["timestamp_us"] + ) + step_0 = self.framework_metrics_df[ + (self.framework_metrics_df["step"] == 0) + & ( + self.framework_metrics_df["framework_metric"].isin( + ["Step:ModeKeys.TRAIN", "Step:ModeKeys.GLOBAL"] + ) + ) + ].reset_index(drop=True) + job_statistics["training_loop_start"] = step_0["start_time"][0] + job_statistics["training_loop_end"] = max(self.framework_metrics_df["end_time"]) + job_statistics["training_loop_duration"] = ( + max(self.framework_metrics_df["end_time_us"]) - step_0["start_time_us"] + ) + job_statistics["initialization"] = step_0["start_time_us"][0] + job_statistics["finalization"] = max(self.sys_metrics_df["timestamp_us"]) - max( + self.framework_metrics_df["end_time_us"] + ) + job_statistics["initialization_%"] = ( + job_statistics["initialization"] / job_statistics["job_duration"] + ) * 100 + job_statistics["training_loop_%"] = ( + job_statistics["training_loop_duration"] / job_statistics["job_duration"] + ) * 100 + job_statistics["finalization_%"] = ( + job_statistics["finalization"] / job_statistics["job_duration"] + ) * 100 + + return job_statistics + + def get_step_statistics(self, by=StatsBy.TRAINING_PHASE): + """ + Get average, minimum, maximum, p50, p95, p99 stats on step duration + :param by: by default, stats are grouped by framework_metric. The other options are + to get stats by training phase - train/eval/global or grouped by process. This parameter + should be of type StatsBy + """ + if not isinstance(by, StatsBy): + get_logger().info(f"{by} should be of type StatsBy") + return None + + by = by.value + step_stats = None + if by in [StatsBy.FRAMEWORK_METRICS.value, StatsBy.PROCESS.value]: + # TODO: Consider that some processes may be optimized. + # For example: data pipeline executed in parallel. + phase_metrics_df = ( + self.framework_metrics_df.groupby(["step", by]) + .agg({"start_time_us": "min", "end_time_us": "max"}) + .reset_index() + ) + phase_metrics_df["duration_us"] = ( + phase_metrics_df["end_time_us"] - phase_metrics_df["start_time_us"] + ) + + step_stats = ( + phase_metrics_df.groupby([by])["duration_us"] + .describe(percentiles=[0.5, 0.95, 0.99]) + .unstack() + .reset_index() + ) + step_stats = step_stats.pivot(index=by, columns="level_0", values=0).reset_index() + step_stats.columns.name = "" + step_stats = step_stats.drop(["count", "std"], axis="columns") + step_stats = step_stats[[by, "mean", "min", "max", "50%", "95%", "99%"]] + elif by == StatsBy.TRAINING_PHASE.value: + phase_metrics_df = self.framework_metrics_df[ + self.framework_metrics_df["framework_metric"].str.contains("Step:ModeKeys") + ] + + # multi-processing + phase_metrics_df = ( + phase_metrics_df.groupby(["step", "framework_metric"]) + .agg({"start_time_us": "min", "end_time_us": "max"}) + .reset_index() + ) + phase_metrics_df["duration_us"] = ( + phase_metrics_df["end_time_us"] - phase_metrics_df["start_time_us"] + ) + + step_stats = ( + phase_metrics_df.groupby(["framework_metric"])["duration_us"] + .describe(percentiles=[0.5, 0.95, 0.99]) + .unstack() + .reset_index() + ) + step_stats = step_stats.pivot( + index="framework_metric", columns="level_0", values=0 + ).reset_index() + step_stats.columns.name = "" + step_stats = step_stats.drop(["count", "std"], axis="columns") + step_stats = step_stats[["framework_metric", "mean", "min", "max", "50%", "95%", "99%"]] + if step_stats is not None: + step_stats.columns = [ + by, + "duration_mean_us", + "duration_min_us", + "duration_max_us", + "duration_p50_us", + "duration_p95_us", + "duration_p99_us", + ] + return step_stats + + def _get_utilization_phase_by_time_interval(self, interval_df): + """ + For a given set of framework metric intervals, what are the corresponding + system metrics duration each period + :param interval_df: DataFrame containing start time, end time, and name of the phase + thats active during the interval. + """ + + def helper(start, end, phase): + self.sys_metrics_df.loc[ + (self.sys_metrics_df["timestamp_us"].between(start, end, inclusive=True)), "phase" + ] = phase + + interval_df.apply( + lambda x: helper(x["start_time_us"], x["end_time_us"], x["phase"]), axis=1 + ) + + def get_utilization_stats(self, resource=None, by=None, phase=None): + """ + Get CPU/GPU utilization stats + :param resource: system resource for which utilization stats have to be computed. Type: Resource + :param by: By default, get overall utilization stats. When by="training_phase", + utilization stats are provided per training phase interval. Type: StatsBy + :param phase: List of training phase to find intervals for. If nothing is mentioned, intervals + are determined for all training phases available. + :return: Dataframe containing utilization stats + """ + if (by is not None) and (not isinstance(by, StatsBy)): + get_logger().info(f"{by} should be of type StatsBy") + return None + if (resource is not None) and (not isinstance(resource, (list, Resource))): + get_logger().info(f"{resource} should be of type list or Resource") + return None + + if resource is None: + resources = [ + Resource.CPU.value, + Resource.GPU.value, + Resource.MEMORY.value, + Resource.IO.value, + Resource.NETWORK.value, + ] + else: + if isinstance(resource, Resource): + resource = [resource] + resources = [x.value for x in resource] + + if by == StatsBy.TRAINING_PHASE: + interval_df = self.get_training_phase_intervals(phase) + self._get_utilization_phase_by_time_interval(interval_df) + + df_for_concat = [] + columns = [ + "Resource", + "nodeID", + "utilization_mean", + "utilization_min", + "utilization_max", + "utilization_p50", + "utilization_p95", + "utilization_p99", + ] + for resrc in resources: + sys_resrc_df = self.sys_metrics_df[ + self.sys_metrics_df["type"].str.contains(resrc) + ].reset_index() + if sys_resrc_df.empty: + # there's no data for this resource + continue + if by == StatsBy.TRAINING_PHASE: + groupby = first_column_name = "phase" + else: + groupby = lambda _: resrc + first_column_name = "level_0" + + sys_resrc_df = ( + sys_resrc_df.groupby([groupby, "nodeID"])["value"] + .describe(percentiles=[0.5, 0.95, 0.99]) + .reset_index() + ) + sys_resrc_df.columns.name = "" + sys_resrc_df = sys_resrc_df.drop(["count", "std"], axis="columns") + sys_resrc_df = sys_resrc_df[ + [first_column_name, "nodeID", "mean", "min", "max", "50%", "95%", "99%"] + ] + + if by == StatsBy.TRAINING_PHASE: + sys_resrc_df.insert(0, "Resource", resrc) + + df_for_concat.append(sys_resrc_df) + + if by == StatsBy.TRAINING_PHASE: + columns.insert(1, "Training_phase") + util_stats = pd.concat(df_for_concat).reset_index(drop=True) + util_stats.columns = columns + return util_stats + + def get_device_usage_stats(self, device=None, utilization_ranges=None): + """ + Find the usage spread based on utilization ranges. If ranges are not provided, + >90, 10-90, <10 are considered + :param device: List of Resource.cpu, Resource.gpu. Type: Resource + :param utilization_ranges: list of tuples + """ + if (device is not None) and (not isinstance(device, (list, Resource))): + get_logger().info(f"{device} should be of type list or Resource") + return pd.DataFrame() + + if device is None: + resources = [Resource.CPU.value, Resource.GPU.value] + else: + if isinstance(device, Resource): + device = [device] + resources = [x.value for x in device] + + if utilization_ranges is None: + utilization_ranges = [(90, 100), (10, 90), (0, 10)] + if not isinstance(utilization_ranges, list): + get_logger().info( + f"{utilization_ranges} should be a list of tuples containing the ranges" + ) + return pd.DataFrame() + if len(utilization_ranges) == 0: + get_logger().info(f"{utilization_ranges} cannot be empty") + return pd.DataFrame() + if any(len(utilization_range) != 2 for utilization_range in utilization_ranges): + get_logger().info( + f"Each interval in {utilization_ranges} must have a start and end value" + ) + return pd.DataFrame() + + def helper(x, util_ranges): + for start, end in util_ranges: + if start <= float(x) <= end: + return (start, end) + return () + + self.sys_metrics_df["ranges"] = self.sys_metrics_df.apply( + lambda x: helper(x["value"], utilization_ranges), axis=1 + ) + device_sys_df = self.sys_metrics_df[self.sys_metrics_df["ranges"] != ()] + + if device_sys_df.empty: + return device_sys_df + + usage_stats = device_sys_df[ + device_sys_df["type"].str.contains("|".join(resources)).any(level=0) + ] + + df_grouped = ( + usage_stats.groupby(["type", "nodeID", "ranges"])["ranges"].describe().reset_index() + ) + df_grouped = df_grouped.drop(["unique", "top", "freq"], axis="columns") + df_grouped = ( + df_grouped.set_index(["type", "nodeID"]).pivot(columns="ranges")["count"].reset_index() + ) + df_grouped = df_grouped.fillna(0) + return df_grouped + + def get_training_phase_intervals(self, phase=None): + """ + This function splits framework data into before train, train, between train and eval, eval, and after eval. + :param phase: List of training phase to find intervals for. If nothing is mentioned, intervals + are determined for all training phases available. Type: string or List of strings + :return: DataFrame containing the intervals + """ + process_list = self.framework_metrics_df["process"].unique() + if phase is None: + phase = [x for x in process_list if "Step:ModeKeys" in x] + + if isinstance(phase, str): + phase = [phase] + + if not isinstance(phase, list): + get_logger().info(f"{phase} should be a list of strings") + return None + + # Filter out phases that are not available in process list + phase = [x for x in phase if x in process_list] + + if len(phase) == 0: + get_logger().info( + f"None of the phase strings matched the phases available in the framework metrics DataFrame" + ) + return None + + mode_df = self.framework_metrics_df[ + self.framework_metrics_df["framework_metric"].isin(phase) + ] + training_phases = mode_df["framework_metric"].unique() + if len(phase) > 1: + mode_df = mode_df.groupby( + mode_df["framework_metric"].ne(mode_df["framework_metric"].shift()).cumsum() + ) + mode_df = mode_df.apply( + lambda x: pd.DataFrame( + { + "start_time_us": [x["start_time_us"].min()], + "end_time_us": [x["end_time_us"].max()], + "phase": [x["framework_metric"].iloc[0]], + } + ) + ).reset_index(drop=True) + else: + mode_df = mode_df[["start_time_us", "end_time_us", "framework_metric"]].reset_index( + drop=True + ) + mode_df.rename({"framework_metric": "phase"}, axis="columns", inplace=True) + + for i in range(len(mode_df.index) - 1): + ind = mode_df.index[i] + next_index = ind + 0.5 + this_phase = mode_df["phase"][ind] + next_phase = mode_df["phase"][mode_df.index[i + 1]] + if this_phase in training_phases and next_phase in training_phases: + row = { + "start_time_us": mode_df["end_time_us"][ind] + 1, + "end_time_us": mode_df["start_time_us"][mode_df.index[i + 1]] - 1, + "phase": "Between " + " and ".join(sorted([this_phase, next_phase])), + } + mode_df.loc[next_index] = row + + row = { + "start_time_us": self.sys_metrics_df["timestamp_us"].min(), + "end_time_us": mode_df["start_time_us"][0] - 1, + "phase": "Before " + mode_df["phase"][0], + } + mode_df.loc[-1] = row + mode_df = mode_df.sort_index().reset_index(drop=True) + row = { + "start_time_us": mode_df["end_time_us"][mode_df.index[-1]] + 1, + "end_time_us": self.sys_metrics_df["timestamp_us"].max(), + "phase": "After " + mode_df["phase"][mode_df.index[-1]], + } + mode_df.loc[mode_df.index[-1] + 1] = row + return mode_df diff --git a/smdebug/profiler/analysis/utils/profiler_data_to_pandas.py b/smdebug/profiler/analysis/utils/profiler_data_to_pandas.py new file mode 100644 index 0000000000..62d4201c8a --- /dev/null +++ b/smdebug/profiler/analysis/utils/profiler_data_to_pandas.py @@ -0,0 +1,559 @@ +# Third Party +import pandas as pd + +# First Party +from smdebug.core.utils import match_inc +from smdebug.profiler.algorithm_metrics_reader import ( + LocalAlgorithmMetricsReader, + S3AlgorithmMetricsReader, +) +from smdebug.profiler.profiler_constants import CONVERT_TO_MICROSECS +from smdebug.profiler.system_metrics_reader import LocalSystemMetricsReader, S3SystemMetricsReader +from smdebug.profiler.utils import ( + convert_utc_datetime_to_microseconds, + us_since_epoch_to_human_readable_time, +) + + +class PandasFrame: + def __init__(self, path, use_in_memory_cache=False, scan_interval=5000000000): + + self.path = path + self.step_time_mapping = dict() + + # Reader for system and framework metrics + if path.startswith("s3"): + self.system_metrics_reader = S3SystemMetricsReader(self.path) + self.framework_metrics_reader = S3AlgorithmMetricsReader( + self.path, use_in_memory_cache=use_in_memory_cache + ) + else: + self.system_metrics_reader = LocalSystemMetricsReader(self.path) + self.framework_metrics_reader = LocalAlgorithmMetricsReader( + self.path, use_in_memory_cache=use_in_memory_cache + ) + + # list to store metrics + self.system_metrics = [] + self.framework_metrics = [] + + # we read data in chunks + self.interval = scan_interval + self.start_time = self.system_metrics_reader.get_timestamp_of_first_available_file() + + def get_all_system_metrics(self, selected_system_metrics=[]): + """ + Get system metrics + :param systemk_metrics_list: list of system metrics.If not empty, function will only return framework events that are part of this list. + :return: System metrics DataFrame + """ + # get all system metrics from last to current timestamp + + start_timestamp = self.system_metrics_reader.get_timestamp_of_first_available_file() + end_timestamp = ( + self.system_metrics_reader.get_timestamp_of_latest_available_file() + self.interval + ) + sys_events_df, _ = self.get_profiler_data_by_time( + start_timestamp, + end_timestamp, + cache_metrics=False, + selected_system_metrics=selected_system_metrics, + get_framework_metrics=False, + ) + + return sys_events_df + + def get_all_framework_metrics(self, selected_framework_metrics=[]): + """ + Get framework metrics + :param selected_framework_metrics: list of framework metrics.If not empty, function will only return framework events that are part of this list. + :return: Framework metrics DataFrame + """ + # get all framework metrics from last to current timestamp + self.framework_metrics_reader.refresh_event_file_list() + + start_timestamp = ( + self.system_metrics_reader.get_timestamp_of_first_available_file() + ) # bug: get_events does not return the very first event + end_timestamp = self.framework_metrics_reader.get_timestamp_of_latest_available_file() + + _, fw_events_df = self.get_profiler_data_by_time( + start_timestamp, + end_timestamp, + cache_metrics=False, + selected_framework_metrics=selected_framework_metrics, + get_system_metrics=False, + ) + + return fw_events_df + + def convert_datetime_to_timestamp(self, timestamp): + """ + A helper function to convert datetime into timestamp + :param timestep: timestamp in datetime + :return: timestamp in microseconds + """ + timestamp = pd.to_datetime(timestamp, format="%Y-%m-%dT%H:%M:%S:%f", utc=True) + return convert_utc_datetime_to_microseconds(timestamp) + + def get_framework_metrics_by_timesteps(self, timestep_list=[], selected_framework_metrics=[]): + """ + Get framework metrics for a list of timeranges. This function is useful when we want to correlate framework metrics with system metrics. Framework metrics have a begin and end timestamp. System metrics have only a single timestamp. + :param timestep_list: list of timestamps + :param selected_framework_metrics: list of framework metrics which will be stored in the dataframe + :return: Framework metrics DataFrame + """ + # get min and max search range + timestep_list = sorted(timestep_list) + start_time_us = self.convert_datetime_to_timestamp(timestep_list[0]) + end_time_us = self.convert_datetime_to_timestamp(timestep_list[-1]) + + # to avoid out of memory issues, we read data in chunks + current_time_us = start_time_us + if end_time_us - start_time_us > self.interval: + current_time_us = start_time_us + self.interval + else: + current_time_us = end_time_us + results = {} + results_detailed = {} + counter = 0 + + while start_time_us < end_time_us: + # get all framework metrics from last to current timestamp + self.framework_metrics_reader.refresh_event_file_list() + events = self.framework_metrics_reader.get_events(start_time_us, current_time_us) + + # iterate over system metrics timestamps and find overlap + for index, timestamp in enumerate(timestep_list[counter:]): + timestamp = self.convert_datetime_to_timestamp(timestamp) + if timestamp >= current_time_us: + counter = index + break + for event in events: + if len(selected_framework_metrics) > 0 and ( + event.event_name not in selected_framework_metrics + and event.event_phase not in selected_framework_metrics + ): + continue + if event.start_time < timestamp and event.end_time > timestamp: + if event.event_phase not in results: + results[event.event_phase] = 0 + results[event.event_phase] += event.end_time - event.start_time + if "Step" not in event.event_name: + if event.event_name not in results_detailed: + results_detailed[event.event_name] = 0 + results_detailed[event.event_name] += event.end_time - event.start_time + # read the next chunk of framework metrics + start_time_us = current_time_us + if current_time_us + self.interval < end_time_us: + current_time_us = current_time_us + self.interval + else: + current_time_us = end_time_us + + framework_metrics = {} + training_phase = {} + + for key in results: + if "Step" in key: + training_phase[key] = results[key] + else: + framework_metrics[key] = results[key] + + if len(framework_metrics.values()) > 0: + max_value = float(max(list(framework_metrics.values()))) + for key in framework_metrics: + framework_metrics[key] = framework_metrics[key] / max_value + + return framework_metrics, results_detailed, training_phase + + def get_framework_metrics_by_begin_and_end_timesteps( + self, begin_timestep_list, end_timestep_list, selected_framework_metrics=[] + ): + """ + Get framework metrics for a set of given timeranges. This function is useful when we want to correlate framework metrics such as steps with other framework metrics such as dataloading, preprocessing etc. + :param begin_timestep_list: list of start of intervals in datetime + :param end_timestep_list: list of end intervals in datetime + :param selected_framework_metrics: list of framework metrics which will be stored in the dataframe + :return: Framework metrics DataFrame + """ + # Get min and max timestamps from the list of timeranges + start_time_us = self.convert_datetime_to_timestamp(min(begin_timestep_list)) + end_time_us = self.convert_datetime_to_timestamp(max(end_timestep_list)) + + # in order to avoid out of memory issues we will read only chunks of data + current_time_us = start_time_us + if end_time_us - start_time_us > self.interval: + current_time_us = start_time_us + self.interval + else: + current_time_us = end_time_us + framework_metrics = {} + framework_metrics_detailed = {} + counter = 0 + while start_time_us < end_time_us: + # get all framework metrics from last to current timestamp + self.framework_metrics_reader.refresh_event_file_list() + events = self.framework_metrics_reader.get_events(start_time_us, current_time_us) + + # iterate over start and end time intervals and find overlaps in the current timerange + for index, (begin_timestamp, end_timestamp) in enumerate( + zip(begin_timestep_list[counter:], end_timestep_list[counter:]) + ): + begin_timestamp = self.convert_datetime_to_timestamp(begin_timestamp) + end_timestamp = self.convert_datetime_to_timestamp(end_timestamp) + + # if we are out of range, stop searching for overlaps + if begin_timestamp >= current_time_us: + counter = index + break + # iterate over events in the current timerange + for event in events: + if len(selected_framework_metrics) > 0 and ( + event.event_name not in selected_framework_metrics + and event.event_phase not in selected_framework_metrics + ): + continue + if event.end_time >= begin_timestamp and event.start_time <= end_timestamp: + if "Step" not in event.event_name: + if event.event_phase not in framework_metrics: + framework_metrics[event.event_phase] = 0 + framework_metrics[event.event_phase] += ( + event.end_time - event.start_time + ) + if event.event_name not in framework_metrics_detailed: + framework_metrics_detailed[event.event_name] = 0 + framework_metrics_detailed[event.event_name] += ( + event.end_time - event.start_time + ) + # read the next chunk of data + start_time_us = current_time_us + if current_time_us + self.interval < end_time_us: + current_time_us = current_time_us + self.interval + else: + current_time_us = end_time_us + + # normalize cumulative time to 0-1 + if len(list(framework_metrics.values())) > 0: + max_value = float(max(list(framework_metrics.values()))) + for key in framework_metrics: + framework_metrics[key] = framework_metrics[key] / max_value + max_value = float(max(list(framework_metrics_detailed.values()))) + for key in framework_metrics_detailed: + framework_metrics_detailed[key] = framework_metrics_detailed[key] / max_value + + return framework_metrics, framework_metrics_detailed + + def get_profiler_data_by_time( + self, + start_time_us, + end_time_us, + cache_metrics=False, + selected_framework_metrics=[], + selected_system_metrics=[], + get_framework_metrics=True, + get_system_metrics=True, + ): + """ + Get metrics data within a time interval. + :param start_time_us: Start of the interval in microseconds + :param end_time_us: End of the interval in microseconds + :param cache_metrics: If True, collect and return all metrics requested so far, else, + :param framework_metrics_list: list of framework metrics. If not empty, function will only return framework events that are part of this list. + :param selected_system_metrics: list of system metrics. If not empty, function will only return system events that are part of this list. + :param selected_framework_metrics: if True, get framework metrics + :param get_system_metrics: if True: get system metrics + return current request + :return: System metrics DataFrame, Framework metrics DataFrame + """ + # read system metrics + system_metrics = [] + if get_system_metrics: + events = self.system_metrics_reader.get_events(start_time_us, end_time_us) + + # append new events to existing list + for event in events: + if len(selected_system_metrics) > 0 and event.name not in selected_system_metrics: + continue + system_metrics.append( + [ + # GPU and CPU metrics are recorded at slightly different timesteps, so we round the numbers + us_since_epoch_to_human_readable_time( + int(event.timestamp * CONVERT_TO_MICROSECS) + ), + int(event.timestamp * CONVERT_TO_MICROSECS), + event.value, + event.name, + event.dimension, + event.node_id, + event.type, + ] + ) + + if cache_metrics is True: + self.system_metrics.extend(system_metrics) + system_metrics = self.system_metrics + + # create data frame for system metrics + system_metrics_df = pd.DataFrame( + system_metrics, + columns=[ + "timestamp", + "timestamp_us", + "value", + "system_metric", + "dimension", + "nodeID", + "type", + ], + ) + + system_metrics_df["timestamp_us"] = system_metrics_df["timestamp_us"] - self.start_time + + # get framework metrics + framework_metrics = [] + if get_framework_metrics: + # only fetch a subset of data to avoid out of memory issues + if end_time_us - start_time_us > self.interval: + current_time_us = start_time_us + self.interval + else: + current_time_us = end_time_us + + while start_time_us < end_time_us: + # get all framework metrics from last to current timestamp + self.framework_metrics_reader.refresh_event_file_list() + events = self.framework_metrics_reader.get_events(start_time_us, current_time_us) + + # append new events to existing list + for event in events: + if len(selected_framework_metrics) > 0 and ( + event.event_name not in selected_framework_metrics + and event.event_phase not in selected_framework_metrics + ): + continue + if event.event_args is not None and "step_num" in event.event_args: + step = int(event.event_args["step_num"]) + else: + step = -1 + if event.event_args is not None and "layer_name" in event.event_args: + name = event.event_args["layer_name"] + elif event.event_args is not None and "name" in event.event_args: + name = event.event_args["name"] + else: + name = event.event_name + if event.event_args is not None and "bytes_fetched" in event.event_args: + bytes_fetched = event.event_args["bytes_fetched"] + else: + bytes_fetched = -1 + + framework_metrics.append( + [ + us_since_epoch_to_human_readable_time(event.start_time), + us_since_epoch_to_human_readable_time(event.end_time), + event.start_time, + event.end_time, + event.tid, + event.pid, + name, + step, + bytes_fetched, + event.event_phase, + event.node_id, + ] + ) + # read the next chunk of data + start_time_us = current_time_us + if current_time_us + self.interval < end_time_us: + current_time_us = current_time_us + self.interval + else: + current_time_us = end_time_us + + if cache_metrics is True: + self.framework_metrics.extend(framework_metrics) + framework_metrics = self.framework_metrics + + # create data frame for framework metrics + framework_metrics_df = pd.DataFrame( + framework_metrics, + columns=[ + "start_time", + "end_time", + "start_time_us", + "end_time_us", + "tid", + "pid", + "framework_metric", + "step", + "bytes", + "process", + "nodeID", + ], + ) + framework_metrics_df["start_time_us"] = ( + framework_metrics_df["start_time_us"] - self.start_time + ) + framework_metrics_df["end_time_us"] = framework_metrics_df["end_time_us"] - self.start_time + + return ( + system_metrics_df[system_metrics_df.duplicated() == False], + framework_metrics_df[framework_metrics_df.duplicated() == False], + ) + + def get_profiler_data_by_step(self, start_step, end_step, cache_metrics=False): + """ + Get metrics data within a step interval. We find the mapping between step number and time interval for + the step as some events may not be associated with a step number yet. + :param start_step: Start of step interval + :param end_step: End of step interval + :param cache_metrics: If True, collect and return all metrics requested so far, else, + return current request + :return: System metrics DataFrame, Framework metrics DataFrame + """ + sys_metrics_df, fw_metrics_df = ( + self.get_all_system_metrics(), + self.get_all_framework_metrics(), + ) + + fw_metrics_df = fw_metrics_df[ + (fw_metrics_df["step"].between(start_step, end_step, inclusive=True)) + ] + start_time, end_time = ( + fw_metrics_df["start_time_us"].min(), + fw_metrics_df["end_time_us"].max(), + ) + + sys_metrics_df = sys_metrics_df[ + (sys_metrics_df["timestamp_us"].between(start_time, end_time, inclusive=True)) + ] + + return sys_metrics_df, fw_metrics_df + + def get_all_dataloader_metrics(self, selected_framework_metrics=[]): + """ + Get framework metrics + :param selected_framework_metrics: list of framework metrics.If not empty, function will only return framework events that are part of this list. + :return: Framework metrics DataFrame + """ + # get all framework metrics from last to current timestamp + self.framework_metrics_reader.refresh_event_file_list() + + start_timestamp = ( + self.system_metrics_reader.get_timestamp_of_first_available_file() + ) # bug: get_events does not return the very first event + end_timestamp = self.framework_metrics_reader.get_timestamp_of_latest_available_file() + + fw_events_df = self._get_dataloader_profiler_data_by_time( + start_timestamp, + end_timestamp, + cache_metrics=False, + selected_framework_metrics=selected_framework_metrics, + ) + + return fw_events_df + + def _get_dataloader_profiler_data_by_time( + self, start_time_us, end_time_us, cache_metrics=False, selected_framework_metrics=[] + ): + """ + Get metrics data within a time interval. + :param start_time_us: Start of the interval in microseconds + :param end_time_us: End of the interval in microseconds + :param cache_metrics: If True, collect and return all metrics requested so far, else, + :param framework_metrics_list: list of framework metrics. If not empty, function will only return framework events that are part of this list. + :return: Framework metrics DataFrame + """ + # get framework metrics + framework_metrics = [] + # only fetch a subset of data to avoid out of memory issues + if end_time_us - start_time_us > self.interval: + current_time_us = start_time_us + self.interval + else: + current_time_us = end_time_us + + while start_time_us < end_time_us: + # get all framework metrics from last to current timestamp + self.framework_metrics_reader.refresh_event_file_list() + events = self.framework_metrics_reader.get_events(start_time_us, current_time_us) + + # append new events to existing list + for event in events: + if len(selected_framework_metrics) > 0 and not ( + match_inc(event.event_name, selected_framework_metrics) + or match_inc(event.event_phase, selected_framework_metrics) + ): + continue + if event.event_args is not None and "step_num" in event.event_args: + step = int(event.event_args["step_num"]) + else: + step = -1 + if event.event_args is not None and "layer_name" in event.event_args: + name = event.event_args["layer_name"] + elif event.event_args is not None and "name" in event.event_args: + name = event.event_args["name"] + else: + name = event.event_name + if event.event_args is not None and "worker_id" in event.event_args: + worker_id = event.event_args["worker_id"] + else: + worker_id = -1 + + if event.event_args is not None and "num_workers" in event.event_args: + num_workers = event.event_args["num_workers"] + else: + num_workers = -1 + + if event.event_args is not None and "pin_memory" in event.event_args: + pin_memory = "True" if event.event_args["pin_memory"] is True else "False" + else: + pin_memory = "NA" + + framework_metrics.append( + [ + us_since_epoch_to_human_readable_time(event.start_time), + us_since_epoch_to_human_readable_time(event.end_time), + event.start_time, + event.end_time, + event.duration, + event.pid, + name, + step, + worker_id, + num_workers, + pin_memory, + event.event_phase, + event.node_id, + ] + ) + # read the next chunk of data + start_time_us = current_time_us + if current_time_us + self.interval < end_time_us: + current_time_us = current_time_us + self.interval + else: + current_time_us = end_time_us + + if cache_metrics is True: + self.framework_metrics.extend(framework_metrics) + framework_metrics = self.framework_metrics + + # create data frame for framework metrics + framework_metrics_df = pd.DataFrame( + framework_metrics, + columns=[ + "start_time", + "end_time", + "start_time_us", + "end_time_us", + "duration_us", + "pid", + "framework_metric", + "step", + "worker_id", + "num_workers", + "pin_memory", + "process", + "node_id", + ], + ) + framework_metrics_df["start_time_us"] = ( + framework_metrics_df["start_time_us"] - self.start_time + ) + framework_metrics_df["end_time_us"] = framework_metrics_df["end_time_us"] - self.start_time + return framework_metrics_df[framework_metrics_df.duplicated() == False] diff --git a/smdebug/profiler/analysis/utils/python_profile_analysis_utils.py b/smdebug/profiler/analysis/utils/python_profile_analysis_utils.py new file mode 100644 index 0000000000..f14524202e --- /dev/null +++ b/smdebug/profiler/analysis/utils/python_profile_analysis_utils.py @@ -0,0 +1,236 @@ +# Standard Library +import pstats +from enum import Enum + +# Third Party +import pandas as pd + +# First Party +from smdebug.profiler.python_profile_utils import ( + PythonProfileModes, + StepPhase, + str_to_python_profile_mode, +) + + +class Metrics(Enum): + """ + Enum to describe the types of metrics recorded in cProfile profiling. + """ + + # total amount of time spent in the scope of this function alone, in seconds. + TOTAL_TIME = "tottime" + + # total amount of time spent in the scope of this function and in the scope of all other functions that this + # function calls, in seconds. + CUMULATIVE_TIME = "cumtime" + + # number of primitive (non-recursive) calls to this function + PRIMITIVE_CALLS = "pcalls" + + # total number of calls to this function, recursive or non-recursive. + TOTAL_CALLS = "ncalls" + + +class StepPythonProfileStats: + """ + Class that represents the metadata for a single instance of profiling: before step 0, during a step, between steps, + end of script, etc. Used so that users can easily filter through which exact portion of their session that + they want profiling stats of. In addition, printing this class will result in a dictionary of the attributes and + its corresponding values. + + ... + + Attributes + ---------- + profiler_name: str + The name of the profiler used to generate this stats file, cProfile or pyinstrument + framework: str + The machine learning framework used in training. + node_id: str + The node ID of the node used in the session. + start_mode: str + The training phase (TRAIN/EVAL/GLOBAL) at which profiling started. + start_phase: str + The step phase (start of step, end of step, etc.) at which python profiling was started. + start_step: float + The step at which python profiling was started. -1 if profiling before step 0. + start_time_since_epoch_in_micros: int + The UTC time (in microseconds) at which profiling started for this step. + end_mode: str + The training phase (TRAIN/EVAL/GLOBAL) at which profiling was stopped. + end_step: float + The step at which python profiling was stopped. Infinity if end of script. + end_phase: str + The step phase (start of step, end of step, etc.) at which python profiling was stopped. + end_time_since_epoch_in_micros: int + The UTC time (in microseconds) at which profiling finished for this step. + stats_path: str + The path to the dumped python stats or html resulting from profiling this step. + """ + + def __init__(self, framework, profiler_name, node_id, stats_dir, stats_path): + start_metadata, end_metadata = stats_dir.split("_") + start_mode, start_step, start_phase, start_time_since_epoch_in_micros = start_metadata.split( + "-" + ) + end_mode, end_step, end_phase, end_time_since_epoch_in_micros = end_metadata.split("-") + + self.profiler_name = profiler_name + self.framework = framework + self.node_id = node_id + + self.start_mode = PythonProfileModes(str_to_python_profile_mode(start_mode)) + self.start_step = -1 if start_step == "*" else int(start_step) + self.start_phase = StepPhase(start_phase) + self.start_time_since_epoch_in_micros = float(start_time_since_epoch_in_micros) + + self.end_mode = PythonProfileModes(str_to_python_profile_mode(end_mode)) + self.end_step = float("inf") if end_step == "*" else int(end_step) + self.end_phase = StepPhase(end_phase) + self.end_time_since_epoch_in_micros = float(end_time_since_epoch_in_micros) + + self.stats_path = stats_path + + def has_start_and_end_mode(self, start_mode, end_mode): + return self.start_mode == start_mode and self.end_mode == end_mode + + def in_time_interval(self, start_time_since_epoch_in_micros, end_time_since_epoch_in_micros): + """Returns whether this step is in the provided time interval. + This is defined as whether there is any overlap between the time interval + of the step and the provided time interval. + """ + return ( + start_time_since_epoch_in_micros + <= self.start_time_since_epoch_in_micros + <= end_time_since_epoch_in_micros + or start_time_since_epoch_in_micros + <= self.end_time_since_epoch_in_micros + <= end_time_since_epoch_in_micros + ) + + def in_step_interval(self, start_step, end_step, start_phase, end_phase): + """Returns whether this is in the provided step interval. This is defined as: + 1. This start step is greater than the provided start step and the end step is greater than the provided end + step. + 2. If this start step equals the provided start step, verify that this start phase does not occur before the + provided start phase. + 3. If this end step equals the provided end step, verify that this end phase does not occur after the provided + end phase. + """ + if start_step < self.start_step and end_step > self.end_step: + return True + elif start_step > self.start_step or end_step < self.end_step: + return False + else: + if ( + start_step == self.start_step + and start_phase in (StepPhase.STEP_END, StepPhase.FORWARD_PASS_END) + and self.start_phase != start_phase + ): + return False + if ( + end_step == self.end_step + and end_phase == StepPhase.STEP_START + and self.end_phase != end_phase + ): + return False + return True + + def has_pre_step_zero_profile_stats(self): + return self.start_phase == StepPhase.START + + def has_post_hook_close_profile_stats(self): + return self.end_phase == StepPhase.END + + def has_node_id(self, node_id): + return self.node_id == node_id + + def __repr__(self): + return repr(self.__dict__) + + +class cProfileStats: + """ + Class used to represent cProfile stats captured, given the pStats.Stats object of the desired interval. + ... + Attributes + ---------- + ps: pstats.Stats + The cProfile stats of Python functions as a pStats.Stats object. Useful for high level analysis like sorting + functions by a desired metric and printing the list of profiled functions. + function_stats_list: list of cProfileFunctionStats + The cProfile stats of Python functions as a list of cProfileFunctionStats objects, which contain specific + metrics corresponding to each function profiled. Parsed from the pStats.Stats object. Useful for more in + depth analysis as it allows users physical access to the metrics for each function. + """ + + def __init__(self, ps): + self.ps = ps + self.function_stats_list = [cProfileFunctionStats(k, v) for k, v in ps.stats.items()] + + def print_top_n_functions(self, by, n=10): + """Print the stats for the top n functions with respect to the provided metric. + :param by The metric to sort the functions by. Must be one of the following from the Metrics enum: TOTAL_TIME, + CUMULATIVE_TIME, PRIMITIVE_CALLS, TOTAL_CALLS. + :param n The first n functions and stats to print after sorting. + + For example, to print the top 20 functions with respect to cumulative time spent in function: + >>> from smdebug.profiler.analysis.utils.python_profile_analysis_utils import Metrics + >>> cprofile_stats.print_top_n_function(self, Metrics.CUMULATIVE_TIME, n=20) + """ + assert isinstance(by, Metrics), "by must be valid metric from Metrics!" + assert isinstance(n, int), "n must be an integer!" + self.ps.sort_stats(by.value).print_stats(n) + + def get_function_stats(self): + """Return the function stats list as a DataFrame, where each row represents a cProfileFunctionStats object. + """ + return pd.DataFrame([repr(function_stats) for function_stats in self.function_stats_list]) + + +class cProfileFunctionStats: + """Class used to represent a single profiled function and parsed cProfile stats pertaining to this function. + Processes the stats dictionary's (key, value) pair to get the function name and the specific stats. + Key is a tuple of (filename, lineno, function). + Value is a tuple of (prim_calls, total_calls, total_time, cumulative_time, callers). See below for details. + ... + Attributes + ---------- + function_name: str + The full function name, derived from the key tuple. Defined as filename:lineno(function). + prim_calls: int + The number of primitive (non-recursive) calls to this function. + total_calls: int + The total number of calls to this function. + total_time: int + The total amount of time spent in the scope of this function alone, in seconds. + cumulative_time: int + The total amount of time spent in the scope of this function and in the scope of all other functions + that this function calls, in seconds. + callers: list of str + The list of functions that call this function. Organized as a list of function names, which follow the above + format for function_name: filename:lineno(function) + """ + + def __init__(self, key, value): + self.function_name = pstats.func_std_string(key) + self.prim_calls, self.total_calls, self.total_time, self.cumulative_time, callers = value + self.callers = [pstats.func_std_string(k) for k in callers.keys()] + + def __repr__(self): + return repr( + { + "function name": self.function_name, + "# of primitive calls": self.prim_calls, + "# of total calls": self.total_calls, + "total time": self.total_time, + "cumulative time": self.cumulative_time, + } + ) + + +class PyinstrumentStepStats: + def __init__(self, html_file_path, json_stats): + self.html_file_path = html_file_path + self.json_stats = json_stats diff --git a/smdebug/profiler/analysis/utils/pytorch_dataloader_analysis.py b/smdebug/profiler/analysis/utils/pytorch_dataloader_analysis.py new file mode 100644 index 0000000000..b13c0914a9 --- /dev/null +++ b/smdebug/profiler/analysis/utils/pytorch_dataloader_analysis.py @@ -0,0 +1,166 @@ +# Third Party +import pandas as pd + + +class PT_dataloader_analysis: + def __init__(self, pandas_frame): + self.pd_data_frame = pandas_frame + self.dataIter_metric = self.pd_data_frame.get_all_dataloader_metrics( + selected_framework_metrics=[".*DataLoaderIter::GetNext"] + ) + self.initialize_metric = self.pd_data_frame.get_all_dataloader_metrics( + selected_framework_metrics=["DataLoaderIterInitialize"] + ) + self.dataWorker_metric = self.pd_data_frame.get_all_dataloader_metrics( + selected_framework_metrics=["DataLoaderWorker"] + ) + self.analyze_dataIter = self.dataIter_metric.size > 0 + self.analyze_initializer = self.initialize_metric.size > 0 + self.analyze_workers = self.dataWorker_metric.size > 0 + + def _inspect_iters(self, itertype): + if not self.analyze_initializer: + print("Trace events corresponding to DataLoaderIter initialization are not present") + return + + multiprocessingIter = self.initialize_metric.loc[ + self.initialize_metric["framework_metric"] == itertype + ] + if multiprocessingIter.size == 0: + print(f"Training is not configured to use {itertype} iterator") + return + pin_memory = multiprocessingIter["pin_memory"].unique()[0] + num_workers = multiprocessingIter["num_workers"].unique()[0] + print( + f"Training is configured to use {itertype} with pin_memory enabled = {pin_memory} and number of workers = {num_workers}" + ) + + print( + f"The {itertype} is initialized for {len(multiprocessingIter.index)} times during training" + ) + median_duration = multiprocessingIter["duration_us"].median() + max_duration = multiprocessingIter["duration_us"].max() + + print( + f"Median Duration {median_duration} and Maximum duration for initialization of iterators {max_duration}" + ) + md = multiprocessingIter.loc[ + multiprocessingIter["duration_us"] >= (2 * median_duration), + ["start_time", "end_time", "duration_us"], + ] + + if md.size > 0: + print( + f"Start time and End time for iterator initialization that are outliers (duration > 2 * median)" + ) + + def analyze_dataloaderIter_initialization(self): + self._inspect_iters("_MultiProcessingDataLoaderIter.__init__") + self._inspect_iters("_SingleProcessDataLoaderIter.__init__") + + def analyze_dataloaderWorkers(self): + if not self.analyze_workers: + print("Trace events corresponding to DataLoaderWorker processes are not present") + return + total_num_workers = len(self.dataWorker_metric.index) + print(f"Total number of workers spun off during the training {total_num_workers}") + median_duration = self.dataWorker_metric["duration_us"].median() + max_duration = self.dataWorker_metric["duration_us"].max() + print( + f"Median Duration {median_duration} and Maximum duration for worker processes {max_duration}" + ) + md = self.dataWorker_metric.loc[ + self.dataWorker_metric["duration_us"] >= (2 * median_duration), + ["start_time", "end_time", "duration_us", "worker_id"], + ].sort_values(by=["duration_us"], ascending=False) + md = md.reset_index(drop=True) + if md.size > 0: + print( + f"Start time and End time for iterator initialization that are outliers (duration > 2 * median)" + ) + return md + else: + print(f"No outliers found in dataloader workers") + return None + + def analyze_dataloader_getnext(self): + if not self.analyze_dataIter: + print("Trace events corresponding to DataLoaderIter::GetNext calls are not present") + return + + total_calls = len(self.dataIter_metric.index) + print(f"Total number of GetNext calls made during the training {total_calls}") + median_duration = self.dataIter_metric["duration_us"].median() + max_duration = self.dataIter_metric["duration_us"].max() + print( + f"Median Duration {median_duration} and Maximum duration for worker processes {max_duration}" + ) + md = self.dataIter_metric.loc[ + self.dataIter_metric["duration_us"] >= (2 * median_duration), + ["start_time", "end_time", "duration_us", "worker_id"], + ].sort_values(by=["duration_us"], ascending=False) + md = md.reset_index(drop=True) + if md.size > 0: + print( + f"Start time and End time for GetNext durations that are outliers (duration > 2 * median)" + ) + return md + else: + print(f"No outliers found in dataloader getnext invocations.") + return None + + def analyze_batchtime(self): + if not self.analyze_dataIter: + print("Trace events corresponding to DataLoaderIter::GetNext calls are not present") + return + # Convert start time and end time in pandas datetime object and sort them by end time. + self.dataIter_metric["start_time"] = pd.to_datetime( + self.dataIter_metric["start_time"], format="%Y-%m-%dT%H:%M:%S:%f" + ) + self.dataIter_metric["end_time"] = pd.to_datetime( + self.dataIter_metric["end_time"], format="%Y-%m-%dT%H:%M:%S:%f" + ) + self.dataIter_metric = self.dataIter_metric.sort_values(by=["node_id", "pid", "end_time"]) + # Compute the 'BatchTime_in_seconds' for every GetNext call. + self.dataIter_metric["BatchTime_in_seconds"] = ( + self.dataIter_metric.groupby(["node_id", "pid"])["start_time"].diff().dt.total_seconds() + ) + self.dataIter_metric["previous_batch_start"] = ( + self.dataIter_metric.groupby(["node_id", "pid"])["start_time"].shift().fillna(-1) + ) + median_duration = self.dataIter_metric["BatchTime_in_seconds"].median() + print(f"Median time spent on each batch of data = {median_duration} seconds") + # Get the outlier duration. The cell prints the dataframe that contains the outlier. + md = self.dataIter_metric.loc[ + self.dataIter_metric["BatchTime_in_seconds"] >= (2 * median_duration), + ["previous_batch_start", "start_time", "BatchTime_in_seconds", "worker_id"], + ].sort_values(by=["BatchTime_in_seconds"], ascending=False) + md = md.reset_index(drop=True) + if md.size > 0: + print( + f"Start time and End time for processing the databatches that are outliers (duration > 2 * median)" + ) + return md + else: + print(f"No outliers found in time taken to process the databatches.") + return None + + def plot_the_window( + self, start_timestamp, end_timestamp, select_events=[".*"], select_dimensions=[".*"] + ): + from smdebug.profiler.analysis.notebook_utils.timeline_charts import TimelineCharts + + framework_metrics_reader = self.pd_data_frame.framework_metrics_reader + system_metrics_reader = self.pd_data_frame.system_metrics_reader + framework_metrics_reader.refresh_event_file_list() + system_metrics_reader.refresh_event_file_list() + + view_timeline_charts = TimelineCharts( + system_metrics_reader, + framework_metrics_reader, + starttime=start_timestamp, + endtime=end_timestamp, + select_events=select_events, + select_dimensions=select_dimensions, + ) + return view_timeline_charts diff --git a/smdebug/profiler/hvd_trace_file_rotation.py b/smdebug/profiler/hvd_trace_file_rotation.py new file mode 100644 index 0000000000..bcdda4ec62 --- /dev/null +++ b/smdebug/profiler/hvd_trace_file_rotation.py @@ -0,0 +1,180 @@ +# Standard Library +import json +import os +import threading +import time + +# First Party +from smdebug.core.tfevent.timeline_file_writer import TimelineFileWriter +from smdebug.profiler.profiler_constants import CONVERT_TO_MICROSECS, HOROVODTIMELINE_SUFFIX + + +""" +Horovod produces a large timeline file which is written to by all +Horovod workers. TraceFileRotation starts a reader thread that reads the +large Horovod file throughout the training process and splits the file into +smaller trace files based on Timeline File Writer's rotation policy. +""" + + +class HvdTraceFileRotation: + def __init__(self, profiler_config_parser): + self._profiler_config_parser = profiler_config_parser + self.hvd_file = os.getenv("HOROVOD_TIMELINE", None) + self.enabled = self._should_enable() + if self.enabled: + # base timestamp for event file + self._base_timestamp_in_us = None + + # initial file seek position + self.file_seek_pos = 0 + + # thread event to break out of the thread loop + self._stopper = threading.Event() + + # clock conversion from monotonic clock to std epoch time + self.clock_conversion_in_us = int( + round((time.monotonic() - time.time()) * CONVERT_TO_MICROSECS) + ) + + # default training phase name + self.training_phase = {} + + # timeline writer for writing trace files + self.tl_writer = TimelineFileWriter( + profiler_config_parser=profiler_config_parser, suffix=HOROVODTIMELINE_SUFFIX + ) + + # Hvd file reader thread + self._readerthread = threading.Thread(target=self._read_write_loop, daemon=True) + self._readerthread.start() + + def _should_enable(self): + """ + Enable Horovod file rotation if a timeline file will be written to (based + on the env variable HOROVOD_TIMELINE) and if SM Profiler is enabled. + """ + if self._profiler_config_parser.profiling_enabled and self.hvd_file: + return True + return False + + def _parse_trace_event(self, event): + """ + Parse event to get some information that can be passed to timeline file writer + """ + # all events that reach here must have timestamp + + # convert steady clock to system clock + # This is the absolute timestamp + timestamp_in_secs = ( + self._convert_monotonic_to_epoch_time(event["ts"]) / CONVERT_TO_MICROSECS + ) + + # make a note of duration if present + duration_in_secs = event.get("dur", 0) / CONVERT_TO_MICROSECS + + # make a note of args if present + args = event.get("args", {}) + + # parse instant events which have a special field scope "s" + if "s" in event: + args.update({"s": event["s"]}) + + # get the operation name from the event string + op_name = event.get("name", "") + # get the event pid + pid = event.get("pid", 0) + + return op_name, timestamp_in_secs, duration_in_secs, pid, args + + def _convert_monotonic_to_epoch_time(self, timestamp_in_us): + """ + Horovod writes events based on steady clock/monotonic clock. + Convert this is standard clock which is time since epoch + """ + return int(round(timestamp_in_us - self.clock_conversion_in_us)) + + def _read_write_loop(self): + """ + Reader thread to constantly read the large Horovod trace file and write + events with Timeline File Writer + """ + + # Let the loop continuously run until stop event is set on smdebug hook cleanup + while True: + try: + with open(self.hvd_file) as json_data: + # set the file pointer to the position up to which the reader + # thread has read. + json_data.seek(self.file_seek_pos) + + # for every line read, verify that it is a valid JSON. + for line in json_data: + try: + event = ( + json.loads(line[:-2]) + if line.endswith(",\n") + else json.loads(line[:-1]) + ) + + # the timestamp of the 1st event is considered as base timestamp + if self._base_timestamp_in_us is None: + if "ts" in event: + timestamp = event["ts"] + + # find out the base timestamp + # this is the base timestamp that will be used by timeline file writer as well. + self._base_timestamp_in_us = self._convert_monotonic_to_epoch_time( + timestamp + ) + + # Hvd base timestamp might be earlier than timeline writer's base start time. + # Update timeline writer and the writer thread to avoid negative relative timestamp + # in the rotated files. + self.tl_writer._update_base_start_time( + self._base_timestamp_in_us + ) + + # the name mentioned in metadata events are used as training_phase in TimelineRecord + # make a note of this name. Timeline File Writer will take care of writing + # metadata event for each event + if event["ph"] == "M": + if "name" in event["args"]: + self.training_phase[event["pid"]] = event["args"]["name"] + else: + # parse the event JSON string + op_name, timestamp_in_secs, duration, pid, args = self._parse_trace_event( + event + ) + # write complete, duration, and instant events + self.tl_writer.write_trace_events( + training_phase=self.training_phase[pid], + op_name=op_name, + phase=event["ph"], + timestamp=timestamp_in_secs, + duration=duration, + **args, + ) + except ValueError: + # invalid JSON string, skip + pass + # update file seek position for the next read + self.file_seek_pos = max(self.file_seek_pos, json_data.tell()) + + # stop event has been set, exiting the thread + if self._stopper.isSet(): + break + except (OSError, IOError) as e: + # unable to open timeline file, try again + pass + + time.sleep(15) + + def close(self): + """Flushes the trace event file to disk and close the file. + """ + if self.enabled: + # stop the reader thread + self._stopper.set() + self._readerthread.join() + self.tl_writer.close() diff --git a/smdebug/profiler/metrics_reader_base.py b/smdebug/profiler/metrics_reader_base.py new file mode 100644 index 0000000000..55f0975db9 --- /dev/null +++ b/smdebug/profiler/metrics_reader_base.py @@ -0,0 +1,295 @@ +# Standard Library + +import bisect +import os +import re +import time + +# First Party +from smdebug.core.access_layer.s3handler import S3Handler, is_s3 +from smdebug.core.logger import get_logger +from smdebug.core.utils import list_files_in_directory +from smdebug.profiler.profiler_constants import ( + CONVERT_TO_MICROSECS, + ENV_TRAIILING_DURATION, + PYTHONTIMELINE_SUFFIX, + TRAILING_DURATION_DEFAULT, +) +from smdebug.profiler.utils import ( + TimeUnits, + convert_utc_timestamp_to_microseconds, + is_valid_tfprof_tracefilename, + is_valid_tracefilename, + validate_system_profiler_file, +) + + +class MetricsReaderBase: + def __init__(self, use_in_memory_cache=False): + self.logger = get_logger() + + self._event_parsers = [] + + # This is a list of timestamp -> [event_file] mapping + self._timestamp_to_filename = dict() + + # This is a set of parsed event files. The entry is made into this file only if the complete file is read. + self._parsed_files = set() + + # The startAfter_prefix is used in ListPrefix call to poll for available tracefiles in the S3 bucket. The + # prefix lags behind the last polled tracefile by tunable trailing duration. This is to ensure that we do not + # miss a + # tracefile corresponding to timestamp earlier than last polled timestamp but arrived after we had polled. + + self._startAfter_prefix = "" + self.prefix = "" + self._cache_events_in_memory = use_in_memory_cache + + def get_all_event_parsers(self): + return self._event_parsers + + """ + The function returns the timestamp of first available file. + This timestamp indicates users can query the events up to this timestamp to gauge + """ + + def get_timestamp_of_first_available_file(self): + return ( + sorted(self._timestamp_to_filename.keys())[0] + if len(self._timestamp_to_filename) > 0 + else 0 + ) + + """ + The function returns the timestamp of last available file. + This timestamp indicates users can query the events up to this timestamp to gauge + """ + + def get_timestamp_of_latest_available_file(self): + return ( + sorted(self._timestamp_to_filename.keys())[-1] + if len(self._timestamp_to_filename) > 0 + else 0 + ) + + """ + This function queries the files that are currently available in the directory (for local mode) or in S3 for download. + It rebuilds the map of timestamp to filename. + """ + + def refresh_event_file_list(self): + pass + + def parse_event_files(self, event_files): + pass + + def _get_event_parser(self, filename): + pass + + def _get_event_file_regex(self): + pass + + """ + Return the profiler system event files that were written during the given range. If use_buffer is True, we will consider adding a + buffer of TIME_BUFFER_DEFAULT microseconds to increase the time range. This is done because the events are written to the + file after they end. It is possible that an event would have started within the window of start and end, however it + did not complete at or before 'end' time. Hence the event will not appear in the event file that corresponds to + 'end' timestamp. It will appear in the future event file. + We will also add a buffer for the 'start' i.e. we will look for event files that were written prior to 'start'. + Those files might contain 'B' type events that had started prior to 'start' + """ + + def _get_event_files_in_the_range( + self, start_time_microseconds, end_time_microseconds, use_buffer=True + ): + pass + + """ + The function returns the events that have recorded within the given time range. + The function will download (or parse) the event files that are available + for the given time range. It is possible that events are recorded during training but are not available for + download. + TODO: Implement blocking call to wait for files to be available for download. + """ + + def get_events( + self, + start_time, + end_time, + unit=TimeUnits.MICROSECONDS, + event_type=None, + file_suffix_filter=None, + ): + start_time = convert_utc_timestamp_to_microseconds(start_time, unit) + end_time = convert_utc_timestamp_to_microseconds(end_time, unit) + + all_event_files = self._get_event_files_in_the_range(start_time, end_time) + event_files = list() + if file_suffix_filter is None: + event_files = all_event_files + else: + for eventfile in all_event_files: + if any([eventfile.endswith(suffix) for suffix in file_suffix_filter]): + event_files.append(eventfile) + self.logger.info(f"Getting {len(event_files)} event files") + self.logger.debug(f"Getting event files : {event_files} ") + + # Download files and parse the events + self.parse_event_files(event_files) + + """ + We might have recorded events from different sources within this timerange. + we will get the events from the relevant event parsers and merge them before returning. + """ + result = [] + event_parsers = self.get_all_event_parsers() + # Not all event parsers support event_type as input, only system_profiler_file_parser accepts it + for eventParser in event_parsers: + if event_type is not None: + result.extend( + eventParser.get_events_within_time_range( + start_time, end_time, TimeUnits.MICROSECONDS, event_type + ) + ) + else: + result.extend( + eventParser.get_events_within_time_range( + start_time, end_time, TimeUnits.MICROSECONDS + ) + ) + if not self._cache_events_in_memory: + # clear eventParser events + eventParser.clear_events() + # cleanup parsed files set to force the reading of files again + self._parsed_files = set() + + return result + + def _get_time_interval_for_step(self, start_step, end_step): + """ + Use python timeline files to get time interval for a step interval + """ + event_list = self.get_events( + 0, + time.time() * CONVERT_TO_MICROSECS, + TimeUnits.MICROSECONDS, + file_suffix_filter=[PYTHONTIMELINE_SUFFIX], + ) + + start_time_us = end_time_us = None + event_list.sort(key=lambda x: x.start_time) + for event in event_list: + if ( + hasattr(event, "event_args") + and event.event_args is not None + and "step_num" in event.event_args + ): + # get the start time of start step + if start_time_us is None and start_step == int(event.event_args["step_num"]): + start_time_us = event.start_time + # get the time just before the start of end_step + if end_time_us is None and (end_step == int(event.event_args["step_num"])): + end_time_us = event.start_time - 1 + if start_time_us is not None and end_time_us is not None: + break + + if start_time_us is None or end_time_us is None: + get_logger().info(f"Invalid step interval [{start_step}, {end_step}]") + start_time_us = end_time_us = 0 + return start_time_us, end_time_us + + def get_events_by_step(self, start_step, end_step, event_type=None, file_suffix_filter=None): + """ + Get list of events by step interval + """ + start_time, end_time = self._get_time_interval_for_step(start_step, end_step) + + return self.get_events( + start_time, end_time, TimeUnits.MICROSECONDS, event_type, file_suffix_filter + ) + + """ + It is possible that event files from different nodes to arrive in S3 in different order. For example, Even if t1 + > t2, a event file with timestamp "t1" can arrive in S3 before the tracefile with timestamp "t2". If we list the + prefix only on the basis of last arrived file (i.e. t1) we will miss the file for t2. Therefore, we will set the + start prefix to a timestamp that is trailing behind the last timestamp by 'trailing duration'. This will ensure + that we will attempt to get tracefiles with older timestamp even if they arrive late. + """ + + def _update_start_after_prefix(self): + trailing_duration = os.getenv(ENV_TRAIILING_DURATION, TRAILING_DURATION_DEFAULT) + sorted_timestamps = sorted(self._timestamp_to_filename.keys()) + if len(self._timestamp_to_filename) == 0: + return + last_timestamp_available = sorted_timestamps[-1] + trailing_timestamp = last_timestamp_available - trailing_duration + # Get the timestamp that is closely matching the trailing_timestamp + trailing_timestamp = sorted_timestamps[ + bisect.bisect_left(sorted_timestamps, trailing_timestamp) + ] + self._startAfter_prefix = self._timestamp_to_filename[trailing_timestamp][0] + s3, bucket_name, self._startAfter_prefix = is_s3(self._startAfter_prefix) + + """ + The function opens and reads the event files if they are not already parsed. + For local metrics reader, we are currently assuming that the downloaded event file is a complete file. + """ + + def _parse_event_files_local_mode(self, event_files): + for event_file in event_files: + self.logger.debug(f"Going to parse file:{event_file}") + if ( + is_valid_tracefilename(event_file) + or is_valid_tfprof_tracefilename(event_file) + or validate_system_profiler_file(event_file) + ): + if event_file not in self._parsed_files: + self._get_event_parser(event_file).read_events_from_file(event_file) + self._parsed_files.add(event_file) + else: + self.logger.info( + f"Skipping parsing of file {event_file} as this doesn't look like valid file" + ) + + def _get_timestamp_from_filename(self, event_file): + pass + + """ + Create a map of timestamp to filename + """ + + def _refresh_event_file_list_s3_mode(self, list_dir): + event_files = [ + x for x in S3Handler.list_prefix(list_dir) if re.search(self._get_event_file_regex(), x) + ] + for event_file in event_files: + event_file_full_path = f"s3://{list_dir.bucket}/{event_file}" + timestamp = self._get_timestamp_from_filename(event_file_full_path) + if timestamp is None: + self.logger.debug(f"Unable to find timestamp from event file name {event_file}.") + continue + if timestamp in self._timestamp_to_filename: + if event_file_full_path not in self._timestamp_to_filename[timestamp]: + self._timestamp_to_filename[timestamp].append(event_file_full_path) + else: + self._timestamp_to_filename[timestamp] = [event_file_full_path] + for timestamp in self._timestamp_to_filename: + self._timestamp_to_filename[timestamp].sort() + self._update_start_after_prefix() + + def _refresh_event_file_list_local_mode(self, trace_root_folder): + path = os.path.expanduser(trace_root_folder) + event_dir = os.path.join(path, self.prefix, "") + event_files = list_files_in_directory(event_dir, file_regex=self._get_event_file_regex()) + for event_file in event_files: + timestamp = self._get_timestamp_from_filename(event_file) + if timestamp is None: + self.logger.debug(f"Unable to find timestamp from event file name {event_file}.") + continue + if timestamp in self._timestamp_to_filename: + if event_file not in self._timestamp_to_filename[timestamp]: + self._timestamp_to_filename[timestamp].append(event_file) + else: + self._timestamp_to_filename[timestamp] = [event_file] + for timestamp in self._timestamp_to_filename: + self._timestamp_to_filename[timestamp].sort() diff --git a/smdebug/profiler/profiler_config.py b/smdebug/profiler/profiler_config.py new file mode 100644 index 0000000000..84615b0a77 --- /dev/null +++ b/smdebug/profiler/profiler_config.py @@ -0,0 +1,250 @@ +# Standard Library +import re +import time +from enum import Enum + +# First Party +from smdebug.profiler.profiler_constants import ( + CPROFILE_NAME, + PROFILING_NUM_STEPS_DEFAULT, + PYINSTRUMENT_NAME, +) +from smdebug.profiler.python_profiler import cProfileTimer + + +class MetricsConfigsField(Enum): + """Enum to track each field parsed from any particular metrics config. + """ + + START_STEP = "startstep" + NUM_STEPS = "numsteps" + START_TIME = "starttimeinsecsinceepoch" + DURATION = "durationinseconds" + DEFAULT_PROFILING = "defaultprofiling" + METRICS_REGEX = "metricsregex" + PROFILER_NAME = "profilername" + CPROFILE_TIMER = "cprofiletimer" + + +class RotationPolicy: + """Configuration corresponding to rotation policy of trace event files. + """ + + def __init__(self, file_max_size, file_close_interval): + self.file_max_size = file_max_size + self.file_close_interval = file_close_interval + + +class TraceFile: + """Configuration corresponding to trace event files. + """ + + def __init__(self, file_max_size, file_close_interval, file_open_fail_threshold): + self.rotation_policy = RotationPolicy(file_max_size, file_close_interval) + self.file_open_fail_threshold = file_open_fail_threshold + + +class ProfileRange: + """Configuration common to all of the configs: the start of profiling, how long profiling should take and whether + an error occurred in parsing the config. Assigns each field from the dictionary, (or `None` if it doesn't exist) + """ + + def __init__(self, name, profile_range): + self.name = name + self.error_message = None + self.disabled = False + + # user did not specify this config, so it is disabled. + if profile_range == {}: + self.disabled = True + return + + # step range + self.start_step = profile_range.get(MetricsConfigsField.START_STEP.value) + self.num_steps = profile_range.get(MetricsConfigsField.NUM_STEPS.value) + self.end_step = None + + # time range + self.start_time_in_sec = profile_range.get(MetricsConfigsField.START_TIME.value) + self.duration_in_sec = profile_range.get(MetricsConfigsField.DURATION.value) + self.end_time = None + + # convert to the correct type + try: + if self.start_step is not None: + self.start_step = int(self.start_step) + if self.num_steps is not None: + self.num_steps = int(self.num_steps) + if self.start_time_in_sec is not None: + self.start_time_in_sec = float(self.start_time_in_sec) + if self.duration_in_sec is not None: + self.duration_in_sec = float(self.duration_in_sec) + except ValueError as e: + self.error_message = f"{e} encountered in {self.name} config. Disabling {self.name}." + self.reset_profile_range() + return + + if self.has_step_range() and self.has_time_range(): + self.error_message = ( + f"Found step and time fields in {self.name} config! Disabling {self.name }." + ) + + def has_step_range(self): + """Return whether one of the step fields (start step or num steps) has been specified in the config. + """ + return self.start_step or self.num_steps + + def has_time_range(self): + """Return whether one of the time fields (start time or duration) has been specified in the config. + """ + return self.start_time_in_sec or self.duration_in_sec + + def reset_profile_range(self): + """Helper function to reset fields in profile range. Used primarily when an error parsing config occurs. + """ + self.start_step = self.num_steps = self.start_time_in_sec = self.duration_in_sec = None + + def is_enabled(self): + """Return whether this config is enabled (specified by the user and no errors during parsing""" + return not self.disabled and self.error_message is None + + def can_start_profiling(self, current_step, current_time): + """Determine whether the values from the config are valid for profiling.""" + if not self.is_enabled(): + return False + + if current_time is None: + current_time = time.time() + if self.has_step_range(): + if self.start_step is None: + self.start_step = current_step + if self.num_steps is None: + self.num_steps = PROFILING_NUM_STEPS_DEFAULT + if not self.end_step: + self.end_step = self.start_step + self.num_steps + return self.start_step <= current_step < self.end_step + elif self.has_time_range(): + if self.start_time_in_sec is None: + self.start_time_in_sec = current_time + if self.duration_in_sec: + if self.end_time is None: + self.end_time = self.start_time_in_sec + self.duration_in_sec + return self.start_time_in_sec <= current_time < self.end_time + else: + if self.start_time_in_sec <= current_time: + if self.end_step is None: + self.end_step = current_step + 1 + return current_step < self.end_step + return False + + +class DetailedProfilingConfig(ProfileRange): + """Configuration corresponding to the detailed profiling config.""" + + def __init__(self, detailed_profiling_config): + super().__init__("detailed profiling", detailed_profiling_config) + + +class DataloaderProfilingConfig(ProfileRange): + """Configuration corresponding to the dataloader profiling config. + + TODO: Use this config to collect dataloader metrics only for the specified steps. + """ + + def __init__(self, dataloader_config): + super().__init__("dataloader profiling", dataloader_config) + + if self.error_message: + return + + try: + self.metrics_regex = re.compile( + dataloader_config.get(MetricsConfigsField.METRICS_REGEX.value, ".*") + ) + except re.error as e: + self.metrics_regex = None + self.error_message = f"{e} encountered in {self.name} config. Disabling {self.name}." + self.reset_profile_range() + + def valid_metrics_name(self, metrics_name): + """Check if the metrics regex matches the provided metrics name. Note: this is case insensitive. + """ + if self.metrics_regex is None: + return False + return self.metrics_regex.match(metrics_name.lower()) is not None + + +class PythonProfilingConfig(ProfileRange): + """Configuration corresponding to the python profiling config. """ + + def __init__(self, python_profiling_config): + super().__init__("python profiling", python_profiling_config) + + if self.error_message: + return + + self.profiler_name = python_profiling_config.get( + MetricsConfigsField.PROFILER_NAME.value, CPROFILE_NAME + ) + if self.profiler_name not in (CPROFILE_NAME, PYINSTRUMENT_NAME): + self.error_message = f"Profiler name must be {CPROFILE_NAME} or {PYINSTRUMENT_NAME}!" + self.profiler_name = self.cprofile_timer = None + self.reset_profile_range() + return + + try: + self.cprofile_timer = None + if self.profiler_name == CPROFILE_NAME: + # only parse this field if pyinstrument is not specified and we are not doing the default three steps + # of profiling. + self.cprofile_timer = cProfileTimer( + python_profiling_config.get( + MetricsConfigsField.CPROFILE_TIMER.value, "total_time" + ) + ) + except ValueError as e: + self.error_message = f"{e} encountered in {self.name} config. Disabling {self.name}." + self.profiler_name = self.cprofile_timer = None + self.reset_profile_range() + + +class SMDataParallelProfilingConfig(ProfileRange): + """Configuration corresponding to the smdataparallel profiling config.""" + + def __init__(self, smdataparallel_profiling_config): + super().__init__("smdataparallel profiling", smdataparallel_profiling_config) + + +class ProfilerConfig: + """Overall profiler configuration + """ + + def __init__( + self, + local_path, + file_max_size, + file_close_interval, + file_open_fail_threshold, + detailed_profiling_config, + dataloader_profiling_config, + python_profiling_config, + smdataparallel_profiling_config, + ): + """ + :param local_path: path where profiler events have to be saved. + :param file_max_size: Max size a trace file can be, before being rotated. + :param file_close_interval: Interval in seconds from the last close, before being rotated. + :param file_open_fail_threshold: Number of times to attempt to open a trace fail before marking the writer as unhealthy. + :param detailed_profiling_config Dictionary holding the detailed profiling config. + :param dataloader_profiling_config Dictionary holding the dataloader profiling config. + :param python_profiling_config Dictionary holding the python profiling config. + :param smdataparallel_profiling_config Dictionary holding the SMDataParallel profiling config. + """ + self.local_path = local_path + self.trace_file = TraceFile(file_max_size, file_close_interval, file_open_fail_threshold) + self.detailed_profiling_config = DetailedProfilingConfig(detailed_profiling_config) + self.dataloader_profiling_config = DataloaderProfilingConfig(dataloader_profiling_config) + self.python_profiling_config = PythonProfilingConfig(python_profiling_config) + self.smdataparallel_profiling_config = SMDataParallelProfilingConfig( + smdataparallel_profiling_config + ) diff --git a/smdebug/profiler/profiler_config_parser.py b/smdebug/profiler/profiler_config_parser.py new file mode 100644 index 0000000000..6f49e0efe3 --- /dev/null +++ b/smdebug/profiler/profiler_config_parser.py @@ -0,0 +1,326 @@ +# Standard Library +import json +import os +from collections import defaultdict +from enum import Enum + +# First Party +from smdebug.core.json_config import get_node_id_from_resource_config +from smdebug.core.logger import get_logger +from smdebug.core.utils import is_first_process +from smdebug.profiler.profiler_config import ProfilerConfig +from smdebug.profiler.profiler_constants import ( + BASE_FOLDER_DEFAULT, + CLOSE_FILE_INTERVAL_DEFAULT, + CONFIG_PATH_DEFAULT, + FILE_OPEN_FAIL_THRESHOLD_DEFAULT, + MAX_FILE_SIZE_DEFAULT, +) + + +class LastProfilingStatus(Enum): + """Enum to track last profiling status so that we log for any changes + """ + + START = 1 + CONFIG_NOT_FOUND = 2 + INVALID_CONFIG = 3 + DEFAULT_ENABLED = 4 + PROFILER_DISABLED = 5 + PROFILER_ENABLED = 6 + DEFAULT_VALUES = 7 + DEFAULT_PYTHON_PROFILING = 8 + INVALID_DETAILED_PROFILING_CONFIG = 9 + INVALID_DATALOADER_PROFILING_CONFIG = 10 + INVALID_PYTHON_PROFILING_CONFIG = 11 + INVALID_DETAILED_CONFIG_FIELDS = 12 + INVALID_DATALOADER_CONFIG_FIELDS = 13 + INVALID_PYTHON_CONFIG_FIELDS = 14 + DETAILED_CONFIG_NOT_FOUND = 15 + INVALID_SMDATAPARALLEL_PROFILING_CONFIG = 16 + + +class MetricsCategory(Enum): + """Enum to track each possible metrics that can be collected. + """ + + DETAILED_PROFILING = 1 + DATALOADER_PROFILING = 2 + PYTHON_PROFILING = 3 + SMDATAPARALLEL_PROFILING = 4 + + +class ProfilingParametersField(Enum): + """Enum to track each field parsed from the profiling parameters. + """ + + PROFILING_PARAMETERS = "profilingparameters" + DISABLE_PROFILER = "disableprofiler" + LOCAL_PATH = "localpath" + FILE_MAX_SIZE = "rotatemaxfilesizeinbytes" + FILE_CLOSE_INTERVAL = "rotatefilecloseintervalinseconds" + FILE_OPEN_FAIL_THRESHOLD = "fileopenfailthreshold" + DETAILED_PROFILING_CONFIG = "detailedprofilingconfig" + DATALOADER_PROFILING_CONFIG = "dataloaderprofilingconfig" + PYTHON_PROFILING_CONFIG = "pythonprofilingconfig" + SMDATAPARALLEL_PROFILING_CONFIG = "smdataparallelprofilingconfig" + + +class ProfilerConfigParser: + """Load the configuration file for the Profiler. + Also set the provided values for the specified variables and default values for the rest. + + TODO: Poll for changes in the config by repeatedly calling `load_config`. + """ + + def __init__(self): + """Initialize the parser to be disabled for profiling and detailed profiling. + """ + self.last_json_config = None + self.config = None + self.profiling_enabled = False + self.logger = get_logger() + self.last_logging_statuses = defaultdict(lambda: False) + self.current_logging_statuses = defaultdict(lambda: False) + self.load_config() + + def _reset_statuses(self): + """Set the last logging statuses to be the current logging statuses and reset the current logging statuses. + """ + self.last_logging_statuses = self.current_logging_statuses + self.current_logging_statuses = defaultdict(lambda: False) + + def _log_new_message(self, status, log_function, message): + """Helper function to log the given message only if the given status was not True in the last call to + load_config. In other words, only log this message if it wasn't already logged before. Use the provided log + function to log the message + + Also mark this status as True so that we do not log the message in the next call to load_config if the status + is once again True. + """ + if not self.last_logging_statuses[status]: + log_function(message) + self.current_logging_statuses[status] = True + + def _parse_metrics_config(self, config, config_name, invalid_status): + """Helper function to parse each metrics config from config given the ProfilingParametersField (config_name). + If an error occurs in parsing, log the message with the provided LastProfilingStatus (invalid_status). + """ + try: + metrics_config = eval(config.get(config_name.value, "{}")) + assert isinstance(metrics_config, dict) + return metrics_config + except (ValueError, AssertionError) as e: + self._log_new_message( + invalid_status, + self.logger.error, + f"{e} in {config_name.value}. Default metrics collection will be enabled.", + ) + return {} + + def load_config(self): + """Load the config file (if it exists) from $SMPROFILER_CONFIG_PATH. + Set the provided values for the specified variables and default values for the rest. + Validate the detailed profiling config (if it exists). + """ + config_path = os.environ.get("SMPROFILER_CONFIG_PATH", CONFIG_PATH_DEFAULT) + + if os.path.isfile(config_path): + with open(config_path) as json_data: + try: + full_config = json.loads(json_data.read().lower()) + + if full_config == self.last_json_config: + return + + self.last_json_config = full_config + self.config = None + + if full_config.get(ProfilingParametersField.DISABLE_PROFILER.value, False): + self._log_new_message( + LastProfilingStatus.PROFILER_DISABLED, + self.logger.info, + f"User has disabled profiler.", + ) + self._reset_statuses() + self.profiling_enabled = False + return + except Exception as e: + self._log_new_message( + LastProfilingStatus.INVALID_CONFIG, + self.logger.error, + f"Error parsing config at {config_path}: {str(e)}", + ) + self._reset_statuses() + self.config = None + self.profiling_enabled = False + return + config = full_config.get(ProfilingParametersField.PROFILING_PARAMETERS.value) + if config is None or config == {}: + self._log_new_message( + LastProfilingStatus.PROFILER_DISABLED, + self.logger.info, + f"User has disabled profiler.", + ) + self._reset_statuses() + self.profiling_enabled = False + return + else: + self._log_new_message( + LastProfilingStatus.PROFILER_ENABLED, + self.logger.info, + f"Using config at {config_path}.", + ) + self.profiling_enabled = True + else: + self._log_new_message( + LastProfilingStatus.CONFIG_NOT_FOUND, + self.logger.info, + f"Unable to find config at {config_path}. Profiler is disabled.", + ) + self._reset_statuses() + self.profiling_enabled = False + return + + try: + local_path = config.get(ProfilingParametersField.LOCAL_PATH.value, BASE_FOLDER_DEFAULT) + file_max_size = int( + float( + config.get(ProfilingParametersField.FILE_MAX_SIZE.value, MAX_FILE_SIZE_DEFAULT) + ) + ) + file_close_interval = float( + config.get( + ProfilingParametersField.FILE_CLOSE_INTERVAL.value, CLOSE_FILE_INTERVAL_DEFAULT + ) + ) + file_open_fail_threshold = int( + config.get( + ProfilingParametersField.FILE_OPEN_FAIL_THRESHOLD.value, + FILE_OPEN_FAIL_THRESHOLD_DEFAULT, + ) + ) + except ValueError as e: + self._log_new_message( + LastProfilingStatus.DEFAULT_VALUES, + self.logger.info, + f"{e} in {ProfilingParametersField.PROFILING_PARAMETERS}. Enabling profiling with default " + f"parameter values.", + ) + local_path = BASE_FOLDER_DEFAULT + file_max_size = MAX_FILE_SIZE_DEFAULT + file_close_interval = CLOSE_FILE_INTERVAL_DEFAULT + file_open_fail_threshold = FILE_OPEN_FAIL_THRESHOLD_DEFAULT + + detailed_profiling_config = self._parse_metrics_config( + config, + ProfilingParametersField.DETAILED_PROFILING_CONFIG, + LastProfilingStatus.INVALID_DETAILED_PROFILING_CONFIG, + ) + dataloader_profiling_config = self._parse_metrics_config( + config, + ProfilingParametersField.DATALOADER_PROFILING_CONFIG, + LastProfilingStatus.INVALID_DATALOADER_PROFILING_CONFIG, + ) + python_profiling_config = self._parse_metrics_config( + config, + ProfilingParametersField.PYTHON_PROFILING_CONFIG, + LastProfilingStatus.INVALID_PYTHON_PROFILING_CONFIG, + ) + smdataparallel_profiling_config = self._parse_metrics_config( + config, + ProfilingParametersField.SMDATAPARALLEL_PROFILING_CONFIG, + LastProfilingStatus.INVALID_SMDATAPARALLEL_PROFILING_CONFIG, + ) + + self.config = ProfilerConfig( + local_path, + file_max_size, + file_close_interval, + file_open_fail_threshold, + detailed_profiling_config, + dataloader_profiling_config, + python_profiling_config, + smdataparallel_profiling_config, + ) + + if self.config.detailed_profiling_config.error_message is not None: + self._log_new_message( + LastProfilingStatus.INVALID_DETAILED_CONFIG_FIELDS, + self.logger.error, + self.config.detailed_profiling_config.error_message, + ) + + if self.config.dataloader_profiling_config.error_message is not None: + self._log_new_message( + LastProfilingStatus.INVALID_DATALOADER_CONFIG_FIELDS, + self.logger.error, + self.config.dataloader_profiling_config.error_message, + ) + + if self.config.python_profiling_config.error_message is not None: + self._log_new_message( + LastProfilingStatus.INVALID_PYTHON_CONFIG_FIELDS, + self.logger.error, + self.config.python_profiling_config.error_message, + ) + + if self.config.smdataparallel_profiling_config.error_message is not None: + self._log_new_message( + LastProfilingStatus.INVALID_SMDATAPARALLEL_PROFILING_CONFIG, + self.logger.error, + self.config.smdataparallel_profiling_config.error_message, + ) + + self._reset_statuses() + + def should_save_metrics( + self, metrics_category, current_step, metrics_name=None, current_time=None + ): + """Takes in a metrics category and current step and returns whether to collect metrics for that step. Metrics + category must be one of the metrics specified in MetricNames. If metrics category is Dataloader, then metrics + name is required and check if the metrics regex specified in the dataloader config matches this name. + """ + if not self.profiling_enabled: + return False + + if metrics_category == MetricsCategory.DETAILED_PROFILING: + metric_config = self.config.detailed_profiling_config + elif metrics_category == MetricsCategory.DATALOADER_PROFILING: + metric_config = self.config.dataloader_profiling_config + if metrics_name is not None and not metric_config.valid_metrics_name(metrics_name): + return False + elif metrics_category == MetricsCategory.PYTHON_PROFILING: + metric_config = self.config.python_profiling_config + elif metrics_category == MetricsCategory.SMDATAPARALLEL_PROFILING: + metric_config = self.config.smdataparallel_profiling_config + else: + return False # unrecognized metrics category + + return metric_config.can_start_profiling(current_step, current_time) + + def write_tf_dataloader_flag(self, flag_filename): + """If dataloader metrics collection is enabled, then write a .tmp file with the provided flag_filename such + that is has this path: //. We simply create the file but never close the + writer, since we don't want the file to be uploaded to s3. + + If flag_filename is TF_DATALOADER_START_FLAG_FILENAME, we are signaling that dataloader metrics should be + collected now. If flag_filename is TF_DATALOADER_END_FLAG_FILENAME, we are signaling that dataloader metrics + should not be collected anymore. In AWS TF, we will collect dataloader metrics when only + TF_DATALOADER_START_FLAG_FILENAME exists and not collect dataloader metrics when neither or both flags exist. + + Return True if writing the flag was successful, False if unsuccessful. + """ + if not self.profiling_enabled or not self.config.dataloader_profiling_config.is_enabled(): + return + + tf_dataloader_flag_path = os.path.join( + self.config.local_path, get_node_id_from_resource_config(), flag_filename + ) + success = is_first_process(tf_dataloader_flag_path, is_dir=False) + + if not os.path.isfile(tf_dataloader_flag_path): + self.logger.error(f"Could not write flag to: {tf_dataloader_flag_path}!") + return False + + return success diff --git a/smdebug/profiler/profiler_constants.py b/smdebug/profiler/profiler_constants.py new file mode 100644 index 0000000000..5459d580cc --- /dev/null +++ b/smdebug/profiler/profiler_constants.py @@ -0,0 +1,72 @@ +# The traceevents will be stored in following format +# $ENV_BASE_FOLDER/framework/pevents/$START_TIME_YYYYMMDDHR/$FILEEVENTSTARTTIMEUTCINEPOCH_{ +# $ENV_NODE_ID_4digits0padded}_model_timeline.json +DEFAULT_PREFIX = "framework/pevents" +DEFAULT_SYSTEM_PROFILER_PREFIX = "system" +TRACE_DIRECTORY_FORMAT = "%Y%m%d%H" +PYTHONTIMELINE_SUFFIX = "pythontimeline.json" +MODELTIMELINE_SUFFIX = "model_timeline.json" +TENSORBOARDTIMELINE_SUFFIX = "trace.json.gz" +HOROVODTIMELINE_SUFFIX = "horovod_timeline.json" +SMDATAPARALLELTIMELINE_SUFFIX = "smdataparallel_timeline.json" +MERGEDTIMELINE_SUFFIX = "merged_timeline.json" +PT_DATALOADER_WORKER = "DataLoaderWorker" +PT_DATALOADER_ITER = "DataLoaderIter" +PT_DATALOADER_INITIALIZE = "DataLoaderIterInitialize" +TF_DATALOADER_ITER = "DataIterator" + +""" +When users query the events within certain time range, the value TIME BUFFER_SECONDS is used to extend the time range. +This is done so that we can find the candidate tracefiles that can potentially contain the events that might have +started within the given range but not completed by the given 'end' timestamp. Such events will be reported in the +tracefile corresponding to timestamp later than the given 'end' timestamp. +""" +# Environment variable to set the time buffer in seconds. +ENV_TIME_BUFFER = "TIME_BUFFER_MICROSECONDS" +# In order to look for events occurred within a window, we will add a buffer of 3 minutes on each side (start and +# time) to look for trace files. +TIME_BUFFER_DEFAULT = 3 * 60 * 1000 * 1000 # 3 minutes + +""" +The S3MetricReader obtains the list of prefixes (i.e. the list of tracefiles available in S3 bucket) using +‘list_objects_v2’ API and providing the ‘start_prefix’. The ‘start_prefix’ indicates ‘list_object_v2’ +API to return the prefixes that are after the ‘start_prefix’. +If we set the 'start_prefix' equivalent to the latest available tracefile obtained in the previous ‘list_object_v2’, +we may miss the files that have timestamps less than latest available timestamp but arrived later (after the previous +invocation of 'list_object_v2'). +For example, assume that the latest available file contains the end timestamp to be ‘T_n’. It is possible that in +one of the nodes in distributed training, there exists a file ‘T_m’ (where T_m < Tn) that is closed but not yet +uploaded to S3. If we set ‘start_prefix’ to be ‘T_n’ for subsequent ‘list_objects_v2’, we will never enumerate ‘T_m’ +file and hence we will never report events from that file. +In order to handle such cases, we set the ‘start_prefix’ to be trailing behind the last available +timestamp. The environment variable "TRAILING_DURATION_SECONDS" controls how far the start_prefix is trailing behind +the latests tracefile available. The default value for this duration is 5 minutes. +This means that we expect that if the file is closed on the node, we will wait for 5 minutes for it to be uploaded to S3. +""" +# Environment variable to set the trailing duration in seconds. +ENV_TRAIILING_DURATION = "TRAILING_DURATION_MICROSECONDS" +# This is a duration used for computing the start after prefix. +TRAILING_DURATION_DEFAULT = 5 * 60 * 1000 * 1000 # 5 minutes + +CONFIG_PATH_DEFAULT = "/opt/ml/input/config/profilerconfig.json" +CONVERT_TO_MICROSECS = 1000000 +CONVERT_MICRO_TO_NS = 1000 +MAX_FILE_SIZE_DEFAULT = 10485760 # default 10MB +CLOSE_FILE_INTERVAL_DEFAULT = 60 # default 60 seconds +FILE_OPEN_FAIL_THRESHOLD_DEFAULT = 50 +BASE_FOLDER_DEFAULT = "/opt/ml/output/profiler" + +PYTHON_PROFILING_START_STEP_DEFAULT = 9 +PROFILING_NUM_STEPS_DEFAULT = 1 +PROFILER_DURATION_DEFAULT = float("inf") + +TF_METRICS_PREFIX = "aws_marker-" + +TF_DATALOADER_START_FLAG_FILENAME = "tf_dataloader_start_flag.tmp" +TF_DATALOADER_END_FLAG_FILENAME = "tf_dataloader_end_flag.tmp" + +CPROFILE_STATS_FILENAME = "python_stats" +PYINSTRUMENT_JSON_FILENAME = "python_stats.json" +PYINSTRUMENT_HTML_FILENAME = "python_stats.html" +CPROFILE_NAME = "cprofile" +PYINSTRUMENT_NAME = "pyinstrument" diff --git a/smdebug/profiler/python_profile_utils.py b/smdebug/profiler/python_profile_utils.py new file mode 100644 index 0000000000..96a62dfa2d --- /dev/null +++ b/smdebug/profiler/python_profile_utils.py @@ -0,0 +1,94 @@ +# Standard Library +import os +from enum import Enum, IntEnum + +# First Party +from smdebug.profiler.profiler_constants import CPROFILE_NAME, PYINSTRUMENT_NAME + + +class PythonProfileModes(IntEnum): + TRAIN = 1 # training/fitting mode + EVAL = 2 # testing/evaluation mode + PREDICT = 3 # prediction/inference mode + GLOBAL = 4 # default mode + PRE_STEP_ZERO = 5 # when hook is imported + POST_HOOK_CLOSE = 6 # when hook is closed + + +class StepPhase(Enum): + START = "start" # pre-step zero + STEP_START = "stepstart" # start of step + FORWARD_PASS_END = "forwardpassend" # end of forward pass + STEP_END = "stepend" # end of training step + END = "end" # end of script + + +class PythonProfilerName(Enum): + CPROFILE = CPROFILE_NAME + PYINSTRUMENT = PYINSTRUMENT_NAME + + +class cProfileTimer(Enum): + TOTAL_TIME = "total_time" + CPU_TIME = "cpu_time" + OFF_CPU_TIME = "off_cpu_time" + DEFAULT = "default" + + +def str_to_python_profile_mode(mode_str): + if mode_str == "train": + return PythonProfileModes.TRAIN + elif mode_str == "eval": + return PythonProfileModes.EVAL + elif mode_str == "predict": + return PythonProfileModes.PREDICT + elif mode_str == "global": + return PythonProfileModes.GLOBAL + elif mode_str == "prestepzero": + return PythonProfileModes.PRE_STEP_ZERO + elif mode_str == "posthookclose": + return PythonProfileModes.POST_HOOK_CLOSE + else: + raise Exception("Invalid mode") + + +def python_profile_mode_to_str(mode): + if mode == PythonProfileModes.TRAIN: + return "train" + elif mode == PythonProfileModes.EVAL: + return "eval" + elif mode == PythonProfileModes.PREDICT: + return "predict" + elif mode == PythonProfileModes.GLOBAL: + return "global" + elif mode == PythonProfileModes.PRE_STEP_ZERO: + return "prestepzero" + elif mode == PythonProfileModes.POST_HOOK_CLOSE: + return "posthookclose" + else: + raise Exception("Invalid mode") + + +def mode_keys_to_python_profile_mode(mode): + return PythonProfileModes(mode.value) + + +def total_time(): + if not os.times: + return -1 + times = os.times() + return times.elapsed + + +def off_cpu_time(): + if not os.times: + return -1 + times = os.times() + return times.elapsed - (times.system + times.user) + + +def cpu_time(): + if not os.times: + return -1 + times = os.times() + return times.system + times.user diff --git a/smdebug/profiler/python_profiler.py b/smdebug/profiler/python_profiler.py new file mode 100644 index 0000000000..4c2b538196 --- /dev/null +++ b/smdebug/profiler/python_profiler.py @@ -0,0 +1,255 @@ +# Standard Library +import json +import os +import pstats +import time +from cProfile import Profile as cProfileProfiler + +# Third Party +from pyinstrument import Profiler as PyinstrumentProfiler +from pyinstrument.renderers import JSONRenderer + +# First Party +from smdebug.core.locations import TraceFileLocation +from smdebug.core.logger import get_logger +from smdebug.profiler.profiler_constants import ( + CONVERT_TO_MICROSECS, + CPROFILE_NAME, + CPROFILE_STATS_FILENAME, + PYINSTRUMENT_HTML_FILENAME, + PYINSTRUMENT_JSON_FILENAME, + PYINSTRUMENT_NAME, + PYTHON_PROFILING_START_STEP_DEFAULT, +) +from smdebug.profiler.python_profile_utils import ( + PythonProfileModes, + cProfileTimer, + cpu_time, + off_cpu_time, + python_profile_mode_to_str, + total_time, +) + + +class PythonProfiler: + _name = "" # placeholder + + def __init__(self, base_folder, framework): + """Higher level class to manage execution of python profiler, dumping of python stats, and retrieval + of stats based on time or step intervals. + + ... + + Attributes + ---------- + base_folder: str + The base folder path for profiling, retrieved from the profiler config parser. + framework: str + The name of the framework associated with the hook that the profiler is being run in. + _profiler: cProfile.Profiler | pyinstrument.Profiler + The python profiler object. Enabled/disabled to create individual stats files. Instantiated for every + step profiled. This will be set in subclass, depending on what profiler is used (cProfile, pyinstrument, + etc.) + _step: int + If the python profiler is running, this is the step that it is profiling. Otherwise, this is `None`. + _start_time_since_epoch_in_micros: int + If the python profiler is running, this is the UTC time (in microseconds) at which it started profiling. + Otherwise, this is `None`. + _is_profiling: bool + Whether the profiler is currently running now or not. + """ + self._base_folder = base_folder + self._framework = framework + self._profiler = None # placeholder + self._reset_profiler() + + def _reset_profiler(self): + """Reset attributes to defaults + """ + self._start_mode = None + self._start_step = None + self._start_phase = None + self._start_time_since_epoch_in_micros = None + self._is_profiling = None + + def _enable_profiler(self): + """Enable the profiler (to be implemented in subclass, where the actual profiler is defined). + """ + + def _disable_profiler(self): + """Disable the profiler (to be implemented in subclass, where the actual profiler is defined). + """ + + def _dump_stats(self, stats_path): + """Dump the stats to the provided path (to be implemented in subclass, where the actual profiler is defined). + """ + + def start_profiling( + self, start_phase, start_mode=PythonProfileModes.PRE_STEP_ZERO, start_step="*" + ): + """Start the python profiler with the provided start phase and start step and start mode. + Start phase must be one of the specified step phases in StepPhase. + Start mode must be one of the specified modes in ModeKeys. + If start step is *, then this is profiling until step 0. + """ + self._start_mode = start_mode + self._start_step = start_step + self._start_phase = start_phase + self._start_time_since_epoch_in_micros = time.time() * CONVERT_TO_MICROSECS + self._is_profiling = True + self._enable_profiler() + + def stop_profiling(self, end_phase, end_mode=PythonProfileModes.POST_HOOK_CLOSE, end_step="*"): + """Stop the python profiler with the provided end phase and end step and end mode. + End phase must be one of the specified step phases in StepPhase. + End mode must be one of the specified modes in ModeKeys. + Dump the python stats for this step with a file path dependent on the base folder, framework, time and step. + Append a record of this step's profiling with the corresponding metadata. + Reset the attributes to prepare for the (possibly) next time we profile. + If end step is *, then this is profiling until the end of the script. + """ + if not self._is_profiling: + return + + self._disable_profiler() + + end_time_since_epoch_in_micros = time.time() * CONVERT_TO_MICROSECS + stats_dir = TraceFileLocation.get_python_profiling_stats_dir( + self._base_folder, + self._name, + self._framework, + python_profile_mode_to_str(self._start_mode), + self._start_step, + self._start_phase.value, + self._start_time_since_epoch_in_micros, + python_profile_mode_to_str(end_mode), + end_step, + end_phase.value, + end_time_since_epoch_in_micros, + ) + self._dump_stats(stats_dir) + + self._reset_profiler() + + @staticmethod + def get_python_profiler(profiler_config, framework): + base_folder = profiler_config.local_path + python_profiling_config = profiler_config.python_profiling_config + if python_profiling_config.profiler_name == CPROFILE_NAME: + cprofile_timer = python_profiling_config.cprofile_timer + if cprofile_timer == cProfileTimer.DEFAULT: + return cProfileDefaultPythonProfiler(base_folder, framework) + else: + return cProfilePythonProfiler(base_folder, framework, cprofile_timer) + else: + return PyinstrumentPythonProfiler(base_folder, framework) + + +class cProfilePythonProfiler(PythonProfiler): + """Higher level class to oversee profiling specific to cProfile, Python's native profiler in . + This is also the default Python profiler used if profiling is enabled. + """ + + _name = CPROFILE_NAME + timer_name_to_function = { + cProfileTimer.TOTAL_TIME: total_time, + cProfileTimer.CPU_TIME: cpu_time, + cProfileTimer.OFF_CPU_TIME: off_cpu_time, + } + + def __init__(self, base_folder, framework, cprofile_timer): + super().__init__(base_folder, framework) + if cprofile_timer == "default": + self.cprofile_timer = None # will be set in subclass + else: + self.cprofile_timer = self.timer_name_to_function[cprofile_timer] + + def _enable_profiler(self): + """Enable the cProfile profiler with the current cProfile timer. + """ + self._profiler = cProfileProfiler(self.cprofile_timer) + self._profiler.enable() + + def _disable_profiler(self): + """Disable the cProfile profiler. + """ + self._profiler.disable() + + def _dump_stats(self, stats_dir): + """Dump the stats by via pstats object to a file `python_stats` in the provided stats directory. + """ + stats_file_path = os.path.join(stats_dir, CPROFILE_STATS_FILENAME) + get_logger().info(f"Dumping cProfile stats to {stats_file_path}.") + pstats.Stats(self._profiler).dump_stats(stats_file_path) + + +class cProfileDefaultPythonProfiler(cProfilePythonProfiler): + """Higher level class on cProfilePythonProfiler to manage the default case where no python profiling config is + specified. Three steps of python profiling are done starting at PYTHON_PROFILING_START_STEP_DEFAULT. For each of + these steps, the cProfile timer function used cycles between total_time, cpu_time and off_cpu_time. + """ + + def __init__(self, base_folder, framework): + super().__init__(base_folder, framework, "default") + + def _enable_profiler(self): + """Set the current cProfile timer based on the step and enable the cProfile profiler. + """ + if self._start_step == PYTHON_PROFILING_START_STEP_DEFAULT: + # first step of default three steps of profiling + self.cprofile_timer = total_time + elif self._start_step == PYTHON_PROFILING_START_STEP_DEFAULT + 1: + # second step of default three steps of profiling + self.cprofile_timer = cpu_time + elif self._start_step == PYTHON_PROFILING_START_STEP_DEFAULT + 2: + # third step of default three steps of profiling + self.cprofile_timer = off_cpu_time + else: + # for pre step zero or post hook close profiling, use total_time + self.cprofile_timer = total_time + super()._enable_profiler() + + +class PyinstrumentPythonProfiler(PythonProfiler): + """Higher level class to oversee profiling specific to Pyinstrument, a third party Python profiler. + """ + + _name = PYINSTRUMENT_NAME + + def _enable_profiler(self): + """Enable the pyinstrument profiler. + """ + self._profiler = PyinstrumentProfiler() + self._profiler.start() + + def _disable_profiler(self): + """Disable the pyinstrument profiler. + """ + self._profiler.stop() + + def _dump_stats(self, stats_dir): + """Dump the stats as a JSON dictionary to a file `python_stats.json` in the provided stats directory. + """ + stats_file_path = os.path.join(stats_dir, PYINSTRUMENT_JSON_FILENAME) + html_file_path = os.path.join(stats_dir, PYINSTRUMENT_HTML_FILENAME) + try: + session = self._profiler.last_session + json_stats = JSONRenderer().render(session) + get_logger().info(f"JSON stats collected for pyinstrument: {json_stats}.") + with open(stats_file_path, "w") as json_data: + json_data.write(json_stats) + get_logger().info(f"Dumping pyinstrument stats to {stats_file_path}.") + + with open(html_file_path, "w") as html_data: + html_data.write(self._profiler.output_html()) + get_logger().info(f"Dumping pyinstrument output html to {html_file_path}.") + except (UnboundLocalError, AssertionError): + # Handles error that sporadically occurs within pyinstrument. + get_logger().info( + f"The pyinstrument profiling session has been corrupted for: {stats_file_path}." + ) + with open(stats_file_path, "w") as json_data: + json.dump({"root_frame": None}, json_data) + + with open(html_file_path, "w") as html_data: + html_data.write("An error occurred during profiling!") diff --git a/smdebug/profiler/system_metrics_reader.py b/smdebug/profiler/system_metrics_reader.py new file mode 100644 index 0000000000..0e9403ba41 --- /dev/null +++ b/smdebug/profiler/system_metrics_reader.py @@ -0,0 +1,172 @@ +# Standard Library + +import bisect +import json +import os + +# First Party +from smdebug.core.access_layer.s3handler import ListRequest, ReadObjectRequest, S3Handler, is_s3 +from smdebug.profiler.metrics_reader_base import MetricsReaderBase +from smdebug.profiler.profiler_constants import ( + DEFAULT_SYSTEM_PROFILER_PREFIX, + ENV_TIME_BUFFER, + TIME_BUFFER_DEFAULT, +) +from smdebug.profiler.system_profiler_file_parser import ProfilerSystemEvents +from smdebug.profiler.utils import get_utctimestamp_us_since_epoch_from_system_profiler_file + + +class SystemMetricsReader(MetricsReaderBase): + def __init__(self, use_in_memory_cache=False): + super().__init__(use_in_memory_cache) + self.prefix = DEFAULT_SYSTEM_PROFILER_PREFIX + self._SystemProfilerEventParser = ProfilerSystemEvents() + self._event_parsers = [self._SystemProfilerEventParser] + + """ + Return the profiler system event files that were written during the given range. If use_buffer is True, we will consider adding a + buffer of TIME_BUFFER_DEFAULT microseconds to increase the time range. This is done because the events are written to the + file after they end. It is possible that an event would have started within the window of start and end, however it + did not complete at or before 'end' time. Hence the event will not appear in the event file that corresponds to + 'end' timestamp. It will appear in the future event file. + We will also add a buffer for the 'start' i.e. we will look for event files that were written prior to 'start'. + Those files might contain 'B' type events that had started prior to 'start' + """ + + def _get_event_files_in_the_range( + self, start_time_microseconds, end_time_microseconds, use_buffer=True + ): + # increase the time range using TIME_BUFFER_DEFAULT + if use_buffer: + time_buffer = os.getenv(ENV_TIME_BUFFER, TIME_BUFFER_DEFAULT) + start_time_microseconds = start_time_microseconds - time_buffer + end_time_microseconds = end_time_microseconds + time_buffer + + """ + We need to intelligently detect whether we need to refresh the list of available event files. + Approach 1: Keep the start prefix for S3, 'x' minutes (say 5) lagging behind the last available timestamp. + This will cover for the case where a node or writer is not efficient enough to upload the files to S3 + immediately. For local mode we may have to walk the directory every time. + This is currently implemented by computing the start prefix and TRAILING DURATION. + TODO: + Approach 2: If we can know the expected number of files per node and per writer, we can intelligently wait + for that type of file for certain amount of time. + """ + + """ + In case of S3, we will refresh the event file list if the requested end timestamp is less than the timestamp + of _startAfterPrefix. + In case of local mode, the event file list will be refreshed if the end timestamp is not less than the last + available timestamp + """ + + if self._startAfter_prefix is not "": + if end_time_microseconds >= get_utctimestamp_us_since_epoch_from_system_profiler_file( + self._startAfter_prefix + ): + self.refresh_event_file_list() + else: + if end_time_microseconds >= self.get_timestamp_of_latest_available_file(): + self.refresh_event_file_list() + + timestamps = sorted(self._timestamp_to_filename.keys()) + + # Find the timestamp that is smaller than or equal start_time_microseconds. The event file corresponding to + # that timestamp will contain events that are active during start_time_microseconds + lower_bound_timestamp_index = bisect.bisect_right(timestamps, start_time_microseconds) + if lower_bound_timestamp_index > 0: + lower_bound_timestamp_index -= 1 + + # Find the timestamp that is immediate right to the end_time_microseconds. The event file corresponding to + # that timestamp will contain events that are active during end_time_microseconds. + upper_bound_timestamp_index = bisect.bisect_left(timestamps, end_time_microseconds) + + event_files = list() + for index in timestamps[lower_bound_timestamp_index : upper_bound_timestamp_index + 1]: + event_files.extend(self._timestamp_to_filename[index]) + return event_files + + def _get_event_parser(self, filename): + return self._SystemProfilerEventParser + + def _get_timestamp_from_filename(self, event_file): + return get_utctimestamp_us_since_epoch_from_system_profiler_file(event_file) + + def _get_event_file_regex(self): + return r"(.+)\.json" + + +class LocalSystemMetricsReader(SystemMetricsReader): + """ + The metrics reader is created with root folder in which the system event files are stored. + """ + + def __init__(self, trace_root_folder, use_in_memory_cache=False): + self.trace_root_folder = trace_root_folder + super().__init__(use_in_memory_cache) + # Pre-build the file list so that user can query get_timestamp_of_latest_available_file() + # and get_current_time_range_for_event_query + self.refresh_event_file_list() + + """ + Create a map of timestamp to filename + """ + + def refresh_event_file_list(self): + self._refresh_event_file_list_local_mode(self.trace_root_folder) + + def parse_event_files(self, event_files): + self._parse_event_files_local_mode(event_files) + + +class S3SystemMetricsReader(SystemMetricsReader): + """ + The s3_trial_path points to a s3 folder in which the system metric event files are stored. e.g. + s3://my_bucket/experiment_base_folder + """ + + def __init__(self, s3_trial_path, use_in_memory_cache=False): + super().__init__(use_in_memory_cache) + s3, bucket_name, base_folder = is_s3(s3_trial_path) + if not s3: + self.logger.error( + "The trial path is expected to be S3 path e.g. s3://bucket_name/trial_folder" + ) + else: + self.bucket_name = bucket_name + self.base_folder = base_folder + self.prefix = os.path.join(self.base_folder, self.prefix, "") + # Pre-build the file list so that user can query get_timestamp_of_latest_available_file() + # and get_current_time_range_for_event_query + self.refresh_event_file_list() + + def parse_event_files(self, event_files): + file_read_requests = [] + event_files_to_read = [] + + for event_file in event_files: + if event_file not in self._parsed_files: + event_files_to_read.append(event_file) + file_read_requests.append(ReadObjectRequest(path=event_file)) + + event_data_list = S3Handler.get_objects(file_read_requests) + for event_data, event_file in zip(event_data_list, event_files_to_read): + event_string = event_data.decode("utf-8") + event_items = event_string.split("\n") + event_items.remove("") + for item in event_items: + event = json.loads(item) + self._SystemProfilerEventParser.read_event_from_dict(event) + self._parsed_files.add(event_file) + + """ + Create a map of timestamp to filename + """ + + def refresh_event_file_list(self): + list_dir = ListRequest( + Bucket=self.bucket_name, + Prefix=self.prefix, + StartAfter=self._startAfter_prefix if self._startAfter_prefix else self.prefix, + ) + self._refresh_event_file_list_s3_mode(list_dir) diff --git a/smdebug/profiler/system_profiler_file_parser.py b/smdebug/profiler/system_profiler_file_parser.py new file mode 100644 index 0000000000..29eff17f37 --- /dev/null +++ b/smdebug/profiler/system_profiler_file_parser.py @@ -0,0 +1,111 @@ +# First Party +# Standard Library +import json + +from smdebug.core.logger import get_logger +from smdebug.profiler.utils import TimeUnits, convert_utc_timestamp_to_nanoseconds + + +class ProfilerSystemEvent: + def __init__(self, event_type, name, dimension, value, node_id, timestamp): + """ + type must be one of the value in ["cpu", "gpu"] + """ + self.type = event_type + """ + name specifies the name of cpu/gpu core, it is optional as memory event doesn't have a type + """ + self.name = name + """ + dimension is a part of the event that is associated with the specified type, example: CPUUtilization + """ + self.dimension = dimension + self.value = value + self.node_id = node_id + """ + timestamp in seconds, example: 1591160699.4570894 + """ + self.timestamp = timestamp + + +class SystemProfilerEventParser: + def __init__(self): + self._events = [] + self.logger = get_logger() + + def _read_event(self, event): + name = None + if "Name" in event: + name = event["Name"] + parsed_event = ProfilerSystemEvent( + event["Type"], + name, + event["Dimension"], + event["Value"], + event["NodeId"], + event["Timestamp"], + ) + self._events.append(parsed_event) + + def read_events_from_file(self, eventfile): + try: + with open(eventfile) as json_data: + event_line = json_data.readline() + while event_line: + json_event = json.loads(event_line) + event_line = json_data.readline() + self.read_event_from_dict(json_event) + except Exception as e: + self.logger.error( + f"Can't open profiler system metric file {eventfile}: Exception {str(e)}" + ) + raise ValueError( + f"Can't open profiler system metric file {eventfile}: Exception {str(e)}" + ) + + def read_events_from_json_data(self, system_profiler_json_data): + for event in system_profiler_json_data: + self._read_event(event) + + def read_event_from_dict(self, event): + self._read_event(event) + + def get_events_within_time_range( + self, start_time, end_time, unit=TimeUnits.MICROSECONDS, event_type=None + ): + start_time_nanos = convert_utc_timestamp_to_nanoseconds(start_time, unit) + end_time_nanos = convert_utc_timestamp_to_nanoseconds(end_time, unit) + result_events = list( + filter( + lambda event: self._valid_event( + event, start_time_nanos, end_time_nanos, event_type + ), + self._events, + ) + ) + return result_events + + @staticmethod + def _valid_event(event, start_time_nanos, end_time_nanos, event_type): + timestamp_in_nanos = convert_utc_timestamp_to_nanoseconds( + event.timestamp, TimeUnits.SECONDS + ) + if event_type is not None: + return ( + event + and event.type == event_type + and start_time_nanos <= timestamp_in_nanos <= end_time_nanos + ) + else: + return event and start_time_nanos <= timestamp_in_nanos <= end_time_nanos + + def get_all_events(self): + return self._events + + def clear_events(self): + self._events = [] + + +class ProfilerSystemEvents(SystemProfilerEventParser): + def __init__(self): + super().__init__() diff --git a/smdebug/profiler/tf_profiler_parser.py b/smdebug/profiler/tf_profiler_parser.py index 46f24acb45..3904df8f7a 100644 --- a/smdebug/profiler/tf_profiler_parser.py +++ b/smdebug/profiler/tf_profiler_parser.py @@ -1,78 +1,202 @@ # Standard Library +import glob +import gzip import json +import os +import zlib +from datetime import datetime # First Party -from smdebug.profiler.trace_event_file_parser import ProcessInfo, TraceEventParser +from smdebug.core.access_layer.s3handler import ReadObjectRequest, S3Handler +from smdebug.core.access_layer.utils import is_s3 +from smdebug.core.logger import get_logger +from smdebug.profiler.profiler_constants import ( + CONVERT_TO_MICROSECS, + TENSORBOARDTIMELINE_SUFFIX, + TF_METRICS_PREFIX, +) +from smdebug.profiler.trace_event_file_parser import TraceEventParser +from smdebug.profiler.utils import read_tf_profiler_metadata_file -class SMTFProfilerEvents(TraceEventParser): - def __init__(self, trace_file): - self._trace_json_file = trace_file - super().__init__() - self.read_trace_file() - - def _populate_start_time(self, event): - event_args = event["args"] if "args" in event else None - if self._start_time_known is False: - if event_args is None: - return - if "start_time_since_epoch_in_micros" in event_args: - self._start_timestamp = event_args["start_time_since_epoch_in_micros"] - self._start_time_known = True - self.logger.info(f"Start time for events in uSeconds = {self._start_timestamp}") - - # TODO implementation of below would be changed to support streaming file and incomplete json file - def read_trace_file(self): - try: - with open(self._trace_json_file) as json_data: - trace_json_data = json.load(json_data) - except Exception as e: - self.logger.error( - f"Can't open TF trace file {self._trace_json_file}: Exception {str(e)}" - ) - return +class SMProfilerEvents(TraceEventParser): + def __init__(self, type="PythontimelineMetrics"): + super().__init__(type=type) - for event in trace_json_data: - self._read_event(event) +class TensorboardProfilerEvents(TraceEventParser): + def __init__(self): + super().__init__(type="TFProfilerMetrics") -class TFProfilerEvents(TraceEventParser): - def __init__(self, trace_file): - self._trace_json_file = trace_file - super().__init__() - self.read_trace_file() - - def _populate_thread_info_for_metaevent(self, event): - if event["name"] == "thread_name": - name = event["args"]["name"] - t_id = event["tid"] - pid = event["pid"] - if pid not in self._processes: - self.logger.warn( - f"Did not find matching process for pid {pid}. Creating a process with name 'Unknown'" - ) - self._processes[pid] = ProcessInfo(pid, "Unknown") - self._processes[pid].add_thread(t_id, name) - - def _populate_start_time(self, event): - # TODO, not sure if we can implement this right now - return - - def read_trace_file(self): + def _get_trace_events_json(self, tracefile): try: - with open(self._trace_json_file) as json_data: - trace_json_data = json.load(json_data) + s3, bucket_name, key_name = is_s3(tracefile) + if s3: + object_requests = ReadObjectRequest(os.path.join("s3://", bucket_name, key_name)) + objects = S3Handler.get_objects([object_requests]) + unzipped = zlib.decompress(objects[0], zlib.MAX_WBITS | 16) + trace_json_data = json.loads(unzipped.decode("utf-8")) + else: + with gzip.GzipFile(tracefile, "r") as fin: + trace_json_data = json.loads(fin.read().decode("utf-8")) except Exception as e: - self.logger.error( - f"Can't open TF trace file {self._trace_json_file}: Exception {str(e)} " - ) - return + self.logger.error(f"Can't open TF trace file {tracefile}: Exception {str(e)} ") + return None if "traceEvents" not in trace_json_data: - self.logger.error( - f"The TF trace file {self._trace_json_file} does not contain traceEvents" - ) - return + self.logger.error(f"The TF trace file {tracefile} does not contain traceEvents") + return None trace_events_json = trace_json_data["traceEvents"] + _, start_time_in_micros, _ = read_tf_profiler_metadata_file(tracefile) + # the first time profiler.start() is called is considered the start time + # for TF profiler + metadata = [] + args = {"start_time_since_epoch_in_micros": int(start_time_in_micros)} + json_dict = {"name": "process_name", "ph": "M", "pid": 0, "args": args} + metadata.append(json_dict) + args = {"sort_index": 0} + json_dict = {"name": "process_sort_index", "ph": "M", "pid": 0, "args": args} + metadata.append(json_dict) + + # insert metadata at the beginning of trace events json + trace_events_json = metadata + trace_events_json + return trace_events_json + + def read_events_from_file(self, tracefile): + trace_events_json = self._get_trace_events_json(tracefile) + + node_id = "default" + file_name_splits = tracefile.split("/") + if len(file_name_splits) > 0: + file = file_name_splits[-1] + file_split = file.split(".") + if len(file_split) > 0: + node_id = file_split[0] + print(f"Reading event from tracefile:{tracefile} and node_id:{node_id}") + if trace_events_json: + for event in trace_events_json: + self._read_event(event, node_id) + + def get_events_within_range(self, start_time: datetime, end_time: datetime): + return None + + def _get_event_phase(self, event): + if not event.event_name or not event.event_name.startswith(TF_METRICS_PREFIX): + return + + # Phase is between aws-marker and first slash. + phase = event.event_name.split("/")[0][len(TF_METRICS_PREFIX) :] + + if phase in ["ForwardPass", "ComputeGradient", "ApplyGradient"]: + return phase + + def get_complete_op_events(self, tracefile): + op_events = [] + self.read_events_from_file(tracefile) + all_events = self.get_all_events() + for event in all_events: + if event.event_args is not None: + phase = self._get_event_phase(event) + if phase: + op_events.append((event, phase)) + return op_events + + def get_training_info(self, tracefile): + all_op_events = self.get_complete_op_events(tracefile) + # each in the list , will be list [name, ts, duration] + training_annotation = { + "ForwardPass": [], + # "BackwardPass": [], + "ComputeGradient": [], + "ApplyGradient": [], + } + + for event, phase in all_op_events: + training_annotation[phase].append([event.event_name, event.start_time, event.duration]) + return training_annotation + + # TODO: The following function has to be revisited when + # AWS-TF forward/backward annotations are finalized. + def _dump_info_to_json(self, training_info, trace_json_file): + """ + This function dumps the training info gathered into the + json file passed. + """ + with open(trace_json_file, "r+") as f: + data = json.load(f) + f.close() + + for phase, metrics in training_info.items(): + if not metrics: + get_logger().error(f"No metrics captured after profiling for {phase}!") + continue + + # Getting the min start_time to get the start_time + start = min(x[1] for x in metrics) + # Calculating the max end time using duration. + end = max(x[1] + x[2] for x in metrics) + phase = "BackwardPass" if phase != "ForwardPass" else phase + main_entry = { + "pid": "/" + phase, + "tid": phase, + "ph": "X", + "ts": start / 1000, + "dur": (end - start) / 1000, + "name": phase, + "args": {"group_id": phase, "long_name": phase}, + } + data["traceEvents"].append(main_entry) + + for idx, metrics in enumerate(metrics): + entry = { + "pid": "/" + phase, + "tid": phase + "ops", + "ph": "X", + "args": {"group_id": phase, "long_name": metrics[0]}, + "ts": metrics[1] / 1000, + "dur": metrics[2] / 1000, + "name": metrics[0], + } + data["traceEvents"].append(entry) + + get_logger().info(f"Dumping into file {trace_json_file}") + with open(trace_json_file, "w+") as outfile: + json.dump(data, outfile) + + # TODO: The following function has to be revisited when + # AWS-TF forward/backward annotations are finalized. + def parse_tf_native_profiler_trace_json(self, log_dir): + """ + Returns: Function returns a dictonary of + {"ForwardPass": [], "ComputeGradient": [], "ApplyGradient": []} + The value is list of list. Each list is [opname, ts, duration] + + """ + tf_profiler_folders = os.listdir(os.path.join(log_dir + "/plugins/profile")) + trace_json_files = [] + for folder in tf_profiler_folders: + folderpath = os.path.join(log_dir + "/plugins/profile", folder) + for file in os.listdir(folderpath): + if file.endswith(".gz"): + trace_json_files.append(os.path.join(folderpath, file)) + + # get the latest file. TF annotations will be appended to this file + trace_json_file = max(glob.glob(trace_json_files), key=os.path.getmtime) + training_info = self.get_training_info(trace_json_file) + + # Dump gathered data into trace_json_file + self._dump_info_to_json(training_info, trace_json_file) + + return training_info, trace_json_file + + +class HorovodProfilerEvents(TraceEventParser): + def __init__(self): + super().__init__() + + def type(self): + return "HorovodMetrics" + - for event in trace_events_json: - self._read_event(event) +class SMDataParallelProfilerEvents(TraceEventParser): + def __init__(self): + super().__init__(type="SMDataParallelMetrics") diff --git a/smdebug/profiler/trace_event_file_parser.py b/smdebug/profiler/trace_event_file_parser.py index 70e503444f..5068c21402 100644 --- a/smdebug/profiler/trace_event_file_parser.py +++ b/smdebug/profiler/trace_event_file_parser.py @@ -1,5 +1,19 @@ # First Party +# Standard Library +import json +import re +from builtins import Exception, dict, hash, sorted, str +from datetime import datetime + from smdebug.core.logger import get_logger +from smdebug.profiler.utils import ( + TimeUnits, + convert_utc_datetime_to_microseconds, + convert_utc_datetime_to_nanoseconds, + convert_utc_timestamp_to_microseconds, + convert_utc_timestamp_to_nanoseconds, + get_node_id_from_tracefilename, +) class ThreadInfo: @@ -7,6 +21,16 @@ def __init__(self, tid, thread_name): self.tid = tid self.thread_name = thread_name + def __repr__(self): + return f"tid:{self.tid} name:{self.thread_name}" + + +""" +This contains infomation about all the phases and list of +threads found for all these phase. +For mutiple worker scenarios, there will be a phase and different workers will be treated as thread for the phase +""" + class ProcessInfo: def __init__(self, id, name): @@ -15,32 +39,96 @@ def __init__(self, id, name): self._threads = dict() def add_thread(self, threadid, thread_name): - self._threads[threadid] = ThreadInfo(threadid, thread_name) + if threadid not in self._threads: + self._threads[threadid] = ThreadInfo(threadid, thread_name) def get_thread_info(self, threadid): return self._threads[threadid] + def __repr__(self): + return f"id:{self.id} name:{self.name} threads:{self._threads}" + class TraceEvent: - def __init__(self, ts, name, dur, pid, tid, event_args): + def __init__( + self, + ts, + name, + dur, + phase_pid, + phase_tid, + event_args, + node_id, + phase, + file_type, + event_phase="", + pid=0, + tid=0, + process_info=None, + ): self.start_time = ts self.event_name = name self.duration = dur self.end_time = self.start_time + self.duration + # this pid points to a unique phase name, the left most column of timeline file + self.phase_pid = phase_pid + # different threads and workers will be under phase, each worker will have its unique phase_tid + self.phase_tid = phase_tid + # Actual process id self.pid = pid + # Actual thread id tid self.tid = tid self.event_args = event_args + self.node_id = node_id + self.phase = phase + # the phase name this event belongs to, this can also come from self._processes[phase_pid].name + self.event_phase = event_phase + self.process_info = process_info + self.file_type = file_type + + def to_json(self): + json_dict = { + "name": self.event_name, + "pid": self.pid, + "tid": self.tid, + "ph": self.phase, + "ts": self.start_time, + } + + # handle Instant event + if self.phase == "i": + if self.event_args: + # Instant events have a field unique to them called scope. + # scope can be "g" - global, "p" - process, "t" - thread. + # parsing this value that is being passed as args. + s = self.event_args["s"] if "s" in self.event_args else "t" + json_dict.update({"s": s}) + if "s" in self.event_args: + self.event_args.pop("s") + elif self.phase == "X": + json_dict.update({"dur": self.duration}) + + if self.event_args: + json_dict["args"] = self.event_args + + return json.dumps(json_dict) class TraceEventParser: - def __init__(self): + def __init__(self, type=""): + # list of ProcessInfo found in this file self._processes = dict() - self._trace_events = list() + # reverse mapping from name to id + self._process_name_to_id = dict() + self._trace_events = [] + """ + The _pid_stacks maintain the directory of stacks indexed using pid. The stack contains 'B' type events. + The stack will be popped as we process the 'E' events for the same pid. + """ + self._pid_stacks = dict() self._start_timestamp = 0 - self._start_time_known = False - # The timestamp in trace events are in micro seconds, we multiply by 1000 to convert to ns - self._timescale_multiplier_for_ns = 1000 - self.logger = get_logger("smdebug-profiler") + self.type = type + self.logger = get_logger() def read_trace_file(self): pass @@ -48,36 +136,200 @@ def read_trace_file(self): def _populate_process_info_for_metaevent(self, event): id = event["pid"] if event["name"] == "process_name": - name = event["args"]["name"] if "name" in event["args"] else "Unknown" - self._processes[id] = ProcessInfo(id, name) + process_name = event["args"]["name"] if "name" in event["args"] else "Unknown" + # we will check if this name has already been seen, possibly for previous node + # if not seen, we will create an entry from id to name and reverse entry from name to id + # otherwise, we will point the id to existing id. + if process_name not in self._process_name_to_id: + self._processes[id] = ProcessInfo(id, process_name) + self._process_name_to_id[process_name] = id + else: + # this looks like multinode scenario, where same phase is coming from different node + # we will tie this id to existing phase, any thread for this id would be related to this phase + existing_id = self._process_name_to_id[process_name] + self._processes[id] = ProcessInfo(existing_id, process_name) - def _populate_thread_info_for_metaevent(self, event): - pass + def _populate_thread_info_for_metaevent( + self, event, node_id="", phase_tid_default=None, phase_name="" + ): + # The `E` event in horovod and SMDataParallel does not have a `name` entity. Add an empty `name` in an event dictionary if it is absent. + if self.type in ["SMDataParallelMetrics", "HorovodMetrics"]: + if "name" not in event and "ph" in event and event["ph"] == "E": + event["name"] = "" + if event["name"] == "thread_name": + name = node_id + "_" + event["args"]["name"] + t_id = event["tid"] + elif event["name"] == "thread_sort_index": + name = "Unknown" + t_id = event["tid"] + elif phase_tid_default is not None: + # there is no thread mentioned and this is unique thread for phase and node + # We will be generating a unique tid here and return this tid to be populated in event + name = node_id + "_" + str(phase_tid_default) + t_id = hash(name + str(phase_name)) + else: + self.logger.debug( + f"Event:{event} doesn't have thread_name nor phase_tid_default. Returning" + ) + return + pid = event["pid"] + if pid not in self._processes: + self.logger.warn( + f"Did not find matching process for pid {pid}. Creating a process with name 'Unknown'" + ) + self._processes[pid] = ProcessInfo(pid, "Unknown") + self._processes[pid].add_thread(t_id, name) + return t_id def _populate_start_time(self, event): - pass + event_args = event["args"] if "args" in event else None + if event_args is None: + return + # the start time since micros is set for each trace event parser type. + # in cases such as multiprocessing, files of the same type may have + # different start times. updating start time here if it is different from + # previous file that was read. + if "start_time_since_epoch_in_micros" in event_args: + start_time_in_us = event_args["start_time_since_epoch_in_micros"] + if self._start_timestamp == start_time_in_us: + return + self._start_timestamp = start_time_in_us + self.logger.info(f"Start time for events in uSeconds = {self._start_timestamp}") - def _read_event(self, event): + def _read_event(self, event, node_id=""): if "ph" not in event: - self.logger.error(f"In correctly formatted trace file. The 'ph' field is not present") return + # if node-id in for of pid-algo-i , interchange to algo-i-pid so that we can have good sorted order + # if we view this in timeline + found_nodeids_parts = re.match("(.*)-(algo.*)", node_id) if node_id is not None else None + if found_nodeids_parts is not None and len(found_nodeids_parts.groups()) == 2: + node_id = found_nodeids_parts[2] + "-" + found_nodeids_parts[1] + phase_type = event["ph"] if phase_type == "M": self._populate_process_info_for_metaevent(event) - self._populate_thread_info_for_metaevent(event) + self._populate_thread_info_for_metaevent(event, node_id) self._populate_start_time(event) - if phase_type == "X": # In nano seconds - start_time = (event["ts"] + self._start_timestamp) * self._timescale_multiplier_for_ns + start_time = event["ts"] + self._start_timestamp # In nano seconds - dur = event["dur"] * self._timescale_multiplier_for_ns + dur = event["dur"] name = event["name"] - id = event["pid"] - tid = event["tid"] if "tid" in event else "0" + pid = phase_pid = event["pid"] # this is phase pid + phase_tid = event["tid"] if "tid" in event else 0 + + phase_name = "Unknown" + if phase_pid in self._processes: + phase_name = self._processes[phase_pid].name + + if ( + "tid" in event + and phase_pid in self._processes + and event["tid"] in self._processes[phase_pid]._threads + ): + phase_tid = event["tid"] + else: + # we will generate unique tid which is hash of 0 + node_id + phase_tid = self._populate_thread_info_for_metaevent( + event, node_id=node_id, phase_tid_default=phase_tid, phase_name=phase_name + ) + event_args = event["args"] if "args" in event else None - t_event = TraceEvent(start_time, name, dur, id, tid, event_args) + tid = phase_tid + if event_args: + # Tf Detailed metrics emits pid and thread_id + if "pid" in event_args: + pid = event_args["pid"] + if "thread_id" in event_args: + tid = event_args["thread_id"] + + t_event = TraceEvent( + start_time, + name, + dur, + phase_pid, + phase_tid, + event_args, + node_id, + phase_type, + file_type=self.type, + event_phase=phase_name, + pid=pid, + tid=tid, + process_info=self._processes[phase_pid], + ) self._trace_events.append(t_event) + # TODO ignoring B and E events for now. Need to check to handle it + if phase_type == "B": + pid = event["pid"] + if pid not in self._pid_stacks: + self._pid_stacks[pid] = [] + self._pid_stacks[pid].append(event) + if phase_type == "E": + pid = phase_pid = event["pid"] + if pid not in self._pid_stacks: + self.logger.info( + f"Did not find the 'B' type event in the pid {pid} . Skipping event: {event}" + ) + else: + b_event = self._pid_stacks[pid][-1] + self._pid_stacks[pid].pop() + start_time = b_event["ts"] + self._start_timestamp + end_time = event["ts"] + self._start_timestamp + duration = end_time - start_time + if duration < 0: + self.logger.error( + f"Error in reading the events 'B' and 'E' or trace file is corrupt: pid = " + f"{pid}, start_time = {b_event['ts']} end_time = {event['ts']} name = " + f"{b_event['name']}" + ) + return + + name = b_event["name"] + event_args = event["args"] if "args" in event else None + phase_tid = tid = b_event["tid"] if "tid" in event else "0" + phase_name = "Unknown" + if phase_pid in self._processes: + phase_name = self._processes[phase_pid].name + + if ( + "tid" in event + and phase_pid in self._processes + and tid in self._processes[phase_pid]._threads + ): + phase_tid = self._processes[phase_pid]._threads[tid] + else: + # we will generate unique tid which is hash of 0 + node_id + phase_tid = self._populate_thread_info_for_metaevent( + event, node_id=node_id, phase_tid_default=phase_tid, phase_name=phase_name + ) + tid = phase_tid + if event_args: + # Tf Detailed metrics emits pid and thread_id + # get actual pid of process + # get actual thread id of processes. depending on file type actual pid and tid may be into args + if "pid" in event_args: + pid = event_args["pid"] + if "thread_id" in event_args: + tid = event_args["thread_id"] + + t_event = TraceEvent( + start_time, + name, + duration, + phase_pid, + phase_tid, + event_args, + node_id, + "X", + file_type=self.type, + event_phase=phase_name, + pid=pid, + tid=tid, + process_info=self._processes[phase_pid], + ) + self._trace_events.append(t_event) def get_all_events(self): return self._trace_events @@ -89,28 +341,52 @@ def get_events_end_time_sorted(self): return sorted(self._trace_events, key=lambda x: x.end_time) """ - Return the events that are in progress at the specified timestamp. - Performance of this function can be improved by implementing interval tree. + Return the events that + 1. are active or running at the start or end timestamps + 2. started and completed within the given range. + In other words, the function will not return the events that have completed before the 'start' timestamp and + started after the 'end' timestamp. + The start and end time, by default, are assumed to be in microseconds. + For tracefiles generated by smdebug, the start and end timestamps need to be seconds elapsed + from epoch ( January 1, 1970 12:00:00 AM) + For horovod and tensorboard generated tracefiles, the start and end timestamps will be interpreted as + seconds elapsed from the first recorded event. """ - def get_events_at(self, timestamp_in_nanoseconds): + def get_events_within_time_range(self, start_time, end_time, unit=TimeUnits.MICROSECONDS): + start_time_microseconds = convert_utc_timestamp_to_microseconds(start_time, unit) + end_time_microseconds = convert_utc_timestamp_to_microseconds(end_time, unit) result_events = list() for x_event in self._trace_events: - if x_event.start_time <= timestamp_in_nanoseconds <= x_event.end_time: - result_events.append(x_event) + # event finished before start time or event started after the end time + if ( + x_event.end_time < start_time_microseconds + or end_time_microseconds < x_event.start_time + ): + continue + result_events.append(x_event) + return result_events """ - Return the events that have started and completed within the given start and end time boundaries. - The events that are in progress during these boundaries are not included. + The TraceEvent class can not support retrieving events based on given datetime objects. + This is because only smdebug based tracefile store the timestamps based on unix epoch timestamp. """ - def get_events_within_range(self, start_time, end_time): - result_events = list() - for x_event in self._trace_events: - if start_time <= x_event.start_time and end_time >= x_event.end_time: - result_events.append(x_event) - return result_events + def get_events_within_range(self, start_time: datetime, end_time: datetime): + """ + Return the events that have started and completed within the given start and end time boundaries. + The start and end time can be specified datetime objects. + The events that are in progress during these boundaries are not included. + """ + start_time_microseconds = end_time_microseconds = 0 + if start_time.__class__ is datetime: + start_time_microseconds = convert_utc_datetime_to_microseconds(start_time) + if end_time.__class__ is datetime: + end_time_microseconds = convert_utc_datetime_to_microseconds(end_time) + return self.get_events_within_time_range( + start_time_microseconds, end_time_microseconds, unit=TimeUnits.MICROSECONDS + ) def get_process_info(self, process_id): return self._processes[process_id] @@ -118,6 +394,21 @@ def get_process_info(self, process_id): def get_processes(self): return self._processes + # TODO implementation of below would be changed to support streaming file and incomplete json file + def read_events_from_file(self, tracefile): + try: + with open(tracefile) as json_data: + trace_json_data = json.load(json_data) + except Exception as e: + self.logger.error(f"Can't open trace file {tracefile}: Exception {str(e)}") + return + node_id = get_node_id_from_tracefilename(tracefile) + self.read_events_from_json_data(trace_json_data, node_id) + + def read_events_from_json_data(self, trace_json_data, node_id): + for event in trace_json_data: + self._read_event(event, node_id) + # TODO def get_events_for_process(self, pid, start_time, end_time): pass @@ -125,3 +416,6 @@ def get_events_for_process(self, pid, start_time, end_time): # TODO def get_events_for_thread(self, tid, start_time, end_time): pass + + def clear_events(self): + self._trace_events = [] diff --git a/smdebug/profiler/utils.py b/smdebug/profiler/utils.py new file mode 100644 index 0000000000..436690f1b9 --- /dev/null +++ b/smdebug/profiler/utils.py @@ -0,0 +1,302 @@ +""" +The TimeUnit enum is to be used while querying the events within timerange or at a given timestamp +The Enum will indicate the unit in which timestamp is provided. +""" +# Standard Library +import os +import re +import shutil +import time +from datetime import datetime +from distutils.util import strtobool +from enum import Enum +from pathlib import Path + +# Third Party +from botocore.exceptions import ClientError + +# First Party +from smdebug.core.access_layer.file import SMDEBUG_TEMP_PATH_SUFFIX, TSAccessFile +from smdebug.core.access_layer.s3 import TSAccessS3 +from smdebug.core.access_layer.s3handler import ListRequest, S3Handler, is_s3 +from smdebug.core.logger import get_logger +from smdebug.core.utils import ensure_dir, get_node_id +from smdebug.profiler.profiler_constants import ( + CONVERT_TO_MICROSECS, + HOROVODTIMELINE_SUFFIX, + SMDATAPARALLELTIMELINE_SUFFIX, +) + +logger = get_logger() + + +class TimeUnits(Enum): + SECONDS = 1 + MILLISECONDS = 2 + MICROSECONDS = 3 + NANOSECONDS = 4 + + +""" +The function assumes that the correct enum is provided. +""" + + +def convert_utc_timestamp_to_nanoseconds(timestamp, unit=TimeUnits.MICROSECONDS): + if unit == TimeUnits.SECONDS: + return int(timestamp * 1000 * 1000 * 1000) + if unit == TimeUnits.MILLISECONDS: + return int(timestamp * 1000 * 1000) + if unit == TimeUnits.MICROSECONDS: + return int(timestamp * 1000) + if unit == TimeUnits.NANOSECONDS: + return timestamp + + +def convert_utc_timestamp_to_microseconds(timestamp, unit=TimeUnits.MICROSECONDS): + if unit == TimeUnits.SECONDS: + return int(timestamp * 1000 * 1000) + if unit == TimeUnits.MILLISECONDS: + return int(timestamp * 1000) + if unit == TimeUnits.MICROSECONDS: + return int(timestamp) + if unit == TimeUnits.NANOSECONDS: + return timestamp / 1000 + + +def convert_utc_timestamp_to_seconds(timestamp, unit=TimeUnits.MICROSECONDS): + if unit == TimeUnits.SECONDS: + return int(timestamp) + if unit == TimeUnits.MILLISECONDS: + return int(timestamp / 1000) + if unit == TimeUnits.MICROSECONDS: + return int(timestamp / 1000 / 1000) + if unit == TimeUnits.NANOSECONDS: + return timestamp / 1000 / 1000 / 1000 + + +""" +The function assumes that the object of datetime is provided. +""" + + +def convert_utc_datetime_to_nanoseconds(timestamp_datetime: datetime): + return convert_utc_timestamp_to_nanoseconds( + timestamp_datetime.timestamp(), unit=TimeUnits.SECONDS + ) + + +def convert_utc_datetime_to_microseconds(timestamp_datetime: datetime): + return convert_utc_timestamp_to_microseconds( + timestamp_datetime.timestamp(), unit=TimeUnits.SECONDS + ) + + +def is_valid_tfprof_tracefilename(filename: str) -> bool: + """ + Ensure that the tracefilename has a valid format. + $ENV_BASE_FOLDER/framework/tensorflow/detailed_profiling/$START_TIME_YYYYMMDDHR/$STEP_NUM/plugins/profile/$HOSTNAME.trace.json.gz + + The filename should have extension trace.json.gz + + """ + return filename.endswith("trace.json.gz") and "tensorflow/detailed_profiling" in filename + + +def is_valid_tracefilename(filename: str) -> bool: + """ + Ensure that the tracefilename has a valid format. + $ENV_BASE_FOLDER/framework/pevents/$START_TIME_YYYYMMDDHR/$FILEEVENTENDTIMEUTCINEPOCH_{$ENV_NODE_ID}_model_timeline.json + + The filename should have extension .json + The filename should have minimum 3 fields viz. $FILEEVENTENDTIMEUTCINEPOCH, {$ENV_NODE_ID} and filetype. + + """ + if filename.endswith(".json") and "pevents" in filename: + if len(filename.split("_")) >= 3: + return True + logger.debug(f"The file {filename} is not a valid tracefile.") + return False + + +def get_node_id_from_tracefilename(filename: str) -> str: + """ + The tracefile has a file name format: + $ENV_BASE_FOLDER/framework/pevents/$START_TIME_YYYYMMDDHR/$FILEEVENTENDTIMEUTCINEPOCH_{$ENV_NODE_ID}_model_timeline.json + + The function extracts and returns the {$ENV_NODE_ID} from file. + """ + if is_valid_tracefilename(filename): + filename = filename.split("/")[-1] + return filename.split("_")[1] + else: + node_id, _, _ = read_tf_profiler_metadata_file(filename) + return node_id + + +def get_node_id_from_system_profiler_filename(filename: str) -> str: + """ + The system metric has a file name format: + /profiler-output/system/incremental/{$TIMESTAMP}.${NODE_ID}.json + Example: /profiler-output/system/incremental/2020060500/1591160699.algo-1.json + + The function extracts and returns the {$NODE_ID} from file. + """ + if validate_system_profiler_file(filename): + filename = filename.split("/")[-1] + return filename.split(".")[1] + return None + + +def get_timestamp_from_tracefilename(filename) -> int: + """ + The tracefile has a file name format: + $ENV_BASE_FOLDER/framework/pevents/$START_TIME_YYYYMMDDHR/$FILEEVENTENDTIMEUTCINEPOCH_{$ENV_NODE_ID}_model_timeline.json + + The function extracts and returns the $FILEEVENTENDTIMEUTCINEPOCH from file. The $FILEEVENTENDTIMEUTCINEPOCH + represents the timestamp of last event written to the tracefile. + The timestamps are used to determine whether an event is available in this this file. If the file name is not + valid, we will written timestamp as 0. + """ + if is_valid_tracefilename(filename): + filename = filename.split("/")[-1] + return int(filename.split("_")[0]) + else: + _, _, timestamp = read_tf_profiler_metadata_file(filename) + return int(timestamp) + + +def get_utctimestamp_us_since_epoch_from_system_profiler_file(filename) -> int: + """ + The system metric file has a file name format: + /profiler-output/system/incremental/..json + Example: /profiler-output/system/incremental/2020060500/1591160699.algo-1.json + + The function extracts and returns the in microseconds from filename. + """ + if validate_system_profiler_file(filename): + filename = filename.split("/")[-1] + return int(filename.split(".")[0]) * 1000 * 1000 + return None + + +def validate_system_profiler_file(filename) -> bool: + filename_regex = re.compile(".+/system/.+/(\d{10}).algo-\d+.json") + stamp = re.match(filename_regex, filename) + if stamp is None: + logger.debug(f"Invalid System Profiler File Found: {filename}, not able to get timestamp.") + return False + return True + + +def str2bool(v): + if isinstance(v, bool): + return v + else: + return bool(strtobool(v)) + + +def us_since_epoch_to_human_readable_time(us_since_epoch): + dt = datetime.utcfromtimestamp(us_since_epoch / 1e6) + return dt.strftime("%Y-%m-%dT%H:%M:%S:%f") + + +def ns_since_epoch_to_human_readable_time(ns_since_epoch): + dt = datetime.utcfromtimestamp(ns_since_epoch / 1e9) + return dt.strftime("%Y-%m-%dT%H:%M:%S:%f") + + +def write_tf_profiler_metadata_file(file_path): + if not file_path.endswith(".metadata"): + return + s3, bucket_name, key_name = is_s3(file_path) + if s3: + writer = TSAccessS3(bucket_name, key_name, binary=False) + else: + writer = TSAccessFile(file_path, "a+") + writer.flush() + try: + writer.close() + except OSError: + """ + In the case of distributed training in local mode, + another worker may have already moved the END_OF_JOB file + from the /tmp directory. + """ + + +def read_tf_profiler_metadata_file(file_path): + if not is_valid_tfprof_tracefilename(file_path): + return "", "0", "0" + s3, bucket_name, key_name = is_s3(file_path) + if s3: + try: + folder_name = "/".join(key_name.split("/")[:-4]) + request = ListRequest(bucket_name, folder_name) + file_available = S3Handler.list_prefixes([request]) + if len(file_available) > 0: + metadata_filename = list(filter(lambda x: ".metadata" in x, file_available[0])) + if len(metadata_filename) > 0: + metadata_filename = metadata_filename[0] + metadata_filename = metadata_filename.split("/")[-1] + node_id, start, end = str(metadata_filename).split("_") + return node_id, start, end.split(".")[0] + else: + return "", "0", "0" + else: + return "", "0", "0" + except ClientError as ex: + status_code = ex.response["ResponseMetadata"]["HTTPStatusCode"] + logger.info(f"Client error occurred : {ex}") + if status_code.startswith("4"): + raise ex + else: + return "", "0", "0" + else: + folder_name = "/".join(file_path.split("/")[:-4]) + metadata_filename = list(Path(folder_name).rglob("*.metadata")) + if len(metadata_filename) > 0: + metadata_filename = metadata_filename[0].name + node_id, start, end = str(metadata_filename).split("_") + return node_id, start, end.split(".")[0] + else: + return "", "0", "0" + + +def stop_tf_profiler(tf_profiler, log_dir, start_time_us): + from smdebug.core.locations import TraceFileLocation + + tf_profiler.stop() + metadata_file = TraceFileLocation.get_tf_profiling_metadata_file( + log_dir, start_time_us, time.time() * CONVERT_TO_MICROSECS + ) + write_tf_profiler_metadata_file(metadata_file) + + +def start_smdataparallel_profiler(smdataparallel, base_dir): + if smdataparallel: + from smdistributed.dataparallel import start_profiler + + smdataparallel_temp_file = os.path.join( + base_dir, f"{get_node_id()}_{SMDATAPARALLELTIMELINE_SUFFIX}{SMDEBUG_TEMP_PATH_SUFFIX}" + ) + ensure_dir(smdataparallel_temp_file) + start_profiler(smdataparallel_temp_file, append_rank=False) + + +def stop_smdataparallel_profiler(smdataparallel, base_dir): + from smdebug.core.locations import TraceFileLocation + + if smdataparallel: + from smdistributed.dataparallel import stop_profiler + + smdataparallel_temp_file = os.path.join( + base_dir, f"{get_node_id()}_{SMDATAPARALLELTIMELINE_SUFFIX}{SMDEBUG_TEMP_PATH_SUFFIX}" + ) + stop_profiler() + new_file_name = TraceFileLocation.get_file_location( + time.time() * CONVERT_TO_MICROSECS, base_dir, suffix=SMDATAPARALLELTIMELINE_SUFFIX + ) + ensure_dir(new_file_name) + if os.path.exists(smdataparallel_temp_file): + shutil.move(smdataparallel_temp_file, new_file_name) diff --git a/smdebug/pytorch/hook.py b/smdebug/pytorch/hook.py index 1eeb0636f0..a1613d768b 100644 --- a/smdebug/pytorch/hook.py +++ b/smdebug/pytorch/hook.py @@ -1,22 +1,73 @@ # Standard Library +import atexit +import os +import time # Third Party import torch import torch.distributed as dist +from packaging import version # First Party from smdebug.core.collection import DEFAULT_PYTORCH_COLLECTIONS, CollectionKeys from smdebug.core.hook import CallbackHook from smdebug.core.json_config import DEFAULT_WORKER_NAME -from smdebug.core.utils import make_numpy_array +from smdebug.core.utils import check_smdataparallel_env, make_numpy_array +from smdebug.profiler.hvd_trace_file_rotation import HvdTraceFileRotation +from smdebug.profiler.profiler_config_parser import MetricsCategory, ProfilerConfigParser +from smdebug.profiler.profiler_constants import CONVERT_TO_MICROSECS +from smdebug.profiler.python_profile_utils import StepPhase, mode_keys_to_python_profile_mode +from smdebug.profiler.python_profiler import PythonProfiler +from smdebug.profiler.utils import start_smdataparallel_profiler, stop_smdataparallel_profiler from smdebug.pytorch.collection import CollectionManager from smdebug.pytorch.singleton_utils import set_hook from smdebug.pytorch.utils import get_reduction_of_data +# smdistributed.dataparallel should be invoked via `mpirun`. +# It supports EC2 machines with 8 GPUs per machine. +smdataparallel = None +if check_smdataparallel_env(): + try: + import smdistributed.dataparallel.torch.distributed as smdataparallel + except ImportError: + pass + + DEFAULT_INCLUDE_COLLECTIONS = [CollectionKeys.LOSSES] +python_profiler = None + +# Enable python profiling if profiling is enabled. +profiler_config_parser = ProfilerConfigParser() +if profiler_config_parser.profiling_enabled: + config = profiler_config_parser.config + if config.python_profiling_config.is_enabled(): + python_profiler = PythonProfiler.get_python_profiler(config, "pytorch") + python_profiler.start_profiling(StepPhase.START) + + class Hook(CallbackHook): + """ + The _TraceEventData is similar to a structure that contains the event data to be written in the event file. + It contains the following metadata: + training_phase such as "X", "M", "I" etc. + start time, duration and end time in microseconds + pid is the process id that wants to record this event. + and event arguments. + """ + + class _TraceEventData: + def __init__(self, phase, op_name, start_time, dur, **kwargs): + self.training_phase = phase + self.end_time = start_time + dur + self.start_time = start_time + self.op_name = op_name + self.kwargs = kwargs + + def update_end_time(self, end_time=time.time()): + self.end_time = end_time + def __init__( self, out_dir=None, @@ -45,6 +96,7 @@ def __init__( include_collections=include_collections, save_all=save_all, include_workers=include_workers, + profiler_config_parser=profiler_config_parser, ) # mapping of module objects to their names, # useful in forward hook for logging input/output of modules @@ -53,16 +105,81 @@ def __init__( self.has_registered_module = False self.has_registered_loss_module = False self.worker = self._get_worker_name() + + # Only the chief worker will read the Horovod timeline file + # if HOROVOD_TIMELINE is a valid file and SM Profiler is enabled + if not self.hvd_reader and self.worker == self.chief_worker: + self.hvd_reader = HvdTraceFileRotation(self.profiler_config_parser) + set_hook(self) + self.parent_forward_event = None + self.parent_backward_event = None + self.step_event = None + self.forward_modules_profile_stats = [] + self.backward_modules_profile_stats = [] + self.first_forward_submodule_name = None + self.autograd_profiler_enabled = False + self.profiler = ( + torch.autograd.ProfilerState.CUDA + if torch.cuda.is_available() + else torch.autograd.ProfilerState.CPU + ) + self.use_cuda = torch.cuda.is_available() + if python_profiler: + atexit.register(python_profiler.stop_profiling, StepPhase.END) + + def log_trace_event(self, event): + self.record_trace_events( + training_phase=event.training_phase, + op_name=event.op_name, + phase="X", + timestamp=event.start_time, + duration=event.end_time - event.start_time, + **(event.kwargs), + ) + + def reset_forward_module_profile_stats(self): + self.parent_forward_event = None + self.forward_modules_profile_stats = [] + + def reset_backward_module_profile_stats(self): + self.parent_backward_event = None + self.backward_modules_profile_stats = [] + + def log_outstanding_timeline_metrics(self): + self.log_outstanding_forward_stats_and_reset() + self.log_outstanding_backward_stats_and_reset() + + def log_outstanding_forward_stats_and_reset(self, log_step_event=True): + # we need to skip the last event for submodules because that is usually the parent event + # and already recorded above + for i in range(len(self.forward_modules_profile_stats) - 1): + event = self.log_trace_event(self.forward_modules_profile_stats[i]) + + if self.parent_forward_event: + self.log_trace_event(self.parent_forward_event) + if self.step_event and log_step_event is True: + self.log_trace_event(self.step_event) + self.step_event = None + + self.reset_forward_module_profile_stats() + + def log_outstanding_backward_stats_and_reset(self): + for i in range(len(self.backward_modules_profile_stats)): + event = self.log_trace_event(self.backward_modules_profile_stats[i]) + + if self.parent_backward_event: + self.log_trace_event(self.parent_backward_event) + self.reset_backward_module_profile_stats() def _get_num_workers(self): - """Check horovod and torch.distributed.""" + """Check horovod, smdataparallel, and torch.distributed.""" # Try torch.distributed # torch.distributed is empty on Mac on Torch <= 1.2 if hasattr(dist, "is_initialized") and dist.is_initialized(): return torch.distributed.get_world_size() - # Try horovod else: + # Try horovod try: import horovod.torch as hvd @@ -70,17 +187,29 @@ def _get_num_workers(self): return hvd.size() except (ModuleNotFoundError, ValueError, ImportError): pass + + # Try smdataparallel + # smdistributed.dataparallel should be invoked via `mpirun`. + # It supports EC2 machines with 8 GPUs per machine. + if check_smdataparallel_env(): + try: + import smdistributed.dataparallel.torch.distributed as smdataparallel + + if smdataparallel.get_world_size(): + return smdataparallel.get_world_size() + except (ModuleNotFoundError, ValueError, ImportError): + pass # Return default return 1 def _get_worker_name(self): - """Check horovod and torch.distributed.""" + """Check horovod, smdataparallel, and torch.distributed.""" # Try torch.distributed # torch.distributed is empty on Mac on Torch <= 1.2 if hasattr(dist, "is_initialized") and dist.is_initialized(): return f"worker_{dist.get_rank()}" - # Try horovod else: + # Try horovod try: import horovod.torch as hvd @@ -88,6 +217,18 @@ def _get_worker_name(self): return f"worker_{hvd.rank()}" except (ModuleNotFoundError, ValueError, ImportError): pass + + # Try smdataparallel + # smdistributed.dataparallel should be invoked via `mpirun`. + # It supports EC2 machines with 8 GPUs per machine. + if check_smdataparallel_env(): + try: + import smdistributed.dataparallel.torch.distributed as smdataparallel + + if smdataparallel.get_world_size(): + return f"worker_{smdataparallel.get_rank()}" + except (ModuleNotFoundError, ValueError, ImportError): + pass # Return default return DEFAULT_WORKER_NAME @@ -117,6 +258,47 @@ def _prepare_collections(self): coll.include(module_name + "_output_") super()._prepare_collections() + def _collect_torch_profiling_data_if_profiler_enabled(self): + if self.autograd_profiler_enabled is False: + return + records = torch.autograd._disable_profiler() + self.autograd_profiler_enabled = False + function_events = torch.autograd.profiler.EventList( + torch.autograd.profiler.parse_cpu_trace(records), use_cuda=self.use_cuda + ) + for index, event in enumerate(function_events): + self.record_trace_events( + training_phase="cpu_functions", + op_name=event.name, + phase="X", + # event.cpu_interval.start is in microseconds + timestamp=(event.cpu_interval.start + self.start_profiler_time_us) + / float( + CONVERT_TO_MICROSECS + ), # timestamp expected is in seconds for record_trace_events + duration=event.cpu_interval.elapsed_us() / float(CONVERT_TO_MICROSECS), + tid=event.thread, + step_num=self.step, + device="cpu", + ) + for k in event.kernels: + self.record_trace_events( + training_phase="gpu_functions-dev:" + str(k.device), + op_name=k.name, + phase="X", + timestamp=(k.interval.start + self.start_profiler_time_us) + / float( + CONVERT_TO_MICROSECS + ), # timestamp expected is in seconds for record_trace_events + duration=k.interval.elapsed_us() / float(CONVERT_TO_MICROSECS), + tid=k.device, + step_num=self.step, + event_name=event.name, + device=k.device, + start_cpu_thread=event.thread, + cpu_thread_start_time=event.cpu_interval.start + self.start_profiler_time_us, + ) + # This hook is invoked by trainer prior to running the forward pass. def forward_pre_hook(self, module, inputs): # Write the gradients of the past step if the writer is still available. @@ -133,6 +315,82 @@ def forward_pre_hook(self, module, inputs): self._increment_step() + ## prepararing for step metrics + # last operation can be forward( eval loop is running or multiple forward for example RNN can have multiple call to forward of module) + # or last operation can be backward (train backward loop just finished and we are at forward again) + + # we will log all outstanding forward and backward events + self.log_outstanding_timeline_metrics() + + self.step_event = self._TraceEventData( + phase="Step:" + str(self.mode), + op_name="Step:" + str(self.mode), + start_time=time.time(), + dur=0, # end time of step_event will be updated every time a forward event or backward is called after this + pid=os.getpid(), + step_num=str(self.mode_steps[self.mode]), + ) + self.parent_forward_event = self._TraceEventData( + phase="Forward", + op_name=module._module_name, + start_time=time.time(), + dur=0, # end time of parent_forward_event will be updated every time a forward event is called after this + pid=os.getpid(), + step_num=str(self.mode_steps[self.mode]), + ) + + self.profiler_config_parser.load_config() + + # Disable python profiling if the python profiler is currently profiling. + if python_profiler: + python_profiler.stop_profiling( + StepPhase.STEP_START, + end_mode=mode_keys_to_python_profile_mode(self.mode), + end_step=self.step, + ) + python_profiler.stop_profiling(StepPhase.STEP_START, self.step) + if self.profiler_config_parser.should_save_metrics( + MetricsCategory.PYTHON_PROFILING, self.step + ): + python_profiler.start_profiling( + StepPhase.STEP_START, + start_mode=mode_keys_to_python_profile_mode(self.mode), + start_step=self.step, + ) + + if self.autograd_profiler_enabled: + self._collect_torch_profiling_data_if_profiler_enabled() + + # should we re-enable profiling for this step? + if ( + self.profiler_config_parser.should_save_metrics( + MetricsCategory.DETAILED_PROFILING, self.step + ) + and not self.autograd_profiler_enabled + ): + if version.parse(torch.__version__) <= version.parse("1.5.1"): + torch.autograd._enable_profiler(torch.autograd.ProfilerConfig(self.profiler, False)) + elif version.parse(torch.__version__) >= version.parse("1.6"): + torch.autograd._enable_profiler( + torch.autograd.ProfilerConfig(self.profiler, False, False) + ) + self.start_profiler_time_us = time.time() * CONVERT_TO_MICROSECS + self.autograd_profiler_enabled = True + + if self.is_smdataparallel_profiling: + # Stop smdataparallel profiling at end step + stop_smdataparallel_profiler( + smdataparallel, self.profiler_config_parser.config.local_path + ) + self.is_smdataparallel_profiling = False + if self.profiler_config_parser.should_save_metrics( + MetricsCategory.SMDATAPARALLEL_PROFILING, self.step + ): + start_smdataparallel_profiler( + smdataparallel, self.profiler_config_parser.config.local_path + ) + self.is_smdataparallel_profiling = True + if self._get_collections_to_save_for_step(): self._initialize_writers() self._log_params(module) @@ -141,6 +399,8 @@ def forward_pre_hook(self, module, inputs): self.export_collections() self.exported_collections = True + self.first_forward_submodule_name = None + def record_tensor_value(self, tensor_name: str, tensor_value: torch.Tensor) -> None: """Used for registering functional directly, such as F.mse_loss().""" assert isinstance( @@ -151,6 +411,31 @@ def record_tensor_value(self, tensor_name: str, tensor_value: torch.Tensor) -> N # This hook is invoked by trainer after running the forward pass. def forward_hook(self, module, inputs, outputs): + # if this is first forward we will use start time of parent as start time, and end time as now + cur_time = time.time() + child_start_time = cur_time + if self.parent_forward_event is not None: + self.parent_forward_event.update_end_time(cur_time) + # if this is first forward we will use start time of parent as start time, and end time as now + child_start_time = self.parent_forward_event.start_time + if self.step_event: + self.step_event.update_end_time(cur_time) + + if len(self.forward_modules_profile_stats) > 0: + # this child start_time is approcximated as last child end time + child_start_time = self.forward_modules_profile_stats[-1].end_time + + event = self._TraceEventData( + phase="Forward-SubModuleInternal", + op_name=module._module_name, + start_time=child_start_time, + dur=cur_time - child_start_time, + pid=os.getpid(), + step_num=str(self.mode_steps[self.mode]), + ) + self.forward_modules_profile_stats.append(event) + if len(self.forward_modules_profile_stats) == 1: + self.first_forward_submodule_name = module._module_name if not self._get_collections_to_save_for_step(): return @@ -197,6 +482,76 @@ def register_hook(self, module): # for compatibility with ZCC patches which call this self.register_module(module) + def fhook(self, module, inputs, outputs): + # we would stop profiling and restart from this phase + if python_profiler: + python_profiler.stop_profiling( + StepPhase.FORWARD_PASS_END, + end_mode=mode_keys_to_python_profile_mode(self.mode), + end_step=self.step, + ) + if self.profiler_config_parser.should_save_metrics( + MetricsCategory.PYTHON_PROFILING, self.step + ): + python_profiler.start_profiling( + StepPhase.FORWARD_PASS_END, + start_mode=mode_keys_to_python_profile_mode(self.mode), + start_step=self.step, + ) + + def bhook(self, module, grad_input, grad_output): + now = time.time() + backward_st_time = now + if self.parent_forward_event is not None: + # note that we are approximating backward start_time here + backward_st_time = self.parent_forward_event.end_time + # we will not log step event yet, we will log step event only during forward pre-step + # as step is FW + Backward + self.log_outstanding_forward_stats_and_reset(log_step_event=False) + # if this is first backward hook call, we will create backward event with start_Ts as last_forward_end_ts + if self.parent_backward_event is None: + self.parent_backward_event = self._TraceEventData( + phase="Backward(post-forward)", + op_name=module._module_name, + start_time=backward_st_time, + dur=now - backward_st_time, + pid=os.getpid(), + step_num=str(self.mode_steps[self.mode]), + ) + if self.parent_backward_event: + self.parent_backward_event.update_end_time(now) + if self.step_event: + self.step_event.update_end_time(now) + # if this is not first backward we will use start time of parent as start time, and end time as now + if len(self.backward_modules_profile_stats) > 0: + # this child start_time is approcximated as last child end time + child_start_time = self.backward_modules_profile_stats[-1].end_time + else: + child_start_time = backward_st_time + event = self._TraceEventData( + phase="Backward-SubModuleInternal", + op_name=module._module_name, + start_time=child_start_time, + dur=now - child_start_time, + pid=os.getpid(), + step_num=str(self.mode_steps[self.mode]), + ) + self.backward_modules_profile_stats.append(event) + + def _closure_for_registering_backward_hook(self, module): + module.register_backward_hook(self.bhook) + + def count_parameters(self, model): + total_params = 0 + for name, parameter in model.named_parameters(): + if not parameter.requires_grad: + continue + param = parameter.numel() + self.logger.info(f"name:{name} count_params:{param}") + total_params += param + self.logger.info(f"Total Trainable Params: {total_params}") + return total_params + def register_module(self, module): """ This function registers the forward hook. If user wants to register the hook @@ -209,10 +564,17 @@ def register_module(self, module): raise ValueError( f"Module type {module.__class__.__name__} must be type torch.nn.Module" ) - + # in case GPU is available but model has been loaded on CPU + for parameter in module.parameters(): + self.profiler = ( + torch.autograd.ProfilerState.CUDA + if parameter.is_cuda + else torch.autograd.ProfilerState.CPU + ) + self.use_cuda = parameter.is_cuda + break # Create an attribute and store the module name in the object # So that it is available in the forward hook. - for name, submodule in module.named_modules(): assert submodule not in self.module_set, f"Don't register module={module} twice" submodule._module_name = name @@ -222,6 +584,7 @@ def register_module(self, module): # Use `forward_pre_hook` for the entire net module.register_forward_pre_hook(self.forward_pre_hook) + module.register_forward_hook(self.fhook) # Set `self.forward_hook` as a callback for each submodule/layer. # `module.apply(fn)` calls fn for each submodule in module.children() @@ -230,7 +593,16 @@ def register_module(self, module): # Capture the gradient for each parameter in the net self._backward_apply(module) + # TODO: Registering the backward hook causes issues in certain cases. There is a ‘Warning’ ( + # https://pytorch.org/docs/stable/generated/torch.nn.Module.html?highlight=register_backward_hook#torch.nn.Module.register_backward_hook) for using this hook in certain cases. + # The ‘__call_impl” in PyTorch Module class makes some assumptions about ‘results’ returned from the forward pass of the module. It can not operate correctly if ‘forward’ pass returns anything other than dictionary of torch.Tensors. Some of the torchvision.transform classes returned ‘PIL’ image object and backward hook used to crash. + # In some cases, we have seen the the training hangs. Hence currently the following functionality is + # commented. We can revisit it after understanding the PyTorch's implementation of backward hook. + + # module.apply(self._closure_for_registering_backward_hook) + self.has_registered_module = True + self.count_parameters(module) def register_loss(self, loss_module): """Register something like `criterion = nn.CrossEntropyLoss()`.""" @@ -245,6 +617,18 @@ def register_loss(self, loss_module): loss_module.register_forward_hook(self.forward_hook) self.has_registered_loss_module = True + def close(self): + self._cleanup() + if python_profiler: + python_profiler.start_profiling( + StepPhase.STEP_END, + start_mode=mode_keys_to_python_profile_mode(self.mode), + start_step=self.mode_steps[self.mode], + ) + + def _cleanup(self): + super()._cleanup() + @staticmethod def _get_reduction_of_data(reduction_name, tensor_value, tensor_name, abs): return get_reduction_of_data(reduction_name, tensor_value, tensor_name, abs) @@ -254,3 +638,11 @@ def _make_numpy_array(tensor_value): if isinstance(tensor_value, torch.Tensor): return tensor_value.to(torch.device("cpu")).data.numpy() return make_numpy_array(tensor_value) + + def should_save_dataloader_metrics(self, metrics_name): + """Determine whether dataloader metrics for the provided metrics_name should be saved. We check for the next + step since the dataloader metrics for the next step are collected on the current step. + """ + return self.profiler_config_parser.should_save_metrics( + MetricsCategory.DATALOADER_PROFILING, self.step + 1, metrics_name=metrics_name + ) diff --git a/smdebug/rules/rule.py b/smdebug/rules/rule.py index 1cebb99726..5e20272fd1 100644 --- a/smdebug/rules/rule.py +++ b/smdebug/rules/rule.py @@ -26,6 +26,13 @@ def __init__(self, base_trial, other_trials=None, action_str=""): self.logger = get_logger() self.rule_name = self.__class__.__name__ self._actions = Actions(actions_str=action_str, rule_name=self.rule_name) + self.report = { + "RuleTriggered": 0, + "Violations": 0, + "Details": {}, + "Datapoints": 0, + "RuleParameters": "", + } def set_required_tensors(self, step): pass diff --git a/smdebug/rules/rule_invoker.py b/smdebug/rules/rule_invoker.py index 04cfe9db2d..8d8472a9c2 100644 --- a/smdebug/rules/rule_invoker.py +++ b/smdebug/rules/rule_invoker.py @@ -1,6 +1,7 @@ # First Party from smdebug.core.logger import get_logger from smdebug.exceptions import ( + NoMoreProfilerData, RuleEvaluationConditionMet, StepUnavailable, TensorUnavailable, @@ -19,10 +20,21 @@ def invoke_rule(rule_obj, start_step=0, end_step=None, raise_eval_cond=False): except (TensorUnavailableForStep, StepUnavailable, TensorUnavailable) as e: logger.debug(str(e)) except RuleEvaluationConditionMet as e: + # If raise_eval_cond specified, pop up the exception. if raise_eval_cond: raise e else: logger.debug(str(e)) + # In case RuleEvaluationConditionMet indicated the end of the rule, break the execution loop. + if e.end_of_rule: + break + except NoMoreProfilerData as e: + logger.info( + "No more profiler data for rule {} at timestamp {}".format( + type(rule_obj).__name__, e.timestamp + ) + ) + break step += 1 # decrementing because we increment step in the above line logger.info( diff --git a/smdebug/tensorflow/base_hook.py b/smdebug/tensorflow/base_hook.py index 357610da2d..64ec2d7ca4 100644 --- a/smdebug/tensorflow/base_hook.py +++ b/smdebug/tensorflow/base_hook.py @@ -14,7 +14,7 @@ from smdebug.core.hook import BaseHook from smdebug.core.modes import ModeKeys from smdebug.core.reductions import get_numpy_reduction, get_reduction_tensor_name -from smdebug.core.utils import make_numpy_array, serialize_tf_device +from smdebug.core.utils import check_smdataparallel_env, make_numpy_array, serialize_tf_device from smdebug.core.writer import FileWriter # Local @@ -61,6 +61,7 @@ def __init__( include_collections=None, save_all=False, include_workers="one", + profiler_config_parser=None, ): collection_manager = CollectionManager() super().__init__( @@ -77,6 +78,7 @@ def __init__( include_collections=include_collections, save_all=save_all, include_workers=include_workers, + profiler_config_parser=profiler_config_parser, ) self.optimizer = None self._custom_collections = None @@ -131,6 +133,18 @@ def _get_distribution_strategy(self) -> TFDistributionStrategy: except (ModuleNotFoundError, ValueError, ImportError): pass + # smdistributed.dataparallel should be invoked via `mpirun`. + # It supports EC2 machines with 8 GPUs per machine. + if check_smdataparallel_env(): + try: + import smdistributed.dataparallel.tensorflow as smdataparallel + + # The total number of GPUs across all the nodes in the cluster + if smdataparallel.size(): + return TFDistributionStrategy.SMDATAPARALLEL + except (ModuleNotFoundError, ValueError, ImportError): + pass + strat = tf.distribute.get_strategy() if is_mirrored_strategy(strat): return TFDistributionStrategy.MIRRORED @@ -172,6 +186,10 @@ def _get_worker_name(self) -> str: import horovod.tensorflow as hvd return f"worker_{hvd.rank()}" + elif self.distribution_strategy == TFDistributionStrategy.SMDATAPARALLEL: + import smdistributed.dataparallel.tensorflow as smdataparallel + + return f"worker_{smdataparallel.rank()}" elif self.distribution_strategy == TFDistributionStrategy.MIRRORED: # unused for this strategy return DEFAULT_WORKER_NAME @@ -201,6 +219,7 @@ def export_collections(self): if self.distribution_strategy in [ TFDistributionStrategy.PARAMETER_SERVER, TFDistributionStrategy.HOROVOD, + TFDistributionStrategy.SMDATAPARALLEL, ]: if self.save_all_workers is False and self.worker != self.chief_worker: return @@ -244,6 +263,10 @@ def _get_num_workers(self): import horovod.tensorflow as hvd return hvd.size() + elif self.distribution_strategy == TFDistributionStrategy.SMDATAPARALLEL: + import smdistributed.dataparallel.tensorflow as smdataparallel + + return smdataparallel.size() elif self.distribution_strategy == TFDistributionStrategy.MIRRORED: strategy = tf.distribute.get_strategy() return strategy.num_replicas_in_sync @@ -259,6 +282,8 @@ def _set_chief_worker(self): # this won't be used if save_all_workers is True if self.distribution_strategy == TFDistributionStrategy.HOROVOD: self.chief_worker = DEFAULT_WORKER_NAME + elif self.distribution_strategy == TFDistributionStrategy.SMDATAPARALLEL: + self.chief_worker = DEFAULT_WORKER_NAME elif self.distribution_strategy == TFDistributionStrategy.MIRRORED: assert self._prepared_tensors[self.mode] if len(self.device_map): @@ -285,6 +310,7 @@ def _get_writers(self, tensor_name, tensor_ref) -> List[FileWriter]: if self.distribution_strategy in [ TFDistributionStrategy.PARAMETER_SERVER, TFDistributionStrategy.HOROVOD, + TFDistributionStrategy.SMDATAPARALLEL, ]: if self.save_all_workers is True or self.worker == self.chief_worker: return self._get_main_writer() @@ -322,6 +348,7 @@ def _initialize_writers(self, only_initialize_if_missing=False) -> None: if self.distribution_strategy in [ TFDistributionStrategy.PARAMETER_SERVER, TFDistributionStrategy.HOROVOD, + TFDistributionStrategy.SMDATAPARALLEL, ]: if self.save_all_workers is True or self.worker == self.chief_worker: if self.writer is None or only_initialize_if_missing is False: diff --git a/smdebug/tensorflow/keras.py b/smdebug/tensorflow/keras.py index f0426c1ed5..a6b265b15b 100644 --- a/smdebug/tensorflow/keras.py +++ b/smdebug/tensorflow/keras.py @@ -1,5 +1,8 @@ # Standard Library +import atexit import functools +import os +import time # Third Party import tensorflow.compat.v1 as tf @@ -8,8 +11,19 @@ from tensorflow.python.util import nest # First Party +from smdebug.core.locations import TraceFileLocation from smdebug.core.modes import ModeKeys from smdebug.core.utils import match_inc +from smdebug.profiler.hvd_trace_file_rotation import HvdTraceFileRotation +from smdebug.profiler.profiler_config_parser import MetricsCategory, ProfilerConfigParser +from smdebug.profiler.profiler_constants import ( + CONVERT_TO_MICROSECS, + TF_DATALOADER_END_FLAG_FILENAME, + TF_DATALOADER_START_FLAG_FILENAME, +) +from smdebug.profiler.python_profile_utils import StepPhase, mode_keys_to_python_profile_mode +from smdebug.profiler.python_profiler import PythonProfiler +from smdebug.profiler.utils import stop_tf_profiler from smdebug.tensorflow.callable_cache import CallableCache from smdebug.tensorflow.utils import InputOutputSaver, get_layer_call_fn @@ -31,11 +45,22 @@ get_model_input_export_name, get_model_output_export_name, is_keras_optimizer, + is_profiler_supported_for_tf_version, is_tf_version_2_3_x, is_tf_version_2x, supported_tf_variables, ) +python_profiler = None + +# Enable python profiling if profiling is enabled. +profiler_config_parser = ProfilerConfigParser() +if profiler_config_parser.profiling_enabled: + config = profiler_config_parser.config + if config.python_profiling_config.is_enabled(): + python_profiler = PythonProfiler.get_python_profiler(config, "tensorflow") + python_profiler.start_profiling(StepPhase.START) + class KerasHook(TensorflowBaseHook, tf.keras.callbacks.Callback): def __init__( @@ -64,6 +89,7 @@ def __init__( include_collections=include_collections, save_all=save_all, include_workers=include_workers, + profiler_config_parser=profiler_config_parser, ) tf.keras.callbacks.Callback.__init__(self) self.tensor_refs_to_save_this_step = set() @@ -74,6 +100,18 @@ def __init__( ) # stores tensors custom tensors saved by users every step self.saved_layers = dict() self.has_registered_model = False + + # Profiling vars + self.tf_profiler = None + if is_profiler_supported_for_tf_version(): + from tensorflow.python.profiler import profiler_v2 as tf_profiler + + self.tf_profiler = tf_profiler + self._log_dir = None + self.is_detailed_profiling = False + self.is_dataloader_profiling = False + self.tf_profiler_start_time_in_micros = 0 + self.warm_up_completed = False # supports_tf_logs property was introduced in TF 2.3.0 # it indicates to the framework that the callback is not # limited to reading only numpy logs @@ -83,6 +121,9 @@ def __init__( # the the step was already incremented in the on_train_begin callback self.step_incremented_in_on_train_begin = False + if python_profiler: + atexit.register(python_profiler.stop_profiling, StepPhase.END) + def _is_not_supported(self): if self.distribution_strategy is None: self.distribution_strategy = self._get_distribution_strategy() @@ -599,6 +640,7 @@ def _get_exec_function(self, mode): if self.distribution_strategy in [ TFDistributionStrategy.NONE, TFDistributionStrategy.HOROVOD, + TFDistributionStrategy.SMDATAPARALLEL, ]: if mode == ModeKeys.TRAIN: x = self.model.train_function @@ -676,6 +718,12 @@ def _on_any_mode_begin(self, mode): if self._is_not_supported(): return self.worker = self._get_worker_name() + + # Only the chief worker will read the Horovod timeline file + # if HOROVOD_TIMELINE is a valid file and SM Profiler is enabled + if not self.hvd_reader and self.worker == self.chief_worker: + self.hvd_reader = HvdTraceFileRotation(self.profiler_config_parser) + self.graph = tf.get_default_graph() self.set_mode(mode) @@ -695,13 +743,43 @@ def on_train_begin(self, logs=None): def on_test_begin(self, logs=None): self._on_any_mode_begin(ModeKeys.EVAL) + def _on_any_mode_end(self, mode): + if python_profiler: + python_profiler.stop_profiling( + StepPhase.STEP_END, + end_mode=mode_keys_to_python_profile_mode(mode), + end_step=self.mode_steps[mode], + ) + python_profiler.start_profiling( + StepPhase.STEP_END, + start_mode=mode_keys_to_python_profile_mode(mode), + start_step=self.mode_steps[mode], + ) + + if self.is_dataloader_profiling and self.profiler_config_parser.write_tf_dataloader_flag( + TF_DATALOADER_END_FLAG_FILENAME + ): + self.is_dataloader_profiling = False + + def on_train_end(self, logs=None): + self._on_any_mode_end(ModeKeys.TRAIN) + + if is_profiler_supported_for_tf_version() and self.is_detailed_profiling: + self.logger.info("Disabling profiler, reached end of training.") + stop_tf_profiler( + tf_profiler=self.tf_profiler, + log_dir=self._log_dir, + start_time_us=self.tf_profiler_start_time_in_micros, + ) + self.is_detailed_profiling = False + # throws error in keras if this fn is absent def on_test_end(self, logs=None): - pass + self._on_any_mode_end(ModeKeys.EVAL) # throws error in keras if this fn is absent def on_predict_end(self, logs=None): - pass + self._on_any_mode_end(ModeKeys.PREDICT) def on_predict_begin(self, logs=None): self._on_any_mode_begin(ModeKeys.PREDICT) @@ -718,6 +796,7 @@ def _wrap_model_with_input_output_saver(self): self.saved_layers[layer.name] = saver def _on_any_batch_begin(self, batch, mode, logs=None): + self.start = time.time() if self._is_not_supported(): return @@ -736,6 +815,34 @@ def _on_any_batch_begin(self, batch, mode, logs=None): else: self.step_incremented_in_on_train_begin = False + self.profiler_config_parser.load_config() + + if self.profiler_config_parser.should_save_metrics( + MetricsCategory.DATALOADER_PROFILING, self.mode_steps[mode] + ) and self.profiler_config_parser.write_tf_dataloader_flag( + TF_DATALOADER_START_FLAG_FILENAME + ): + self.is_dataloader_profiling = True + elif self.is_dataloader_profiling and self.profiler_config_parser.write_tf_dataloader_flag( + TF_DATALOADER_END_FLAG_FILENAME + ): + self.is_dataloader_profiling = False + + if python_profiler: + python_profiler.stop_profiling( + StepPhase.STEP_START, + end_mode=mode_keys_to_python_profile_mode(mode), + end_step=self.mode_steps[mode], + ) + if self.profiler_config_parser.should_save_metrics( + MetricsCategory.PYTHON_PROFILING, self.mode_steps[mode] + ): + python_profiler.start_profiling( + StepPhase.STEP_START, + start_mode=mode_keys_to_python_profile_mode(mode), + start_step=self.mode_steps[mode], + ) + if self.prepared_collections is False: # sets prepared_collections to True here self._prepare_collections() @@ -744,6 +851,7 @@ def _on_any_batch_begin(self, batch, mode, logs=None): if (is_tf_version_2x() and tf.executing_eagerly()) or self._validate_exec_function( self._get_exec_function(mode) ): + self._wrap_model_with_input_output_saver() self._prepare_layers(mode) self._prepare_non_layer_tensors() self._prepare_tensors_available_post_step() @@ -768,6 +876,37 @@ def _on_any_batch_begin(self, batch, mode, logs=None): def on_train_batch_begin(self, batch, logs=None): self._on_any_batch_begin(batch, ModeKeys.TRAIN, logs=logs) + if is_profiler_supported_for_tf_version(): + if self.profiler_config_parser.should_save_metrics( + MetricsCategory.DETAILED_PROFILING, self.mode_steps[ModeKeys.TRAIN] + ): + if not self.is_detailed_profiling: + self._log_dir = TraceFileLocation.get_detailed_profiling_log_dir( + self.profiler_config_parser.config.local_path, + "tensorflow", + self.mode_steps[ModeKeys.TRAIN], + ) + self.logger.info( + f"Enabling TF profiler on step: = {self.mode_steps[ModeKeys.TRAIN]}" + ) + if not self.warm_up_completed: + # warming up profiler before it will be profiling. + self.tf_profiler.warmup() + self.warm_up_completed = True + self.tf_profiler.start(self._log_dir) + self.tf_profiler_start_time_in_micros = time.time() * CONVERT_TO_MICROSECS + self.is_detailed_profiling = True + elif self.is_detailed_profiling: + self.logger.info( + f"Disabling TF profiler on step: ={self.mode_steps[ModeKeys.TRAIN]}" + ) + stop_tf_profiler( + tf_profiler=self.tf_profiler, + log_dir=self._log_dir, + start_time_us=self.tf_profiler_start_time_in_micros, + ) + self.is_detailed_profiling = False + def on_test_batch_begin(self, batch, logs=None): self._on_any_batch_begin(batch, ModeKeys.EVAL, logs=logs) @@ -821,6 +960,16 @@ def _on_any_batch_end(self, batch, mode, logs=None): if self._is_not_supported(): return + self.record_trace_events( + training_phase="Step:" + str(mode), + op_name="Step:" + str(mode), + phase="X", + timestamp=self.start, # this is start time for step + duration=time.time() - self.start, + pid=os.getpid(), + step_num=str(self.mode_steps[mode]), + ) + if not is_tf_version_2x() or (is_tf_version_2x() and not tf.executing_eagerly()): self._remove_fetches_and_callbacks(mode) @@ -849,6 +998,21 @@ def _on_any_batch_end(self, batch, mode, logs=None): self._export_model() self._exported_model[self.mode] = True + if python_profiler: + python_profiler.stop_profiling( + StepPhase.STEP_END, + end_mode=mode_keys_to_python_profile_mode(mode), + end_step=self.mode_steps[mode], + ) + if self.profiler_config_parser.should_save_metrics( + MetricsCategory.PYTHON_PROFILING, self.mode_steps[mode] + ): + python_profiler.start_profiling( + StepPhase.STEP_END, + start_mode=mode_keys_to_python_profile_mode(mode), + start_step=self.mode_steps[mode], + ) + def on_train_batch_end(self, batch, logs=None): self._on_any_batch_end(batch, ModeKeys.TRAIN, logs=logs) @@ -927,6 +1091,15 @@ def unwrap(func): self.tape.__class__._pop_tape = unwrap(self.tape.__class__._pop_tape) self.tape.__class__.gradient = unwrap(self.tape.__class__.gradient) + def close(self): + self._cleanup() + if python_profiler: + python_profiler.start_profiling( + StepPhase.STEP_END, + start_mode=mode_keys_to_python_profile_mode(self.mode), + start_step=self.mode_steps[self.mode], + ) + def _cleanup(self): # Unwrap the tape before closing if self.tape: @@ -1080,6 +1253,14 @@ def wrap_tape(self, tape): :return: Wrapped tape of same type as passed. This tape should be used for training """ + # Disable python profiling, because now we are starting wrap tape. + if python_profiler: + python_profiler.stop_profiling( + StepPhase.STEP_START, + end_mode=mode_keys_to_python_profile_mode(self.mode), + end_step=0, + ) + from tensorflow.python.eager.backprop import GradientTape if isinstance(tape, GradientTape): diff --git a/smdebug/tensorflow/utils.py b/smdebug/tensorflow/utils.py index 552a72fff8..e4a3cd4491 100644 --- a/smdebug/tensorflow/utils.py +++ b/smdebug/tensorflow/utils.py @@ -6,6 +6,7 @@ # Third Party import tensorflow as tf +import tensorflow.compat.v1 as tf_v1 from packaging import version from tensorflow.python.distribute import values @@ -23,9 +24,9 @@ def supported_tf_variables(): if does_tf_support_mixed_precision_training(): from tensorflow.python.keras.mixed_precision.experimental import autocast_variable - return tf.Variable, autocast_variable.AutoCastVariable + return tf_v1.Variable, autocast_variable.AutoCastVariable else: - return tf.Variable + return tf_v1.Variable class ModelOutput: @@ -69,6 +70,7 @@ class TFDistributionStrategy(Enum): HOROVOD = 1 MIRRORED = 2 PARAMETER_SERVER = 3 + SMDATAPARALLEL = 4 UNSUPPORTED = 100 @@ -401,5 +403,13 @@ def is_tf_version_2x(): return version.parse(tf.__version__) >= version.parse("2.0.0") +def is_tf_version_2_2_x(): + return version.parse("2.2.0") <= version.parse(tf.__version__) < version.parse("2.3.0") + + def is_tf_version_2_3_x(): - return version.parse(tf.__version__) >= version.parse("2.3.0") + return version.parse("2.3.0") <= version.parse(tf.__version__) < version.parse("2.4.0") + + +def is_profiler_supported_for_tf_version(): + return is_tf_version_2_2_x() or is_tf_version_2_3_x() diff --git a/smdebug/trials/profiler_trial.py b/smdebug/trials/profiler_trial.py new file mode 100644 index 0000000000..29f4af008e --- /dev/null +++ b/smdebug/trials/profiler_trial.py @@ -0,0 +1,109 @@ +# Standard Library +import os +import pathlib +import time + +# First Party +from smdebug.core.access_layer.utils import has_training_ended, is_rule_signalled_gracetime_passed +from smdebug.core.logger import get_logger +from smdebug.exceptions import NoMoreProfilerData +from smdebug.profiler.algorithm_metrics_reader import ( + LocalAlgorithmMetricsReader, + S3AlgorithmMetricsReader, +) +from smdebug.profiler.system_metrics_reader import LocalSystemMetricsReader, S3SystemMetricsReader + + +class ProfilerTrial: + def __init__(self, name, trial_dir, output_dir): + self.name = name + # Trial dir is the s3/local directory contains profiling data captured during runtime. + self.path = trial_dir + + self.logger = get_logger() + self.first_timestamp = 0 + self.get_first_timestamp() + + # Output directory will contains data emitted by rules further published to S3. + self.output_dir = output_dir + if output_dir and not os.path.exists(output_dir): + pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True) + + # .sagemaker-ignore will not be picked by service code for uploading. It will be used to save temp files. + self.temp_dir = os.path.join(output_dir, ".sagemaker-ignore") + if not os.path.exists(self.temp_dir): + pathlib.Path(self.temp_dir).mkdir(parents=True, exist_ok=True) + self.logger.info( + "Output files of ProfilerTrial will be saved to {}".format(self.output_dir) + ) + + def job_finished(self): + training_ended = has_training_ended(self.path + "/system") or has_training_ended( + self.path + "/framework" + ) + # rule job should finish if training job has ended or rule job has been signalled + return training_ended or is_rule_signalled_gracetime_passed(self.path) + + def get_first_timestamp(self): + while self.first_timestamp == 0: + if self.path.startswith("s3"): + self.system_metrics_reader = S3SystemMetricsReader(self.path) + self.framework_metrics_reader = S3AlgorithmMetricsReader(self.path) + else: + self.system_metrics_reader = LocalSystemMetricsReader(self.path) + self.framework_metrics_reader = LocalAlgorithmMetricsReader(self.path) + if self.system_metrics_reader.get_timestamp_of_first_available_file() != 0: + self.first_timestamp = ( + self.system_metrics_reader.get_timestamp_of_first_available_file() + ) + if self.framework_metrics_reader.get_timestamp_of_first_available_file() != 0: + if ( + self.framework_metrics_reader.get_timestamp_of_first_available_file() + < self.first_timestamp + ): + self.first_timestamp = ( + self.framework_metrics_reader.get_timestamp_of_first_available_file() + ) + self.logger.info("Waiting for profiler data.") + time.sleep(10) + + def get_latest_timestamp(self): + latest_timestamp = 0 + self.system_metrics_reader.refresh_event_file_list() + self.framework_metrics_reader.refresh_event_file_list() + if self.system_metrics_reader.get_timestamp_of_latest_available_file() != 0: + latest_timestamp = self.system_metrics_reader.get_timestamp_of_latest_available_file() + if self.framework_metrics_reader.get_timestamp_of_latest_available_file() != 0: + if ( + self.framework_metrics_reader.get_timestamp_of_latest_available_file() + < latest_timestamp + ): + latest_timestamp = ( + self.framework_metrics_reader.get_timestamp_of_latest_available_file() + ) + return latest_timestamp + + def wait_for_data(self, end_time, start_time): + + if end_time < self.get_latest_timestamp(): + return + + while end_time > self.get_latest_timestamp() and self.job_finished() is not True: + self.logger.info( + f"Current timestamp {end_time} latest timestamp {self.get_latest_timestamp()}: waiting for new profiler data." + ) + + time.sleep(10) + + if self.job_finished(): + if start_time >= self.get_latest_timestamp(): + raise NoMoreProfilerData(end_time) + return + + def get_system_metrics(self, start_time, end_time): + events = self.system_metrics_reader.get_events(start_time, end_time) + return events + + def get_framework_metrics(self, start_time, end_time): + events = self.framework_metrics_reader.get_events(start_time, end_time) + return events diff --git a/smdebug/trials/utils.py b/smdebug/trials/utils.py index 6fcab710b2..448e5a752c 100644 --- a/smdebug/trials/utils.py +++ b/smdebug/trials/utils.py @@ -6,14 +6,19 @@ # Local from .local_trial import LocalTrial +from .profiler_trial import ProfilerTrial from .s3_trial import S3Trial -def create_trial(path, name=None, **kwargs): +def create_trial( + path, name=None, profiler=False, output_dir="/opt/ml/processing/outputs/", **kwargs +): path = path.strip() # Remove any accidental leading/trailing whitespace input by the user if name is None: name = os.path.basename(path) s3, bucket_name, prefix_name = is_s3(path) + if profiler: + return ProfilerTrial(name=name, trial_dir=path, output_dir=output_dir, **kwargs) if s3: return S3Trial(name=name, bucket_name=bucket_name, prefix_name=prefix_name, **kwargs) else: diff --git a/tests/conftest.py b/tests/conftest.py index 44616dc64d..694775c34b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,12 +5,18 @@ """ # Standard Library +import json +import os import shutil import sys # Third Party import pytest +# First Party +from smdebug.core.json_config import DEFAULT_RESOURCE_CONFIG_FILE +from smdebug.profiler.profiler_config_parser import ProfilerConfigParser + def pytest_addoption(parser): # Anything taking longer than 2 seconds is slow @@ -52,6 +58,32 @@ def out_dir(): return out_dir +@pytest.fixture +def config_folder(): + """Path to folder used for storing different config artifacts for testing timeline writer and + profiler config parser. + """ + return "tests/core/json_configs" + + +@pytest.fixture +def set_up_resource_config(): + os.makedirs(os.path.dirname(DEFAULT_RESOURCE_CONFIG_FILE), exist_ok=True) + with open(DEFAULT_RESOURCE_CONFIG_FILE, "w") as config_file: + json.dump({"current_host": "test_hostname"}, config_file) + + +@pytest.fixture() +def set_up_smprofiler_config_path(monkeypatch): + config_path = "tests/core/json_configs/simple_profiler_config_parser.json" + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + + +@pytest.fixture() +def simple_profiler_config_parser(set_up_smprofiler_config_path): + return ProfilerConfigParser() + + # In TF, once we disable eager execution, we cannot re-enable eager execution. # The following two fixtures will enable the script `tests.sh` to execute all # tests in eager mode first followed by non-eager mode. diff --git a/tests/core/json_configs/case_insensitive_profiler_config_parser.json b/tests/core/json_configs/case_insensitive_profiler_config_parser.json new file mode 100644 index 0000000000..8892118fd7 --- /dev/null +++ b/tests/core/json_configs/case_insensitive_profiler_config_parser.json @@ -0,0 +1,10 @@ +{ + "profilingparameters": { + "PROFILERENABLED": true, + "localPath": "/tmp/test", + "rotateMaxFileSizeInBYTES": 100, + "RotatefileCloseintervalInseconds": 1, + "FILEopenFAILthreshold": 5, + "detailedProfilingconfig": "{\"startstep\": \"2\", \"NUMSTEPS\": \"3\"}" + } +} diff --git a/tests/core/json_configs/complete_profiler_config_parser.json b/tests/core/json_configs/complete_profiler_config_parser.json new file mode 100644 index 0000000000..cb1e349fe8 --- /dev/null +++ b/tests/core/json_configs/complete_profiler_config_parser.json @@ -0,0 +1,9 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": true, + "LocalPath": "/tmp/test", + "RotateMaxFileSizeInBytes": 300, + "RotateFileCloseIntervalInSeconds": 0.5, + "FileOpenFailThreshold": 2 + } +} diff --git a/tests/core/json_configs/file_interval_rotation_profiler_config_parser.json b/tests/core/json_configs/file_interval_rotation_profiler_config_parser.json new file mode 100644 index 0000000000..7810280990 --- /dev/null +++ b/tests/core/json_configs/file_interval_rotation_profiler_config_parser.json @@ -0,0 +1,8 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": true, + "LocalPath": "/tmp/test", + "RotateFileCloseIntervalInSeconds": 0.5, + "FileOpenFailThreshold": 2 + } +} diff --git a/tests/core/json_configs/file_open_fail_profiler_config_parser.json b/tests/core/json_configs/file_open_fail_profiler_config_parser.json new file mode 100644 index 0000000000..f5d28bd6e5 --- /dev/null +++ b/tests/core/json_configs/file_open_fail_profiler_config_parser.json @@ -0,0 +1,10 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": true, + "LocalPath": "/dev/null/fileopenfailtest", + "RotateMaxFileSizeInBytes": 300, + "RotateFileCloseIntervalInSeconds": 0.5, + "FileOpenFailThreshold": 2, + "DetailedProfilingConfig": "{\"StartStep\": 2, \"NumSteps\": 1}" + } +} diff --git a/tests/core/json_configs/file_size_rotation_profiler_config_parser.json b/tests/core/json_configs/file_size_rotation_profiler_config_parser.json new file mode 100644 index 0000000000..c75c3ce68e --- /dev/null +++ b/tests/core/json_configs/file_size_rotation_profiler_config_parser.json @@ -0,0 +1,8 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": true, + "LocalPath": "/tmp/test", + "RotateMaxFileSizeInBytes": 800, + "FileOpenFailThreshold": 2 + } +} diff --git a/tests/core/json_configs/hvd_rotation_profiler_config_parser.json b/tests/core/json_configs/hvd_rotation_profiler_config_parser.json new file mode 100644 index 0000000000..bbcfdf7180 --- /dev/null +++ b/tests/core/json_configs/hvd_rotation_profiler_config_parser.json @@ -0,0 +1,8 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": true, + "LocalPath": "/tmp/test", + "RotateMaxFileSizeInBytes": 100000, + "RotateFileCloseIntervalInSeconds": 1 + } +} diff --git a/tests/core/json_configs/invalid_profiler_config_parser.json b/tests/core/json_configs/invalid_profiler_config_parser.json new file mode 100644 index 0000000000..38d91d8974 --- /dev/null +++ b/tests/core/json_configs/invalid_profiler_config_parser.json @@ -0,0 +1 @@ +{1, 2, 3, 4, 5} diff --git a/tests/core/json_configs/invalid_string_data_profiler_config_parser.json b/tests/core/json_configs/invalid_string_data_profiler_config_parser.json new file mode 100644 index 0000000000..f605f1e0d3 --- /dev/null +++ b/tests/core/json_configs/invalid_string_data_profiler_config_parser.json @@ -0,0 +1,10 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": "None", + "LocalPath": "/tmp/test", + "RotateMaxFileSizeInBytes": "None", + "RotateFileCloseIntervalInSeconds": "None", + "FileOpenFailThreshold": "None", + "DetailedProfilingConfig": "{\"StartStep\": \"None\", \"NumSteps\": \"None\", \"StartTimeInSecSinceEpoch\": \"None\", \"DurationInSeconds\": \"None\"}" + } +} diff --git a/tests/core/json_configs/new_step_profiler_config_parser.json b/tests/core/json_configs/new_step_profiler_config_parser.json new file mode 100644 index 0000000000..635009ae44 --- /dev/null +++ b/tests/core/json_configs/new_step_profiler_config_parser.json @@ -0,0 +1,7 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": true, + "LocalPath": "/tmp/test", + "DetailedProfilingConfig": "{\"StartStep\": 10, \"NumSteps\": 5}" + } +} diff --git a/tests/core/json_configs/new_time_profiler_config_parser.json b/tests/core/json_configs/new_time_profiler_config_parser.json new file mode 100644 index 0000000000..8c5c2610d5 --- /dev/null +++ b/tests/core/json_configs/new_time_profiler_config_parser.json @@ -0,0 +1,7 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": true, + "LocalPath": "/tmp/test", + "DetailedProfilingConfig": "{\"StartTimeInSecSinceEpoch\": 1700000000, \"DurationInSeconds\": 5}" + } +} diff --git a/tests/core/json_configs/old_step_profiler_config_parser.json b/tests/core/json_configs/old_step_profiler_config_parser.json new file mode 100644 index 0000000000..f68bf081c2 --- /dev/null +++ b/tests/core/json_configs/old_step_profiler_config_parser.json @@ -0,0 +1,7 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": true, + "LocalPath": "/tmp/test", + "DetailedProfilingConfig": "{\"NumSteps\": 2}" + } +} diff --git a/tests/core/json_configs/old_time_profiler_config_parser.json b/tests/core/json_configs/old_time_profiler_config_parser.json new file mode 100644 index 0000000000..b8faf41633 --- /dev/null +++ b/tests/core/json_configs/old_time_profiler_config_parser.json @@ -0,0 +1,7 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": true, + "LocalPath": "/tmp/test", + "DetailedProfilingConfig": "{\"DurationInSeconds\": 0.1}" + } +} diff --git a/tests/core/json_configs/simple_profiler_config_parser.json b/tests/core/json_configs/simple_profiler_config_parser.json new file mode 100644 index 0000000000..64c31555d7 --- /dev/null +++ b/tests/core/json_configs/simple_profiler_config_parser.json @@ -0,0 +1,6 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": true, + "LocalPath": "/tmp/test" + } +} diff --git a/tests/core/json_configs/string_data_profiler_config_parser.json b/tests/core/json_configs/string_data_profiler_config_parser.json new file mode 100644 index 0000000000..e6f40fe68b --- /dev/null +++ b/tests/core/json_configs/string_data_profiler_config_parser.json @@ -0,0 +1,10 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": "True", + "LocalPath": "/tmp/test", + "RotateMaxFileSizeInBytes": "300", + "RotateFileCloseIntervalInSeconds": "0.5", + "FileOpenFailThreshold": "2", + "DetailedProfilingConfig": "{\"StartStep\": \"2\", \"NumSteps\": \"1\", \"StartTimeInSecSinceEpoch\": \"1594429959.418771\", \"DurationInSeconds\": \"0.1\"}" + } +} diff --git a/tests/core/json_configs/test_pytorch_profiler_config_parser.json b/tests/core/json_configs/test_pytorch_profiler_config_parser.json new file mode 100644 index 0000000000..15d88c3eb4 --- /dev/null +++ b/tests/core/json_configs/test_pytorch_profiler_config_parser.json @@ -0,0 +1,7 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": true, + "LocalPath": "/tmp/test", + "DetailedProfilingConfig": "{\"StartStep\": 2, \"NumSteps\": 2}" + } +} diff --git a/tests/core/json_configs/test_tf2_profiler_config_parser_by_step.json b/tests/core/json_configs/test_tf2_profiler_config_parser_by_step.json new file mode 100644 index 0000000000..15d88c3eb4 --- /dev/null +++ b/tests/core/json_configs/test_tf2_profiler_config_parser_by_step.json @@ -0,0 +1,7 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": true, + "LocalPath": "/tmp/test", + "DetailedProfilingConfig": "{\"StartStep\": 2, \"NumSteps\": 2}" + } +} diff --git a/tests/core/json_configs/test_tf2_profiler_config_parser_by_time.json b/tests/core/json_configs/test_tf2_profiler_config_parser_by_time.json new file mode 100644 index 0000000000..b8faf41633 --- /dev/null +++ b/tests/core/json_configs/test_tf2_profiler_config_parser_by_time.json @@ -0,0 +1,7 @@ +{ + "ProfilingParameters": { + "ProfilerEnabled": true, + "LocalPath": "/tmp/test", + "DetailedProfilingConfig": "{\"DurationInSeconds\": 0.1}" + } +} diff --git a/tests/core/json_configs/user_disabled_profile_config_parser.json b/tests/core/json_configs/user_disabled_profile_config_parser.json new file mode 100644 index 0000000000..fc3da4c720 --- /dev/null +++ b/tests/core/json_configs/user_disabled_profile_config_parser.json @@ -0,0 +1,3 @@ +{ + "ProfilingParameters": {} +} diff --git a/tests/core/test_timeline_writer.py b/tests/core/test_timeline_writer.py new file mode 100644 index 0000000000..fc530d9e23 --- /dev/null +++ b/tests/core/test_timeline_writer.py @@ -0,0 +1,377 @@ +# Standard Library +import calendar +import json +import multiprocessing as mp +import os +import time +from datetime import datetime +from pathlib import Path + +# Third Party +import pytest + +# First Party +from smdebug.core.tfevent.timeline_file_writer import TimelineFileWriter +from smdebug.profiler.profiler_config_parser import ProfilerConfigParser +from smdebug.profiler.profiler_constants import ( + CONVERT_TO_MICROSECS, + DEFAULT_PREFIX, + TRACE_DIRECTORY_FORMAT, +) + + +@pytest.fixture() +def complete_profiler_config_parser(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "complete_profiler_config_parser.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + +@pytest.fixture() +def file_open_fail_profiler_config_parser(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "file_open_fail_profiler_config_parser.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + +@pytest.fixture() +def rotation_profiler_config_parser(config_folder, monkeypatch): + def _choose_config(rotation_policy=None): + nonlocal config_folder + nonlocal monkeypatch + config_path = os.path.join( + config_folder, rotation_policy + "_rotation_profiler_config_parser.json" + ) + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + return _choose_config + + +def test_create_timeline_file(simple_profiler_config_parser, out_dir): + """ + This test is meant to test successful creation of the timeline file according to file path specification. + $ENV_BASE_FOLDER/framework/pevents/$START_TIME_YYMMDDHR/$FILEEVENTSTARTTIMEUTCINEPOCH_ + {$ENV_NODE_ID_4digits0padded}_pythontimeline.json + + It reads backs the file contents to make sure it is in valid JSON format. + """ + assert simple_profiler_config_parser.profiling_enabled + + timeline_writer = TimelineFileWriter(profiler_config_parser=simple_profiler_config_parser) + assert timeline_writer + + for i in range(1, 11): + n = "event" + str(i) + timeline_writer.write_trace_events( + training_phase="FileCreationTest", op_name=n, step_num=i, timestamp=time.time() + ) + + timeline_writer.flush() + timeline_writer.close() + + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob("*.json"): + files.append(path) + + assert len(files) == 1 + + file_ts = files[0].name.split("_")[0] + folder_name = files[0].parent.name + assert folder_name == time.strftime( + TRACE_DIRECTORY_FORMAT, time.gmtime(int(file_ts) / CONVERT_TO_MICROSECS) + ) + assert folder_name == datetime.strptime(folder_name, TRACE_DIRECTORY_FORMAT).strftime( + TRACE_DIRECTORY_FORMAT + ) + + with open(files[0]) as timeline_file: + events_dict = json.load(timeline_file) + + assert events_dict + + +def run(rank, profiler_config_parser): + timeline_writer = TimelineFileWriter(profiler_config_parser=profiler_config_parser) + assert timeline_writer + + for i in range(1, 6): + n = "event" + str(i) + timeline_writer.write_trace_events( + training_phase="MultiProcessTest", + op_name=n, + step_num=0, + worker=os.getpid(), + process_rank=rank, + timestamp=time.time(), + ) + + timeline_writer.flush() + timeline_writer.close() + + +def test_multiprocess_write(simple_profiler_config_parser, out_dir): + """ + This test is meant to test timeline events written multiple processes. Each process or worker, will have its own trace file. + """ + assert simple_profiler_config_parser.profiling_enabled + + cpu_count = mp.cpu_count() + + processes = [] + for rank in range(cpu_count): + p = mp.Process(target=run, args=(rank, simple_profiler_config_parser)) + # We first train the model across `num_processes` processes + p.start() + processes.append(p) + for p in processes: + p.join() + + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob("*.json"): + files.append(path) + + assert len(files) == cpu_count + + event_ctr = 0 + for file_name in files: + with open(file_name) as timeline_file: + events_dict = json.load(timeline_file) + for e in events_dict: + if e["name"].startswith("event"): + event_ctr += 1 + + assert event_ctr == cpu_count * 5 + + +def test_duration_events(simple_profiler_config_parser, out_dir): + """ + This test is meant to test duration events. By default, write_trace_events records complete events. + TODO: Make TimelineWriter automatically calculate duration while recording "E" event + """ + assert simple_profiler_config_parser.profiling_enabled + + timeline_writer = timeline_writer = TimelineFileWriter( + profiler_config_parser=simple_profiler_config_parser + ) + assert timeline_writer + + for i in range(1, 11): + n = "event" + str(i) + timeline_writer.write_trace_events( + training_phase="DurationEventTest", + op_name=n, + step_num=i, + phase="B", + timestamp=time.time(), + ) + timeline_writer.write_trace_events( + training_phase="DurationEventTest", + op_name=n, + step_num=i, + phase="E", + timestamp=time.time(), + ) + + timeline_writer.flush() + timeline_writer.close() + + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob("*.json"): + files.append(path) + + assert len(files) == 1 + + with open(files[0]) as timeline_file: + events_dict = json.load(timeline_file) + + assert events_dict + + +@pytest.mark.slow +@pytest.mark.parametrize("policy", ["file_size", "file_interval"]) +def test_rotation_policy(rotation_profiler_config_parser, policy, out_dir): + """ + This test is meant to test if files are being closed and open correctly according to the 2 rotation policies - + file_size -> close file if it exceeds certain size and open a new file + file_interval -> close file if the file's folder was created before a certain time period and open a new file in a new folder + :param policy: file_size or file_interval + """ + rotation_profiler_config = rotation_profiler_config_parser(policy) + assert rotation_profiler_config.profiling_enabled + + timeline_writer = TimelineFileWriter(profiler_config_parser=rotation_profiler_config) + assert timeline_writer + + for i in range(1, 100): + n = "event" + str(i) + # adding a sleep here to trigger rotation policy + time.sleep(0.05) + timeline_writer.write_trace_events( + training_phase=f"RotationPolicyTest_{policy}", + op_name=n, + step_num=i, + timestamp=time.time(), + ) + + timeline_writer.flush() + timeline_writer.close() + + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob("*.json"): + files.append(path) + + # check if files have been generated + assert files + + # count the number of event JSON strings. This is to ensure all events have been written. + # also check if the timestamp of all events in a file are <= filename timestamp + event_ctr = 0 + start_time_since_epoch = 0 + file_details = [] + for file_name in files: + if policy == "file_size": + file_details.append(os.path.getsize(file_name)) + else: + file_details.append(os.path.getmtime(file_name)) + path = file_name.name.split(DEFAULT_PREFIX) + file_timestamp = int(path[0].split("_")[0]) + num_events_in_file = 0 + with open(file_name) as timeline_file: + events_dict = json.load(timeline_file) + for e in events_dict: + if "args" in e and "start_time_since_epoch_in_micros" in e["args"]: + start_time_since_epoch = int(e["args"]["start_time_since_epoch_in_micros"]) + if "event" in e["name"]: + num_events_in_file += 1 + event_ctr += 1 + assert ( + int(round(e["ts"] + start_time_since_epoch) / CONVERT_TO_MICROSECS) + <= file_timestamp + ) + # if rotation occurs too often, there might be only 1 event per file + # the below assertion checks for this + assert num_events_in_file >= 2 + + if policy == "file_size": + # assuming rotation max file size if 800 bytes, check if all the files are in the + # range +- 60 bytes + assert [pytest.approx(800, 60) == x for x in file_details] + else: + # assuming rotation file close interval is 0.5 seconds, check if the close time + # difference between consecutive files is at least 0.5 seconds + sorted(file_details) + res = [j - i for i, j in zip(file_details[:-1], file_details[1:])] + assert [x >= 0.5 for x in res] + + assert event_ctr == 99 + + +@pytest.mark.parametrize("timezone", ["Europe/Dublin", "Australia/Melbourne", "US/Eastern"]) +def test_utc_timestamp(simple_profiler_config_parser, timezone, out_dir): + """ + This test is meant to set to create files/events in different timezones and check if timeline writer stores + them in UTC. + """ + assert simple_profiler_config_parser.profiling_enabled + + time.tzset() + event_time_in_timezone = time.mktime(time.localtime()) + time_in_utc = event_time_in_utc = calendar.timegm(time.gmtime()) + + timeline_writer = TimelineFileWriter(profiler_config_parser=simple_profiler_config_parser) + assert timeline_writer + + event_times_in_utc = [] + for i in range(1, 3): + event_times_in_utc.append(event_time_in_utc) + timeline_writer.write_trace_events( + training_phase=f"TimestampTest", + op_name="event_in_" + timezone + str(i), + timestamp=event_time_in_timezone, + duration=20, + ) + event_time_in_timezone = time.mktime(time.localtime()) + event_time_in_utc = calendar.timegm(time.gmtime()) + + timeline_writer.flush() + timeline_writer.close() + + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob("*.json"): + files.append(path) + + file_path = files[0] + path = file_path.name.split(DEFAULT_PREFIX) + file_timestamp = int(path[0].split("_")[0]) + + # file timestamp uses end of event + assert (time_in_utc + 20) * CONVERT_TO_MICROSECS == file_timestamp + + start_time_since_epoch = 0 + idx = 0 + for file_name in files: + with open(file_name) as timeline_file: + events_dict = json.load(timeline_file) + for e in events_dict: + if "args" in e and "start_time_since_epoch_in_micros" in e["args"]: + start_time_since_epoch = int(e["args"]["start_time_since_epoch_in_micros"]) + if "event" in e["name"]: + assert ( + e["ts"] + start_time_since_epoch + == event_times_in_utc[idx] * CONVERT_TO_MICROSECS + ) + idx += 1 + + +def test_file_open_fail(file_open_fail_profiler_config_parser): + assert file_open_fail_profiler_config_parser.profiling_enabled + + # writing to an invalid path to trigger file open failure + timeline_writer = TimelineFileWriter( + profiler_config_parser=file_open_fail_profiler_config_parser + ) + assert timeline_writer + + for i in range(1, 5): + n = "event" + str(i) + # Adding a sleep here to slow down event queuing + time.sleep(0.001) + timeline_writer.write_trace_events( + training_phase=f"FileOpenTest", op_name=n, step_num=i, timestamp=time.time() + ) + + timeline_writer.flush() + timeline_writer.close() + + # hacky way to check if the test passes + assert not timeline_writer._worker._healthy + + +def test_events_far_apart(complete_profiler_config_parser, out_dir): + assert complete_profiler_config_parser.profiling_enabled + + timeline_writer = TimelineFileWriter(profiler_config_parser=complete_profiler_config_parser) + assert timeline_writer + + event_time_now = time.time() + event_time_after_2hours = event_time_now + 120 + + timeline_writer.write_trace_events( + training_phase=f"FileOpenTest", op_name="event1", timestamp=event_time_now + ) + time.sleep(2) + timeline_writer.write_trace_events( + training_phase=f"FileOpenTest", op_name="event2", timestamp=event_time_after_2hours + ) + + timeline_writer.flush() + timeline_writer.close() + + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob("*.json"): + files.append(path) + + # rotate by file_size, gives 4 files - 1 per event + # rotate by file_interval, gives 2 files + assert len(files) == 2 diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py index 68c878c603..cabe27535a 100644 --- a/tests/core/test_utils.py +++ b/tests/core/test_utils.py @@ -1,14 +1,21 @@ # Third Party # Standard Library +import os import shutil import tempfile +import time from multiprocessing import Manager, Process from os import makedirs import pytest # First Party -from smdebug.core.access_layer import check_dir_exists +from smdebug.core.access_layer import ( + DEFAULT_GRACETIME_FOR_RULE_STOP_SEC, + ENV_RULE_STOP_SIGNAL_FILENAME, + check_dir_exists, + is_rule_signalled_gracetime_passed, +) from smdebug.core.collection_manager import CollectionManager from smdebug.core.index_reader import ReadIndexFilesCache from smdebug.core.json_config import ( @@ -69,6 +76,53 @@ def test_check_dir_exists_s3(): check_dir_exists("s3://smdebug-testing/resources/exists") +def setup_rule_stop_file(temp_file, time_str, monkeypatch, write=True): + dir = os.path.dirname(temp_file.name) + rel_filename = os.path.relpath(temp_file.name, start=dir) + if write is True: + # write timestamp in temp file + temp_file.write(str(time_str)) + temp_file.flush() + monkeypatch.setenv(ENV_RULE_STOP_SIGNAL_FILENAME, rel_filename) + + +def test_is_rule_signalled_gracetime_not_passed(monkeypatch): + temp_file = tempfile.NamedTemporaryFile(mode="w+") + time_str = str(int(time.time())) + setup_rule_stop_file(temp_file, time_str, monkeypatch) + dir = os.path.dirname(temp_file.name) + assert is_rule_signalled_gracetime_passed(dir) is False + + +def test_is_rule_signalled_gracetime_passed(monkeypatch): + temp_file = tempfile.NamedTemporaryFile(mode="w+") + time_str = str(int(time.time() - 2 * DEFAULT_GRACETIME_FOR_RULE_STOP_SEC)) + setup_rule_stop_file(temp_file, time_str, monkeypatch) + dir = os.path.dirname(temp_file.name) + assert is_rule_signalled_gracetime_passed(dir) is True + + +def test_is_rule_signalled_no_env_var_set(monkeypatch): + assert is_rule_signalled_gracetime_passed("/fake-file") is False + + +def test_is_rule_signalled_no_signal_file(monkeypatch): + temp_file = tempfile.NamedTemporaryFile(mode="w+") + time_str = str(int(time.time() - 2 * DEFAULT_GRACETIME_FOR_RULE_STOP_SEC)) + setup_rule_stop_file(temp_file, time_str, monkeypatch, write=False) + dir = os.path.dirname(temp_file.name) + # env variable is set, remove the file. + temp_file.close() + assert is_rule_signalled_gracetime_passed(dir) is False + + +def test_is_rule_signalled_invalid_gracetime(monkeypatch): + temp_file = tempfile.NamedTemporaryFile(mode="w+") + setup_rule_stop_file(temp_file, "Invalid_time", monkeypatch) + dir = os.path.dirname(temp_file.name) + assert is_rule_signalled_gracetime_passed(dir) is True + + @pytest.mark.skip(reason="It's unclear what this is testing.") def test_check_dir_not_exists(): with pytest.raises(Exception): diff --git a/tests/mxnet/test_hook.py b/tests/mxnet/test_hook.py index 955d1c0c91..9d86392088 100644 --- a/tests/mxnet/test_hook.py +++ b/tests/mxnet/test_hook.py @@ -1,13 +1,17 @@ # Standard Library +import json import os import shutil +import time from datetime import datetime +from pathlib import Path # First Party from smdebug import SaveConfig from smdebug.core.access_layer.utils import has_training_ended from smdebug.core.json_config import CONFIG_FILE_PATH_ENV_STR from smdebug.mxnet.hook import Hook as t_hook +from smdebug.profiler.profiler_constants import DEFAULT_PREFIX # Local from .mnist_gluon_model import run_mnist_gluon_model @@ -51,3 +55,35 @@ def test_hook_from_json_config_full(): hook=hook, num_steps_train=10, num_steps_eval=10, register_to_loss_block=True ) shutil.rmtree(out_dir, True) + + +def test_hook_timeline_file_write(set_up_smprofiler_config_path, out_dir): + """ + This test is meant to test TimelineFileWriter through a MXNet hook. + """ + hook = t_hook(out_dir=out_dir) + + for i in range(1, 11): + n = "event" + str(i) + hook.record_trace_events( + training_phase="MXNet_TimelineFileWriteTest", + op_name=n, + step_num=i, + timestamp=time.time(), + ) + + # need to explicitly close hook for the test here so that the JSON file is written and + # can be read back below. + # In training scripts, this is not necessary as _cleanup will take care of closing the trace file. + hook.close() + + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob("*.json"): + files.append(path) + + assert len(files) == 1 + + with open(files[0]) as timeline_file: + events_dict = json.load(timeline_file) + + assert events_dict diff --git a/tests/profiler/__init__.py b/tests/profiler/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/profiler/core/__init__.py b/tests/profiler/core/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/profiler/core/test_algorithm_metric_readers.py b/tests/profiler/core/test_algorithm_metric_readers.py new file mode 100644 index 0000000000..fe27e4fd9f --- /dev/null +++ b/tests/profiler/core/test_algorithm_metric_readers.py @@ -0,0 +1,84 @@ +# First Party +# Standard Library +import time + +# Third Party +import pytest + +from smdebug.profiler.algorithm_metrics_reader import ( + LocalAlgorithmMetricsReader, + S3AlgorithmMetricsReader, +) +from smdebug.profiler.profiler_constants import CONVERT_TO_MICROSECS +from smdebug.profiler.utils import TimeUnits + + +@pytest.mark.parametrize("use_in_memory_cache", [True, False]) +def test_S3MetricsReader(use_in_memory_cache): + bucket_name = "s3://smdebug-testing/resources/model_timeline_traces" + tt = S3AlgorithmMetricsReader(bucket_name, use_in_memory_cache=use_in_memory_cache) + events = tt.get_events(1590461127873222, 1590461139949971) + print(f"Number of events {len(events)}") + + +@pytest.mark.parametrize("use_in_memory_cache", [True, False]) +def test_LocalMetricsReader( + use_in_memory_cache, tracefolder="./tests/profiler/resources/test_traces" +): + lt = LocalAlgorithmMetricsReader(tracefolder, use_in_memory_cache=use_in_memory_cache) + events = lt.get_events(1589930980, 1589930995, unit=TimeUnits.SECONDS) + print(f"Number of events {len(events)}") + assert len(events) == 4 + + +@pytest.mark.parametrize("use_in_memory_cache", [True, False]) +def test_LocalMetricsReader_Model_timeline( + use_in_memory_cache, tracefolder="./tests/profiler/resources/model_timeline_traces" +): + lt = LocalAlgorithmMetricsReader(tracefolder, use_in_memory_cache=use_in_memory_cache) + events = lt.get_events(1590461127873222, 1590461139949971) + + print(f"Number of events {len(events)}") + assert len(events) == 54 + assert events[0].node_id == "0001" + + +@pytest.mark.parametrize("use_in_memory_cache", [True, False]) +def test_LocalMetricsReader_Horovod_timeline( + use_in_memory_cache, tracefolder="./tests/profiler/resources/horovod_timeline_traces" +): + lt = LocalAlgorithmMetricsReader(tracefolder, use_in_memory_cache=use_in_memory_cache) + events = lt.get_events(1593673051472800, 1593673051473100) + + print(f"Number of events {len(events)}") + assert len(events) == 15 + + +@pytest.mark.parametrize("use_in_memory_cache", [True, False]) +@pytest.mark.parametrize("trace_location", ["local", "s3"]) +def test_MetricsReader_TFProfiler_timeline(use_in_memory_cache, trace_location): + if trace_location == "local": + tracefolder = "./tests/profiler/resources/tfprofiler_timeline_traces" + lt = LocalAlgorithmMetricsReader(tracefolder, use_in_memory_cache=use_in_memory_cache) + elif trace_location == "s3": + bucket_name = "s3://smdebug-testing/resources/tf2_detailed_profile/profiler-output" + lt = S3AlgorithmMetricsReader(bucket_name, use_in_memory_cache=use_in_memory_cache) + else: + return + events = lt.get_events(0, time.time() * CONVERT_TO_MICROSECS) + + print(f"Number of events {len(events)}") + if trace_location == "local": + assert len(events) == 798 + elif trace_location == "s3": + assert len(events) >= 73000 + + +@pytest.mark.parametrize("use_in_memory_cache", [True, False]) +def test_MetricReader_all_files(use_in_memory_cache): + bucket_name = "s3://smdebug-testing/resources/pytorch_traces_with_pyinstru/profiler-output" + lt = S3AlgorithmMetricsReader(bucket_name, use_in_memory_cache=use_in_memory_cache) + + events = lt.get_events(0, time.time() * CONVERT_TO_MICROSECS) + + assert len(events) != 0 diff --git a/tests/profiler/core/test_horovodprofiler_events.py b/tests/profiler/core/test_horovodprofiler_events.py new file mode 100644 index 0000000000..c1044c3928 --- /dev/null +++ b/tests/profiler/core/test_horovodprofiler_events.py @@ -0,0 +1,83 @@ +# Standard Library +import json +import time + +# Third Party +import psutil +import pytest + +# First Party +from smdebug.profiler import HorovodProfilerEvents +from smdebug.profiler.hvd_trace_file_rotation import HvdTraceFileRotation +from smdebug.profiler.profiler_constants import CONVERT_TO_MICROSECS + + +def test_horovodprofiler_events( + trace_file="./tests/profiler/resources/horovod_timeline_traces/framework/pevents/2020070206/1593673051473228_88359-8c859046be41.ant.amazon.com_horovod_timeline.json" +): + trace_json_file = trace_file + print(f"Reading the trace file {trace_json_file}") + t_events = HorovodProfilerEvents() + t_events.read_events_from_file(trace_json_file) + + all_trace_events = t_events.get_all_events() + num_trace_events = len(all_trace_events) + + print(f"Number of events read = {num_trace_events}") + assert num_trace_events == 19 + + completed_event_list = t_events.get_events_within_time_range( + 1593673051473000, 1593673051473228 + ) # microseconds + print( + f"Number of events occurred between 1593673051473000 and 1593673051473228 are {len(completed_event_list)}" + ) + assert len(completed_event_list) == 12 + + start_time_sorted = t_events.get_events_start_time_sorted() + start_time_for_first_event = start_time_sorted[0].start_time + print(f"The first event started at {start_time_for_first_event}") + assert start_time_for_first_event == 1592860696000713 + + end_time_sorted = t_events.get_events_end_time_sorted() + end_time_for_last_event = end_time_sorted[-1].end_time + print(f"The last event ended at {end_time_for_last_event}") + assert end_time_for_last_event == 1593673051473228 + + +def test_steady_clock_to_epoch_time_conversion( + simple_profiler_config_parser, + monkeypatch, + trace_file="./tests/profiler/resources/horovod_timeline_small.json", +): + """ + This test checks if steady clock/ monotonic time to time since epoch conversion + works as expected. This is being done, by converting timestamps from Horovod + event files (that record timestamps according to monotonic clock) to epoch time + and checking if it is within +/- 1 year/month/day of system boot time. + If the conversion was incorrect, the time/year would be much earlier/later than current year. + """ + assert simple_profiler_config_parser.profiling_enabled + + monkeypatch.setenv("HOROVOD_TIMELINE", trace_file) + + hvd_file_reader = HvdTraceFileRotation(simple_profiler_config_parser) + + assert hvd_file_reader.enabled + + boot_time = time.gmtime(psutil.boot_time()) + + with open(trace_file) as hvd_file: + events_dict = json.load(hvd_file) + for e in events_dict: + if "ts" in e: + timestamp_in_us = hvd_file_reader._convert_monotonic_to_epoch_time(e["ts"]) + gmtime = time.gmtime(timestamp_in_us / CONVERT_TO_MICROSECS) + + # assuming that the test is being run the same day as system boot or + # at the maximum one day after system boot. + assert pytest.approx(boot_time.tm_year, 1) == gmtime.tm_year + assert pytest.approx(boot_time.tm_mon, 1) == gmtime.tm_mon + assert pytest.approx(boot_time.tm_mday, 1) == gmtime.tm_mday + + hvd_file_reader.close() diff --git a/tests/profiler/core/test_pandas_frames.py b/tests/profiler/core/test_pandas_frames.py new file mode 100644 index 0000000000..a076217446 --- /dev/null +++ b/tests/profiler/core/test_pandas_frames.py @@ -0,0 +1,278 @@ +# First Party +# Third Party +import pytest + +from smdebug.profiler.analysis.utils.pandas_data_analysis import ( + PandasFrameAnalysis, + Resource, + StatsBy, +) +from smdebug.profiler.analysis.utils.profiler_data_to_pandas import PandasFrame + + +@pytest.mark.parametrize("framework", ["tf2", "pt"]) +def test_pandas_frames(framework): + bucket_name = ( + "s3://smdebug-testing/resources/" + framework + "_detailed_profile/profiler-output" + ) + pf = PandasFrame(bucket_name, scan_interval=50000000000) + system_metrics_df = pf.get_all_system_metrics() + + print(f"Number of rows in system metrics dataframe = {system_metrics_df.shape[0]}") + if framework == "tf2": + assert system_metrics_df.shape[0] == 39392 + if framework == "pt": + assert system_metrics_df.shape[0] == 84768 + print(f"Number of columns in system metrics dataframe = {system_metrics_df.shape[1]}") + if framework == "tf2": + assert system_metrics_df.shape[1] == 7 + if framework == "pt": + assert system_metrics_df.shape[1] == 7 + + pf = PandasFrame(bucket_name, scan_interval=50000000000) + framework_metrics_df = pf.get_all_framework_metrics() + + print(f"Number of rows in framework metrics dataframe = {framework_metrics_df.shape[0]}") + if framework == "tf2": + assert framework_metrics_df.shape[0] == 73984 + if framework == "pt": + assert framework_metrics_df.shape[0] == 154192 + print(f"Number of columns in framework metrics dataframe = {framework_metrics_df.shape[1]}") + if framework == "tf2": + assert framework_metrics_df.shape[1] == 11 + if framework == "pt": + assert framework_metrics_df.shape[1] == 11 + + +@pytest.mark.parametrize("framework", ["tf2", "pt"]) +def test_get_data_by_time(framework): + bucket_name = ( + "s3://smdebug-testing/resources/" + framework + "_detailed_profile/profiler-output" + ) + pf = PandasFrame(bucket_name, scan_interval=50000000000) + if framework == "tf2": + system_metrics_df, framework_metrics_df = pf.get_profiler_data_by_time( + 1596668220000000, 1596678220000000 + ) + assert system_metrics_df.shape[0] == 39392 + if framework == "pt": + system_metrics_df, framework_metrics_df = pf.get_profiler_data_by_time( + 1596493560000000, 1596499560000000 + ) + assert system_metrics_df.shape[0] == 84768 + print(f"Number of rows in system metrics dataframe = {system_metrics_df.shape[0]}") + + print(f"Number of columns in system metrics dataframe = {system_metrics_df.shape[1]}") + assert system_metrics_df.shape[1] == 7 + + print(f"Number of rows in framework metrics dataframe = {framework_metrics_df.shape[0]}") + if framework == "tf2": + assert framework_metrics_df.shape[0] == 73984 + if framework == "pt": + assert framework_metrics_df.shape[0] == 154192 + print(f"Number of columns in framework metrics dataframe = {framework_metrics_df.shape[1]}") + assert framework_metrics_df.shape[1] == 11 + + +@pytest.mark.parametrize("framework", ["tf2", "pt"]) +def test_get_data_by_step(framework): + bucket_name = ( + "s3://smdebug-testing/resources/" + framework + "_detailed_profile/profiler-output" + ) + pf = PandasFrame(bucket_name) + _, framework_metrics_df = pf.get_profiler_data_by_step(2, 3) + + assert not framework_metrics_df.empty + + assert framework_metrics_df.groupby("step").ngroups == 2 + + print(f"Number of rows in framework metrics dataframe = {framework_metrics_df.shape[0]}") + if framework == "tf2": + assert framework_metrics_df.shape[0] == 5 + if framework == "pt": + assert framework_metrics_df.shape[0] == 738 + + print(f"Number of columns in framework metrics dataframe = {framework_metrics_df.shape[1]}") + assert framework_metrics_df.shape[1] == 11 + + +def get_metrics(framework): + bucket_name = ( + "s3://smdebug-testing/resources/" + framework + "_detailed_profile/profiler-output" + ) + pf = PandasFrame(bucket_name, use_in_memory_cache=True) + system_metrics_df, framework_metrics_df = ( + pf.get_all_system_metrics(), + pf.get_all_framework_metrics(), + ) + return system_metrics_df, framework_metrics_df + + +@pytest.fixture(scope="module", autouse=True) +def tf_pandas_frame_analysis(): + return PandasFrameAnalysis(*get_metrics("tf2")) + + +@pytest.fixture(scope="module", autouse=True) +def pt_pandas_frame_analysis(): + return PandasFrameAnalysis(*get_metrics("pt")) + + +@pytest.mark.slow +@pytest.mark.parametrize("framework", ["tf2", "pt"]) +@pytest.mark.parametrize( + "by", [StatsBy.TRAINING_PHASE, StatsBy.FRAMEWORK_METRICS, StatsBy.PROCESS, "step"] +) +def test_get_step_stats(framework, by, tf_pandas_frame_analysis, pt_pandas_frame_analysis): + if framework == "tf2": + pf_analysis = tf_pandas_frame_analysis + else: + pf_analysis = pt_pandas_frame_analysis + + step_stats = pf_analysis.get_step_statistics(by=by) + + if by == "step": + assert step_stats is None + else: + assert not step_stats.empty + assert step_stats.shape[1] == 7 + + if by == "training_phase": + if framework == "tf2": + assert step_stats.shape[0] == 2 + else: + assert step_stats.shape[0] == 1 + elif by == "framework_metric": + if framework == "tf2": + assert step_stats.shape[0] == 111 + else: + assert step_stats.shape[0] == 207 + elif by == "process": + if framework == "tf2": + assert step_stats.shape[0] == 6 + else: + assert step_stats.shape[0] == 7 + + +@pytest.mark.slow +@pytest.mark.parametrize("framework", ["tf2", "pt"]) +@pytest.mark.parametrize("phase", [["Step:ModeKeys.TRAIN"], None]) +def test_get_util_stats_by_training_phase( + framework, phase, tf_pandas_frame_analysis, pt_pandas_frame_analysis +): + if framework == "tf2": + pf_analysis = tf_pandas_frame_analysis + else: + if phase and phase[0] == "Step:ModeKeys.TRAIN": + phase = ["Step:ModeKeys.GLOBAL"] + pf_analysis = pt_pandas_frame_analysis + + util_stats = pf_analysis.get_utilization_stats(phase=phase, by=StatsBy.TRAINING_PHASE) + + assert not util_stats.empty + if phase is None: + assert util_stats.shape[0] <= 8 + else: + assert util_stats.shape[0] <= 8 + assert util_stats.shape[1] == 9 + + assert all(util_stats["Resource"].unique() == ["cpu", "gpu"]) + + +@pytest.mark.slow +@pytest.mark.parametrize("framework", ["tf2", "pt"]) +@pytest.mark.parametrize("resource", [None, Resource.CPU, [Resource.CPU, Resource.GPU], "cpu"]) +@pytest.mark.parametrize("by", [None, "step"]) +def test_get_util_stats( + framework, resource, by, tf_pandas_frame_analysis, pt_pandas_frame_analysis +): + if framework == "tf2": + pf_analysis = tf_pandas_frame_analysis + else: + pf_analysis = pt_pandas_frame_analysis + + util_stats = pf_analysis.get_utilization_stats(resource=resource, by=by) + + if by == "step" or resource == "cpu": + assert util_stats is None + else: + assert not util_stats.empty + + +@pytest.mark.slow +@pytest.mark.parametrize("framework", ["tf2", "pt"]) +@pytest.mark.parametrize("device", ["cpu", Resource.CPU, Resource.GPU]) +@pytest.mark.parametrize( + "ranges", [None, [(0, 10), (10, 20), (30, 80), (80, 100)], [(30,)], [], ((0, 10), (10, 90))] +) +def test_get_device_usage_stats( + framework, device, ranges, tf_pandas_frame_analysis, pt_pandas_frame_analysis +): + if framework == "tf2": + pf_analysis = tf_pandas_frame_analysis + else: + pf_analysis = pt_pandas_frame_analysis + + usage_stats = pf_analysis.get_device_usage_stats(device=device, utilization_ranges=ranges) + + if ranges in [[(30,)], [], ((0, 10), (10, 90))] or device == "cpu": + assert usage_stats.empty + else: + assert not usage_stats.empty + + if ranges is None: + assert usage_stats.shape[1] <= 3 + 3 + else: + assert usage_stats.shape[1] <= 3 + len(ranges) + + +@pytest.mark.slow +@pytest.mark.parametrize("framework", ["tf2", "pt"]) +@pytest.mark.parametrize( + "phase", + [ + ["Step:ModeKeys.TRAIN"], + "Step:ModeKeys.TRAIN", + ["Step:ModeKeys.GLOBAL"], + ("Step:ModeKeys.GLOBAL"), + ], +) +def test_get_training_phase_intervals( + framework, phase, tf_pandas_frame_analysis, pt_pandas_frame_analysis +): + if framework == "tf2": + valid_phase = ["Step:ModeKeys.TRAIN"] + pf_analysis = tf_pandas_frame_analysis + else: + valid_phase = ["Step:ModeKeys.GLOBAL"] + pf_analysis = pt_pandas_frame_analysis + + interval_stats = pf_analysis.get_training_phase_intervals(phase=phase) + + if isinstance(phase, str): + phase = [phase] + if not isinstance(phase, list) or phase != valid_phase: + print(not isinstance(phase, (str, list))) + print(phase != valid_phase, phase, valid_phase) + print((isinstance(phase, str) and [phase] != valid_phase)) + assert interval_stats is None + else: + assert not interval_stats.empty + assert interval_stats.shape[1] == 3 + + if framework == "tf2": + assert interval_stats.shape[0] == 11251 + else: + assert interval_stats.shape[0] == 785 + + +@pytest.mark.slow +@pytest.mark.parametrize("framework", ["tf2"]) +def test_get_jobs_stats(framework, tf_pandas_frame_analysis, pt_pandas_frame_analysis): + if framework == "tf2": + pf_analysis = tf_pandas_frame_analysis + else: + pf_analysis = pt_pandas_frame_analysis + + job_stats = pf_analysis.get_job_statistics() + assert job_stats is not None diff --git a/tests/profiler/core/test_profiler_config_parser.py b/tests/profiler/core/test_profiler_config_parser.py new file mode 100644 index 0000000000..7eb28e5090 --- /dev/null +++ b/tests/profiler/core/test_profiler_config_parser.py @@ -0,0 +1,512 @@ +# Standard Library +import json +import os +import time + +# Third Party +import pytest +from tests.profiler.resources.profiler_config_parser_utils import ( + current_step, + current_time, + dataloader_test_cases, + detailed_profiling_test_cases, + python_profiling_test_cases, + smdataparallel_profiling_test_cases, +) + +# First Party +from smdebug.profiler.profiler_config_parser import MetricsCategory, ProfilerConfigParser +from smdebug.profiler.profiler_constants import ( + CLOSE_FILE_INTERVAL_DEFAULT, + FILE_OPEN_FAIL_THRESHOLD_DEFAULT, + MAX_FILE_SIZE_DEFAULT, + PROFILING_NUM_STEPS_DEFAULT, +) + + +@pytest.fixture +def detailed_profiler_config_path(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "detailed_profiler_config.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + yield config_path + if os.path.isfile(config_path): + os.remove(config_path) + + +@pytest.fixture +def dataloader_profiler_config_path(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "dataloader_profiler_config.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + yield config_path + if os.path.isfile(config_path): + os.remove(config_path) + + +@pytest.fixture +def python_profiler_config_path(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "python_profiler_config.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + yield config_path + if os.path.isfile(config_path): + os.remove(config_path) + + +@pytest.fixture +def smdataparallel_profiler_config_path(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "smdataparallel_profiler_config.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + yield config_path + if os.path.isfile(config_path): + os.remove(config_path) + + +@pytest.fixture +def missing_config_profiler_config_parser(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "missing_profile_config_parser.json") # doesn't exist + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + +@pytest.fixture +def invalid_config_profiler_config_parser(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "invalid_profile_config_parser.json") # invalid JSON + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + +@pytest.fixture +def user_disabled_profiler_config_parser(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "user_disabled_profile_config_parser.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + +@pytest.fixture +def string_data_profiler_config_parser(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "string_data_profiler_config_parser.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + +@pytest.fixture +def invalid_string_data_profiler_config_parser(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "invalid_string_data_profiler_config_parser.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + +@pytest.fixture +def case_insensitive_profiler_config_parser(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "case_insensitive_profiler_config_parser.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + +@pytest.fixture +def old_step_profiler_config_parser_path(config_folder): + return os.path.join(config_folder, "old_step_profiler_config_parser.json") + + +@pytest.fixture +def new_step_profiler_config_parser_path(config_folder): + return os.path.join(config_folder, "new_step_profiler_config_parser.json") + + +@pytest.fixture +def old_time_profiler_config_parser_path(config_folder): + return os.path.join(config_folder, "old_time_profiler_config_parser.json") + + +@pytest.fixture +def new_time_profiler_config_parser_path(config_folder): + return os.path.join(config_folder, "new_time_profiler_config_parser.json") + + +def _convert_to_string(item): + return '"{0}"'.format(item) if isinstance(item, str) else item + + +def _convert_key_and_value(key, value): + return "{0}: {1}, ".format(_convert_to_string(key), _convert_to_string(value)) + + +@pytest.mark.parametrize("test_case", detailed_profiling_test_cases) +def test_detailed_profiling_ranges(detailed_profiler_config_path, test_case): + profiling_parameters, expected_enabled, expected_can_save, expected_values = test_case + start_step, num_steps, start_time, duration = profiling_parameters + detailed_profiler_config = "{" + if start_step: + detailed_profiler_config += _convert_key_and_value("StartStep", start_step) + if num_steps: + detailed_profiler_config += _convert_key_and_value("NumSteps", num_steps) + if start_time: + detailed_profiler_config += _convert_key_and_value("StartTimeInSecSinceEpoch", start_time) + if duration: + detailed_profiler_config += _convert_key_and_value("DurationInSeconds", duration) + detailed_profiler_config += "}" + + full_config = { + "ProfilingParameters": { + "ProfilerEnabled": True, + "DetailedProfilingConfig": detailed_profiler_config, + } + } + + with open(detailed_profiler_config_path, "w") as f: + json.dump(full_config, f) + + profiler_config_parser = ProfilerConfigParser() + assert profiler_config_parser.profiling_enabled + + detailed_profiling_config = profiler_config_parser.config.detailed_profiling_config + assert detailed_profiling_config.is_enabled() == expected_enabled + assert ( + profiler_config_parser.should_save_metrics( + MetricsCategory.DETAILED_PROFILING, current_step, current_time=current_time + ) + == expected_can_save + ) + + expected_start_step, expected_end_step, expected_start_time, expected_end_time = expected_values + assert detailed_profiling_config.start_step == expected_start_step + assert detailed_profiling_config.end_step == expected_end_step + assert detailed_profiling_config.start_time_in_sec == expected_start_time + assert detailed_profiling_config.end_time == expected_end_time + + +@pytest.mark.parametrize("test_case", dataloader_test_cases) +def test_dataloader_profiling_ranges(detailed_profiler_config_path, test_case): + profiling_parameters, expected_enabled, expected_can_save, expected_values = test_case + start_step, metrics_regex, metrics_name = profiling_parameters + dataloader_config = "{" + if start_step: + dataloader_config += _convert_key_and_value("StartStep", start_step) + if metrics_regex: + dataloader_config += _convert_key_and_value("MetricsRegex", metrics_regex) + dataloader_config += "}" + + full_config = { + "ProfilingParameters": { + "ProfilerEnabled": True, + "DataloaderProfilingConfig": dataloader_config, + } + } + + with open(detailed_profiler_config_path, "w") as f: + json.dump(full_config, f) + + profiler_config_parser = ProfilerConfigParser() + assert profiler_config_parser.profiling_enabled + + dataloader_profiling_config = profiler_config_parser.config.dataloader_profiling_config + assert dataloader_profiling_config.is_enabled() == expected_enabled + assert ( + profiler_config_parser.should_save_metrics( + MetricsCategory.DATALOADER_PROFILING, current_step, metrics_name=metrics_name + ) + == expected_can_save + ) + + expected_start_step, expected_end_step, expected_metrics_regex = expected_values + assert dataloader_profiling_config.start_step == expected_start_step + assert dataloader_profiling_config.end_step == expected_end_step + assert dataloader_profiling_config.metrics_regex == expected_metrics_regex + + +@pytest.mark.parametrize("test_case", python_profiling_test_cases) +def test_python_profiling_ranges(python_profiler_config_path, test_case): + profiling_parameters, expected_enabled, expected_can_save, expected_values = test_case + start_step, num_steps, profiler_name, cprofile_timer = profiling_parameters + python_profiler_config = "{" + if start_step is not None: + python_profiler_config += _convert_key_and_value("StartStep", start_step) + if num_steps is not None: + python_profiler_config += _convert_key_and_value("NumSteps", num_steps) + if profiler_name is not None: + python_profiler_config += _convert_key_and_value("ProfilerName", profiler_name) + if cprofile_timer is not None: + python_profiler_config += _convert_key_and_value("cProfileTimer", cprofile_timer) + python_profiler_config += "}" + + full_config = { + "ProfilingParameters": { + "ProfilerEnabled": True, + "PythonProfilingConfig": python_profiler_config, + } + } + + with open(python_profiler_config_path, "w") as f: + json.dump(full_config, f) + + profiler_config_parser = ProfilerConfigParser() + assert profiler_config_parser.profiling_enabled + + python_profiling_config = profiler_config_parser.config.python_profiling_config + assert python_profiling_config.is_enabled() == expected_enabled + assert ( + profiler_config_parser.should_save_metrics(MetricsCategory.PYTHON_PROFILING, current_step) + == expected_can_save + ) + + expected_start_step, expected_end_step, expected_profiler_name, expected_cprofile_timer = ( + expected_values + ) + assert python_profiling_config.start_step == expected_start_step + assert python_profiling_config.end_step == expected_end_step + assert python_profiling_config.profiler_name == expected_profiler_name + assert python_profiling_config.cprofile_timer == expected_cprofile_timer + + +@pytest.mark.parametrize("test_case", smdataparallel_profiling_test_cases) +def test_smdataparallel_profiling_ranges(smdataparallel_profiler_config_path, test_case): + profiling_parameters, expected_enabled, expected_can_save, expected_values = test_case + start_step, num_steps = profiling_parameters + + smdataparallel_profiler_config = "{" + if start_step: + smdataparallel_profiler_config += _convert_key_and_value("StartStep", start_step) + if num_steps: + smdataparallel_profiler_config += _convert_key_and_value("NumSteps", num_steps) + smdataparallel_profiler_config += "}" + + full_config = { + "ProfilingParameters": { + "ProfilerEnabled": True, + "SMDataparallelProfilingConfig": smdataparallel_profiler_config, + } + } + + with open(smdataparallel_profiler_config_path, "w") as f: + json.dump(full_config, f) + + profiler_config_parser = ProfilerConfigParser() + assert profiler_config_parser.profiling_enabled + + smdataparallel_profiling_config = profiler_config_parser.config.smdataparallel_profiling_config + assert smdataparallel_profiling_config.is_enabled() == expected_enabled + assert ( + profiler_config_parser.should_save_metrics( + MetricsCategory.SMDATAPARALLEL_PROFILING, current_step, current_time=current_time + ) + == expected_can_save + ) + + expected_start_step, expected_end_step = expected_values + assert smdataparallel_profiling_config.start_step == expected_start_step + assert smdataparallel_profiling_config.end_step == expected_end_step + + +def test_disabled_profiler( + missing_config_profiler_config_parser, + invalid_config_profiler_config_parser, + user_disabled_profiler_config_parser, +): + """ + This test is meant to test that a missing config JSON or invalid config JSON or the user setting `ProfilerEnabled` + to `false` will disable the profiler. + """ + assert not missing_config_profiler_config_parser.profiling_enabled + assert not invalid_config_profiler_config_parser.profiling_enabled + assert not user_disabled_profiler_config_parser.profiling_enabled + + +def test_default_trace_file_values(simple_profiler_config_parser): + """ + This test is meant to test setting default trace file values when the config is present. + """ + assert simple_profiler_config_parser.profiling_enabled + + trace_file_config = simple_profiler_config_parser.config.trace_file + assert trace_file_config.file_open_fail_threshold == FILE_OPEN_FAIL_THRESHOLD_DEFAULT + + rotation_policy = trace_file_config.rotation_policy + assert rotation_policy.file_max_size == MAX_FILE_SIZE_DEFAULT + assert rotation_policy.file_close_interval == CLOSE_FILE_INTERVAL_DEFAULT + + +def test_string_data_in_config(string_data_profiler_config_parser): + """ + This test is meant to test that the profiler config parser can handle string data + and typecast to appropriate types before use. + """ + assert string_data_profiler_config_parser.profiling_enabled + + assert isinstance( + string_data_profiler_config_parser.config.trace_file.rotation_policy.file_max_size, int + ) + assert isinstance( + string_data_profiler_config_parser.config.trace_file.rotation_policy.file_close_interval, + float, + ) + assert isinstance( + string_data_profiler_config_parser.config.trace_file.file_open_fail_threshold, int + ) + + assert isinstance( + string_data_profiler_config_parser.config.detailed_profiling_config.start_step, int + ) + assert isinstance( + string_data_profiler_config_parser.config.detailed_profiling_config.num_steps, int + ) + assert isinstance( + string_data_profiler_config_parser.config.detailed_profiling_config.start_time_in_sec, float + ) + assert isinstance( + string_data_profiler_config_parser.config.detailed_profiling_config.duration_in_sec, float + ) + + +def test_invalid_string_data_in_config(invalid_string_data_profiler_config_parser): + """ + This test is meant to test that the profiler config parser can handle invalid string data + and fallback gracefully. + """ + # Profiler is enabled even if data is invalid + assert invalid_string_data_profiler_config_parser.profiling_enabled + + # Fallback to default values for profiling parameters + assert ( + invalid_string_data_profiler_config_parser.config.trace_file.rotation_policy.file_max_size + == MAX_FILE_SIZE_DEFAULT + ) + assert ( + invalid_string_data_profiler_config_parser.config.trace_file.rotation_policy.file_close_interval + == CLOSE_FILE_INTERVAL_DEFAULT + ) + assert ( + invalid_string_data_profiler_config_parser.config.trace_file.file_open_fail_threshold + == FILE_OPEN_FAIL_THRESHOLD_DEFAULT + ) + + # Disable detailed profiling config if any of the fields are invalid + assert ( + not invalid_string_data_profiler_config_parser.config.detailed_profiling_config.is_enabled() + ) + + assert ( + not invalid_string_data_profiler_config_parser.config.detailed_profiling_config.start_step + ) + assert not invalid_string_data_profiler_config_parser.config.detailed_profiling_config.num_steps + assert ( + not invalid_string_data_profiler_config_parser.config.detailed_profiling_config.start_time_in_sec + ) + assert ( + not invalid_string_data_profiler_config_parser.config.detailed_profiling_config.duration_in_sec + ) + + +def test_case_insensitive_profiler_config_parser(case_insensitive_profiler_config_parser): + """ + This test is meant to test that the keys in the profiler config JSON are case insensitive. In other words, + the profiler config parser can successfully parse the values from the config even if the case of the key is not + camel case. + """ + assert case_insensitive_profiler_config_parser.profiling_enabled + assert ( + case_insensitive_profiler_config_parser.config.trace_file.rotation_policy.file_max_size + == 100 + ) + assert ( + case_insensitive_profiler_config_parser.config.trace_file.rotation_policy.file_close_interval + == 1 + ) + assert case_insensitive_profiler_config_parser.config.trace_file.file_open_fail_threshold == 5 + + assert case_insensitive_profiler_config_parser.config.detailed_profiling_config.is_enabled() + assert case_insensitive_profiler_config_parser.config.detailed_profiling_config.start_step == 2 + assert case_insensitive_profiler_config_parser.config.detailed_profiling_config.num_steps == 3 + + +def test_update_step_profiler_config_parser( + monkeypatch, old_step_profiler_config_parser_path, new_step_profiler_config_parser_path +): + """ + This test is meant to test two behaviors when profiler config parser dynamically reloads a config with step fields: + - Reloading the config when the JSON hasn't changed will not reload the step fields (this is important when the + JSON does not have specified step parameters, for example). + - Reloading the config when the JSON has changed will reload the step fields in the new JSON. + """ + # sanity check that the parser first parses the range fields as is. + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", old_step_profiler_config_parser_path) + profiler_config_parser = ProfilerConfigParser() + assert profiler_config_parser.profiling_enabled + assert profiler_config_parser.config.detailed_profiling_config.is_enabled() + assert profiler_config_parser.config.detailed_profiling_config.start_step is None + assert profiler_config_parser.config.detailed_profiling_config.num_steps == 2 + + # sanity check that calling should save metrics will replace unspecified range fields, and leave the rest as is. + profiler_config_parser.should_save_metrics(MetricsCategory.DETAILED_PROFILING, 5) + assert profiler_config_parser.config.detailed_profiling_config.start_step == 5 + assert profiler_config_parser.config.detailed_profiling_config.num_steps == 2 + + # check that reloading the config when it hasn't changed won't change the config fields. + profiler_config_parser.load_config() + assert profiler_config_parser.config.detailed_profiling_config.start_step == 5 + assert profiler_config_parser.config.detailed_profiling_config.num_steps == 2 + + # check that reloading the config when it has changed will update the config fields. + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", new_step_profiler_config_parser_path) + profiler_config_parser.load_config() + assert profiler_config_parser.profiling_enabled + assert profiler_config_parser.config.detailed_profiling_config.is_enabled() + assert profiler_config_parser.config.detailed_profiling_config.start_step == 10 + assert profiler_config_parser.config.detailed_profiling_config.num_steps == 5 + + +def test_update_time_profiler_config_parser( + monkeypatch, old_time_profiler_config_parser_path, new_time_profiler_config_parser_path +): + """ + This test is meant to test two behaviors when profiler config parser dynamically reloads a config with time fields: + - Reloading the config when the JSON hasn't changed will not reload the time fields (this is important when the + JSON does not have specified time parameters, for example). + - Reloading the config when the JSON has changed will reload the time fields in the new JSON. + """ + # sanity check that the parser first parses the range fields as is. + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", old_time_profiler_config_parser_path) + profiler_config_parser = ProfilerConfigParser() + assert profiler_config_parser.profiling_enabled + assert profiler_config_parser.config.detailed_profiling_config.is_enabled() + assert profiler_config_parser.config.detailed_profiling_config.start_time_in_sec is None + assert profiler_config_parser.config.detailed_profiling_config.duration_in_sec == 0.1 + + # sanity check that calling should save metrics will replace unspecified range fields, and leave the rest as is. + timestamp1 = int(time.time()) + profiler_config_parser.should_save_metrics( + MetricsCategory.DETAILED_PROFILING, 5, current_time=timestamp1 + ) + assert profiler_config_parser.config.detailed_profiling_config.start_time_in_sec == timestamp1 + assert profiler_config_parser.config.detailed_profiling_config.duration_in_sec == 0.1 + + # check that reloading the config when it hasn't changed won't change the config fields. + profiler_config_parser.load_config() + assert profiler_config_parser.config.detailed_profiling_config.start_time_in_sec == timestamp1 + assert profiler_config_parser.config.detailed_profiling_config.duration_in_sec == 0.1 + + # check that reloading the config when it has changed will update the config fields. + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", new_time_profiler_config_parser_path) + profiler_config_parser.load_config() + assert profiler_config_parser.profiling_enabled + assert profiler_config_parser.config.detailed_profiling_config.is_enabled() + assert profiler_config_parser.config.detailed_profiling_config.start_time_in_sec == 1700000000 + assert profiler_config_parser.config.detailed_profiling_config.duration_in_sec == 5 + + +def test_update_disabled_profiler_config_parser( + monkeypatch, user_disabled_profiler_config_parser, new_step_profiler_config_parser_path +): + """ + This test is meant to test that reloading the config from a disabled config to an enabled config will actually + shift the parser from having profiling disabled to profiling enabled. + """ + # sanity check that the disabled parser has profiling enabled set to False. + profiler_config_parser = user_disabled_profiler_config_parser + assert not profiler_config_parser.profiling_enabled + + # check that reloading the config when it has changed will update the config fields. + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", new_step_profiler_config_parser_path) + profiler_config_parser.load_config() + assert profiler_config_parser.profiling_enabled + assert profiler_config_parser.config.detailed_profiling_config.is_enabled() + assert profiler_config_parser.config.detailed_profiling_config.start_step == 10 + assert profiler_config_parser.config.detailed_profiling_config.num_steps == 5 diff --git a/tests/profiler/core/test_python_profiler.py b/tests/profiler/core/test_python_profiler.py new file mode 100644 index 0000000000..6ee652cfa0 --- /dev/null +++ b/tests/profiler/core/test_python_profiler.py @@ -0,0 +1,388 @@ +# Standard Library +import json +import os +import pstats +import shutil +import time +from multiprocessing.pool import ThreadPool + +# Third Party +import boto3 +import pandas as pd +import pytest + +# First Party +from smdebug.core.access_layer.utils import is_s3 +from smdebug.profiler.analysis.python_profile_analysis import PyinstrumentAnalysis, cProfileAnalysis +from smdebug.profiler.profiler_constants import ( + CONVERT_TO_MICROSECS, + CPROFILE_NAME, + CPROFILE_STATS_FILENAME, + PYINSTRUMENT_HTML_FILENAME, + PYINSTRUMENT_JSON_FILENAME, + PYINSTRUMENT_NAME, +) +from smdebug.profiler.python_profile_utils import PythonProfileModes, StepPhase +from smdebug.profiler.python_profiler import ( + PyinstrumentPythonProfiler, + cProfilePythonProfiler, + cProfileTimer, +) + + +@pytest.fixture +def test_framework(): + return "test-framework" + + +@pytest.fixture() +def cprofile_python_profiler(out_dir, test_framework): + return cProfilePythonProfiler(out_dir, test_framework, cProfileTimer.TOTAL_TIME) + + +@pytest.fixture() +def pyinstrument_python_profiler(out_dir, test_framework): + return PyinstrumentPythonProfiler(out_dir, test_framework) + + +@pytest.fixture() +def framework_dir(out_dir, test_framework): + return "{0}/framework/{1}".format(out_dir, test_framework) + + +@pytest.fixture(autouse=True) +def reset_python_profiler_dir(framework_dir): + shutil.rmtree(framework_dir, ignore_errors=True) + + +@pytest.fixture(scope="session") +def bucket_prefix(): + return f"s3://smdebug-testing/resources/python_profile/{int(time.time())}" + + +def pre_step_zero_function(): + time.sleep( + 0.0011 + ) # stall long enough to be recorded by pyinstrument, which records every 0.001 seconds + + +def start_end_step_function(): + time.sleep( + 0.0011 + ) # stall long enough to be recorded by pyinstrument, which records every 0.001 seconds + + +def end_start_step_function(): + time.sleep( + 0.0011 + ) # stall long enough to be recorded by pyinstrument, which records every 0.001 seconds + + +def between_modes_function(): + time.sleep( + 0.0011 + ) # stall long enough to be recorded by pyinstrument, which records every 0.001 seconds + + +def eval_function(): + time.sleep( + 0.0011 + ) # stall long enough to be recorded by pyinstrument, which records every 0.001 seconds + + +def post_hook_close_function(): + time.sleep( + 0.0011 + ) # stall long enough to be recorded by pyinstrument, which records every 0.001 seconds + + +def time_function(): + time.sleep( + 0.0011 + ) # stall long enough to be recorded by pyinstrument, which records every 0.001 seconds + + +def _upload_s3_folder(bucket, key, folder): + s3_client = boto3.client("s3") + filenames = [] + for root, _, files in os.walk(folder): + for file in files: + node_id = os.path.basename(os.path.dirname(root)) + stats_dir = os.path.basename(root) + full_key = os.path.join(key, node_id, stats_dir, file) + filenames.append((os.path.join(root, file), bucket, full_key)) + + def upload_files(args): + s3_client.upload_file(*args) + + pool = ThreadPool(processes=10) + pool.map(upload_files, filenames) + + +def _validate_analysis(profiler_name, stats, expected_functions): + function_names = [ + pre_step_zero_function.__name__, + start_end_step_function.__name__, + end_start_step_function.__name__, + between_modes_function.__name__, + eval_function.__name__, + post_hook_close_function.__name__, + time_function.__name__, + ] + + assert stats is not None, "No stats found!" + + for analysis_function in function_names: + if profiler_name == CPROFILE_NAME: + function_stats_list = stats.function_stats_list + assert len(function_stats_list) > 0 + + if analysis_function in expected_functions: + assert any( + [analysis_function in stat.function_name for stat in function_stats_list] + ), f"{analysis_function} should be found in function stats!" + else: + assert all( + [analysis_function not in stat.function_name for stat in function_stats_list] + ), f"{analysis_function} should not be found in function stats!" + else: + assert len(stats) == 1 + actual_functions = map( + lambda x: x["function"], stats[0].json_stats["root_frame"]["children"] + ) + assert set(actual_functions) == set(expected_functions) + + +@pytest.mark.parametrize("use_pyinstrument", [False, True]) +@pytest.mark.parametrize("steps", [(1, 2), (1, 5)]) +def test_python_profiling( + use_pyinstrument, cprofile_python_profiler, pyinstrument_python_profiler, framework_dir, steps +): + if use_pyinstrument: + python_profiler = pyinstrument_python_profiler + profiler_name = PYINSTRUMENT_NAME + allowed_files = [PYINSTRUMENT_JSON_FILENAME, PYINSTRUMENT_HTML_FILENAME] + else: + python_profiler = cprofile_python_profiler + profiler_name = CPROFILE_NAME + allowed_files = [CPROFILE_STATS_FILENAME] + + python_stats_dir = os.path.join(framework_dir, profiler_name) + + start_step, end_step = steps + current_step = start_step + + while current_step < end_step: + python_profiler.start_profiling(StepPhase.STEP_START, start_step=current_step) + assert python_profiler._start_step == current_step + assert python_profiler._start_phase == StepPhase.STEP_START + python_profiler.stop_profiling(StepPhase.STEP_END, current_step) + current_step += 1 + + # Test that directory and corresponding files exist. + assert os.path.isdir(python_stats_dir) + + for node_id in os.listdir(python_stats_dir): + node_dir_path = os.path.join(python_stats_dir, node_id) + stats_dirs = os.listdir(node_dir_path) + assert len(stats_dirs) == (end_step - start_step) + + for stats_dir in stats_dirs: + # Validate that the expected files are in the stats dir + stats_dir_path = os.path.join(node_dir_path, stats_dir) + stats_files = os.listdir(stats_dir_path) + assert set(stats_files) == set(allowed_files) + + # Validate the actual stats files + for stats_file in stats_files: + stats_path = os.path.join(stats_dir_path, stats_file) + if stats_file == CPROFILE_STATS_FILENAME: + assert pstats.Stats(stats_path) + elif stats_file == PYINSTRUMENT_JSON_FILENAME: + with open(stats_path, "r") as f: + assert json.load(f) + + +@pytest.mark.parametrize("use_pyinstrument", [False, True]) +@pytest.mark.parametrize("s3", [False, True]) +def test_python_analysis( + use_pyinstrument, + cprofile_python_profiler, + pyinstrument_python_profiler, + framework_dir, + test_framework, + bucket_prefix, + s3, +): + """ + This test is meant to test that the cProfile/pyinstrument analysis retrieves the correct step's stats based on the + specified interval. Stats are either retrieved from s3 or generated manually through python profiling. + """ + if use_pyinstrument: + python_profiler = pyinstrument_python_profiler + analysis_class = PyinstrumentAnalysis + profiler_name = PYINSTRUMENT_NAME + num_expected_files = 14 + else: + python_profiler = cprofile_python_profiler + analysis_class = cProfileAnalysis + profiler_name = CPROFILE_NAME + num_expected_files = 7 + + python_stats_dir = os.path.join(framework_dir, profiler_name) + + if s3: + # Fetch stats from s3 + os.makedirs(python_stats_dir) + python_profile_analysis = analysis_class( + local_profile_dir=python_stats_dir, s3_path=bucket_prefix + ) + else: + # Do analysis and use those stats. + + # pre_step_zero_function is called in between the start of the script and the start of first step of TRAIN. + python_profiler.start_profiling(StepPhase.START) + pre_step_zero_function() + python_profiler.stop_profiling( + StepPhase.STEP_START, end_mode=PythonProfileModes.TRAIN, end_step=1 + ) + + # start_end_step_function is called in between the start and end of first step of TRAIN. + python_profiler.start_profiling( + StepPhase.STEP_START, start_mode=PythonProfileModes.TRAIN, start_step=1 + ) + start_end_step_function() + python_profiler.stop_profiling( + StepPhase.STEP_END, end_mode=PythonProfileModes.TRAIN, end_step=1 + ) + + # end_start_step_function is called in between the end of first step and the start of second step of TRAIN. + python_profiler.start_profiling( + StepPhase.STEP_END, start_mode=PythonProfileModes.TRAIN, start_step=1 + ) + end_start_step_function() + python_profiler.stop_profiling( + StepPhase.STEP_START, end_mode=PythonProfileModes.TRAIN, end_step=2 + ) + + # train_and_eval function is called in between the TRAIN and EVAL modes. + python_profiler.start_profiling( + StepPhase.STEP_END, start_mode=PythonProfileModes.TRAIN, start_step=1 + ) + between_modes_function() + python_profiler.stop_profiling( + StepPhase.STEP_START, end_mode=PythonProfileModes.EVAL, end_step=1 + ) + + # eval function is called in between the start and end of first step of EVAL. + python_profiler.start_profiling( + StepPhase.STEP_START, start_mode=PythonProfileModes.EVAL, start_step=1 + ) + eval_function() + python_profiler.stop_profiling( + StepPhase.STEP_END, end_mode=PythonProfileModes.EVAL, end_step=1 + ) + + # post_hook_close_function is called in between the end of the last step of EVAL and the end of the script. + python_profiler.start_profiling( + StepPhase.STEP_END, start_mode=PythonProfileModes.EVAL, start_step=1 + ) + post_hook_close_function() + python_profiler.stop_profiling(StepPhase.END) + + # time function is called in between start and end of second step of TRAIN. + # NOTE: This needs to be profiled last for tests to pass. + python_profiler.start_profiling( + StepPhase.STEP_START, start_mode=PythonProfileModes.TRAIN, start_step=2 + ) + time_function() + python_profiler.stop_profiling( + StepPhase.STEP_END, end_mode=PythonProfileModes.TRAIN, end_step=2 + ) + + python_profile_analysis = analysis_class(local_profile_dir=python_stats_dir) + _, bucket, prefix = is_s3(bucket_prefix) + key = os.path.join(prefix, "framework", test_framework, profiler_name) + _upload_s3_folder(bucket, key, python_stats_dir) + + python_profile_stats_df = python_profile_analysis.list_profile_stats() + assert isinstance(python_profile_stats_df, pd.DataFrame) + assert python_profile_stats_df.shape[0] == num_expected_files + + # Test that pre_step_zero_function call is recorded in received stats, but not the other functions. + stats = python_profile_analysis.fetch_pre_step_zero_profile_stats(refresh_stats=False) + _validate_analysis(profiler_name, stats, [pre_step_zero_function.__name__]) + + # Test that start_end_step_function call is recorded in received stats, but not the other functions. + stats = python_profile_analysis.fetch_profile_stats_by_step(1, refresh_stats=False) + _validate_analysis(profiler_name, stats, [start_end_step_function.__name__]) + + # Test that end_start_step_function call is recorded in received stats, but not the other functions. + stats = python_profile_analysis.fetch_profile_stats_by_step( + 1, + end_step=2, + start_phase=StepPhase.STEP_END, + end_phase=StepPhase.STEP_START, + refresh_stats=False, + ) + _validate_analysis(profiler_name, stats, [end_start_step_function.__name__]) + + # Test that train_and_eval_function call is recorded in received stats, but not the other functions. + stats = python_profile_analysis.fetch_profile_stats_between_modes( + PythonProfileModes.TRAIN, PythonProfileModes.EVAL, refresh_stats=False + ) + _validate_analysis(profiler_name, stats, [between_modes_function.__name__]) + + # Test that eval_function call is recorded in received stats, but not the other functions. + stats = python_profile_analysis.fetch_profile_stats_by_step( + 1, mode=PythonProfileModes.EVAL, refresh_stats=False + ) + _validate_analysis(profiler_name, stats, [eval_function.__name__]) + + # Test that pre_step_zero_function call is recorded in received stats, but not the other functions. + stats = python_profile_analysis.fetch_post_hook_close_profile_stats(refresh_stats=False) + _validate_analysis(profiler_name, stats, [post_hook_close_function.__name__]) + + # Test that time_function call is recorded in received stats, but not the other functions. + time_function_step_stats = python_profile_analysis.python_profile_stats[-1] + step_start_time = ( + time_function_step_stats.start_time_since_epoch_in_micros / CONVERT_TO_MICROSECS + ) + stats = python_profile_analysis.fetch_profile_stats_by_time( + step_start_time, time.time(), refresh_stats=False + ) + _validate_analysis(profiler_name, stats, [time_function.__name__]) + + # Following analysis functions are for cProfile only + if use_pyinstrument: + return + + # Test that functions called in TRAIN are recorded in received stats, but not the other functions. + stats = python_profile_analysis.fetch_profile_stats_by_training_phase(refresh_stats=False)[ + (PythonProfileModes.TRAIN, PythonProfileModes.TRAIN) + ] + _validate_analysis( + profiler_name, + stats, + [ + start_end_step_function.__name__, + end_start_step_function.__name__, + time_function.__name__, + ], + ) + + # Test that functions called in training loop are recorded in received stats, but not the other functions. + stats = python_profile_analysis.fetch_profile_stats_by_job_phase(refresh_stats=False)[ + "training_loop" + ] + _validate_analysis( + profiler_name, + stats, + [ + start_end_step_function.__name__, + end_start_step_function.__name__, + between_modes_function.__name__, + eval_function.__name__, + time_function.__name__, + ], + ) diff --git a/tests/profiler/core/test_smtfprofiler_events.py b/tests/profiler/core/test_smtfprofiler_events.py new file mode 100644 index 0000000000..1409a4a0c6 --- /dev/null +++ b/tests/profiler/core/test_smtfprofiler_events.py @@ -0,0 +1,51 @@ +# First Party +# Standard Library +from datetime import datetime + +from smdebug.profiler import SMProfilerEvents +from smdebug.profiler.utils import TimeUnits, get_node_id_from_tracefilename + + +def test_smprofiler_events( + trace_file="./tests/profiler/resources/1589314018481947000_1234-testhost_model_timeline.json" +): + trace_json_file = trace_file + print(f"Reading the trace file {trace_json_file}") + t_events = SMProfilerEvents() + t_events.read_events_from_file(trace_json_file) + + all_trace_events = t_events.get_all_events() + num_trace_events = len(all_trace_events) + + print(f"Number of events read = {num_trace_events}") + assert num_trace_events == 49 + + node_id_from_file = get_node_id_from_tracefilename(trace_json_file) + node_id_from_event = all_trace_events[10].node_id + assert node_id_from_event == node_id_from_file + + completed_event_list = t_events.get_events_within_time_range( + 0, 1589314018.4700, unit=TimeUnits.SECONDS + ) + print( + f"Number of events occurred between 0 and 1589314018.4700 are {len(completed_event_list)}" + ) + assert len(completed_event_list) == 39 + + start_dt = datetime.fromtimestamp(0) + end_dt = datetime.fromtimestamp(1589314018.4700) + completed_event_list = t_events.get_events_within_range(start_dt, end_dt) + print( + f"Number of events occurred between {start_dt} and {end_dt} are {len(completed_event_list)}" + ) + assert len(completed_event_list) == 39 + + start_time_sorted = t_events.get_events_start_time_sorted() + start_time_for_first_event = start_time_sorted[0].start_time + print(f"The first event started at {start_time_for_first_event}") + assert start_time_for_first_event == 1589314018458743 + + end_time_sorted = t_events.get_events_end_time_sorted() + end_time_for_last_event = end_time_sorted[-1].end_time + print(f"The first event started at {end_time_for_last_event}") + assert end_time_for_last_event == 1589314018481947 diff --git a/tests/profiler/core/test_system_metric_reader.py b/tests/profiler/core/test_system_metric_reader.py new file mode 100644 index 0000000000..cf547a2b12 --- /dev/null +++ b/tests/profiler/core/test_system_metric_reader.py @@ -0,0 +1,95 @@ +# First Party +# Third Party +import pytest + +from smdebug.profiler.system_metrics_reader import LocalSystemMetricsReader, S3SystemMetricsReader +from smdebug.profiler.utils import TimeUnits + + +@pytest.mark.parametrize("use_in_memory_cache", [True, False]) +def test_SystemLocalMetricsReader( + use_in_memory_cache, metricfolder="./tests/profiler/resources/test_traces" +): + lt = LocalSystemMetricsReader(metricfolder, use_in_memory_cache=use_in_memory_cache) + events = lt.get_events(1591100000, 1692300000, unit=TimeUnits.SECONDS) + + print(f"Number of events {len(events)}") + assert len(events) == 78 + + events = lt.get_events(1591100000, 1692300000, TimeUnits.SECONDS, "cpu") + + print(f"Number of cpu events {len(events)}") + assert len(events) == 68 + + events = lt.get_events(1591100000, 1591167699, TimeUnits.SECONDS) + + print(f"Number of events {len(events)}") + assert len(events) == 10 + + events = lt.get_events(1591748165, 1600000000, TimeUnits.SECONDS) + + print(f"Number of events {len(events)}") + assert len(events) == 32 + + +@pytest.mark.parametrize("use_in_memory_cache", [True, False]) +def test_SystemS3MetricsReader(use_in_memory_cache): + bucket_name = "s3://smdebug-testing/resources/trainingjob_name/profiler-output" + tt = S3SystemMetricsReader(bucket_name, use_in_memory_cache=use_in_memory_cache) + events = tt.get_events(1591100000, 1692300000, unit=TimeUnits.SECONDS) + + print(f"Number of events {len(events)}") + assert len(events) == 14 + + events = tt.get_events(1591100000, 1692300000, TimeUnits.SECONDS, "cpu") + + print(f"Number of cpu events {len(events)}") + assert len(events) == 4 + + events = tt.get_events(1591100000, 1591167699, TimeUnits.SECONDS) + + print(f"Number of events {len(events)}") + assert len(events) == 10 + + +@pytest.mark.parametrize("use_in_memory_cache", [True, False]) +def test_SystemS3MetricsReader_2(use_in_memory_cache): + bucket_name = "s3://smdebug-testing/resources/trainingjob_name2/profiler-output" + tt = S3SystemMetricsReader(bucket_name, use_in_memory_cache=use_in_memory_cache) + events = tt.get_events(1591748000, 1591749000, unit=TimeUnits.SECONDS) + + print(f"Number of events {len(events)}") + assert len(events) == 96 + + print(f"Start after prefix is {tt._startAfter_prefix}") + assert ( + tt._startAfter_prefix + == "resources/trainingjob_name2/profiler-output/system/incremental/1591748100.algo-1.json" + ) + + events = tt.get_events(1591748160, 1591749000, TimeUnits.SECONDS) + + print(f"Number of events {len(events)}") + assert len(events) == 64 + + events = tt.get_events(1591748170, 1591749000, TimeUnits.SECONDS) + + print(f"Number of events {len(events)}") + assert len(events) == 32 + + events = tt.get_events(1591748165, 1591749000, TimeUnits.SECONDS) + + print(f"Number of events {len(events)}") + assert len(events) == 64 + + +@pytest.mark.parametrize("use_in_memory_cache", [True, False]) +def test_SystemS3MetricsReader_updateStartAfterPrefix(use_in_memory_cache): + bucket_name = "s3://smdebug-testing/resources/trainingjob_name/profiler-output" + tt = S3SystemMetricsReader(bucket_name, use_in_memory_cache=use_in_memory_cache) + tt.get_events(1591100000, 1692300000, unit=TimeUnits.SECONDS) + print(f"Start after prefix is {tt._startAfter_prefix}") + assert ( + tt._startAfter_prefix + == "resources/trainingjob_name/profiler-output/system/incremental/1591160699.algo-1.json" + ) diff --git a/tests/profiler/core/test_system_profiler_file_parser.py b/tests/profiler/core/test_system_profiler_file_parser.py new file mode 100644 index 0000000000..d580e53ade --- /dev/null +++ b/tests/profiler/core/test_system_profiler_file_parser.py @@ -0,0 +1,62 @@ +# First Party +from smdebug.profiler.system_profiler_file_parser import ProfilerSystemEvents +from smdebug.profiler.utils import TimeUnits + + +def test_profiler_system_events(file_path="./tests/profiler/resources/1591160699.algo-1.json"): + event_json_file = file_path + print(f"Reading the profiler system event file {event_json_file}") + events = ProfilerSystemEvents() + events.read_events_from_file(event_json_file) + + all_events = events.get_all_events() + num_events = len(all_events) + + print(f"Number of events read = {num_events}") + assert num_events == 14 + + completed_event_list = events.get_events_within_time_range( + 0, 1591170700.4570, unit=TimeUnits.SECONDS + ) + print( + f"Number of events occurred between 0 and 1591170700.4570 are {len(completed_event_list)}" + ) + assert len(completed_event_list) == 14 + + completed_event_list = events.get_events_within_time_range( + 0, 1591170700.4570, TimeUnits.SECONDS, "cpu" + ) + print( + f"Number of cpu events occurred between 0 and 1591170700.4570 are {len(completed_event_list)}" + ) + assert len(completed_event_list) == 4 + + completed_event_list = events.get_events_within_time_range( + 0, 1591167699.9572, unit=TimeUnits.SECONDS + ) + print( + f"Number of events occurred between 0 and 1591167699.9572 are {len(completed_event_list)}" + ) + assert len(completed_event_list) == 11 + + completed_event_list = events.get_events_within_time_range( + 1591169699.4579, 2591167699.9572, unit=TimeUnits.SECONDS + ) + print( + f"Number of events occurred between 1591169699.4579 and 2591167699.9572 are {len(completed_event_list)}" + ) + assert len(completed_event_list) == 2 + + completed_event_list = events.get_events_within_time_range( + 1591169699.4579, 2591167699.9572, TimeUnits.SECONDS, "gpu" + ) + + print( + f"Number of gpu events occurred between 1591169699.4579 and 2591167699.9572 are {len(completed_event_list)}" + ) + assert len(completed_event_list) == 2 + + completed_event_list = events.get_events_within_time_range(0, 0, TimeUnits.SECONDS, "gpu") + + print(f"Number of gpu events occurred between 0 and 0 are {len(completed_event_list)}") + assert len(completed_event_list) == 0 diff --git a/tests/profiler/core/test_tfprofiler_events.py b/tests/profiler/core/test_tfprofiler_events.py new file mode 100644 index 0000000000..28228811b9 --- /dev/null +++ b/tests/profiler/core/test_tfprofiler_events.py @@ -0,0 +1,62 @@ +# First Party +# Standard Library +import os +import time + +from smdebug.profiler import TensorboardProfilerEvents +from smdebug.profiler.utils import TimeUnits, read_tf_profiler_metadata_file + + +def test_tensorboardprofiler_events( + trace_file="./tests/profiler/resources/tfprofiler_timeline_traces" +): + trace_json_file = "" + for dirpath, subdirs, files in os.walk(trace_file): + for x in files: + if x.endswith(".json.gz"): + trace_json_file = os.path.join(dirpath, x) + break + if trace_json_file == "": + assert False + + _, start_time_micros, end_time_micros = read_tf_profiler_metadata_file(trace_json_file) + + print(f"Reading the trace file {trace_json_file}") + t_events = TensorboardProfilerEvents() + t_events.read_events_from_file(trace_json_file) + + all_trace_events = t_events.get_all_events() + num_trace_events = len(all_trace_events) + + print(f"Number of events read = {num_trace_events}") + assert num_trace_events == 798 + + completed_event_list = t_events.get_events_within_time_range( + 0, time.time(), unit=TimeUnits.SECONDS + ) + print(f"Number of events occurred between 0 and {time.time()} are {len(completed_event_list)}") + assert len(completed_event_list) == 798 + + start_time_sorted = t_events.get_events_start_time_sorted() + start_time_for_first_event = start_time_sorted[0].start_time + relative_start_time = start_time_for_first_event - int(start_time_micros) + print(f"The first event started at {relative_start_time}") + assert relative_start_time == 21307.0 + + end_time_sorted = t_events.get_events_end_time_sorted() + end_time_for_last_event = end_time_sorted[-1].end_time + pid_last_event = end_time_sorted[-1].pid + tid_last_event = end_time_sorted[-1].tid + relative_end_time = end_time_for_last_event - int(start_time_micros) + print(f"The last event ended at {relative_end_time}") + assert relative_end_time == 293205.0 + + processes = t_events.get_processes() + print(f"Number of processes = {len(processes)}") + assert len(processes) == 2 + + process_info = t_events.get_process_info(pid_last_event) + print(f"Process Name = {process_info.name} Process Id = {process_info.id}") + + thread_info = process_info.get_thread_info(tid_last_event) + print(f"Thread name = {thread_info.thread_name} Thread id = {thread_info.tid}") diff --git a/tests/profiler/core/test_utils.py b/tests/profiler/core/test_utils.py new file mode 100644 index 0000000000..3cfd45f601 --- /dev/null +++ b/tests/profiler/core/test_utils.py @@ -0,0 +1,218 @@ +# Third Party +# Standard Library +import json +import time +import uuid +from pathlib import Path + +import pytest + +# First Party +from smdebug.core.access_layer.s3handler import ListRequest, S3Handler +from smdebug.profiler.algorithm_metrics_reader import ( + LocalAlgorithmMetricsReader, + S3AlgorithmMetricsReader, +) +from smdebug.profiler.analysis.utils.merge_timelines import MergedTimeline, MergeUnit +from smdebug.profiler.profiler_constants import ( + CONVERT_TO_MICROSECS, + HOROVODTIMELINE_SUFFIX, + MERGEDTIMELINE_SUFFIX, + MODELTIMELINE_SUFFIX, + PYTHONTIMELINE_SUFFIX, + TENSORBOARDTIMELINE_SUFFIX, +) +from smdebug.profiler.tf_profiler_parser import SMProfilerEvents +from smdebug.profiler.utils import ( + get_node_id_from_system_profiler_filename, + get_utctimestamp_us_since_epoch_from_system_profiler_file, + read_tf_profiler_metadata_file, +) + + +def test_get_node_id_from_system_profiler_filename(): + filename = "job-name/profiler-output/system/incremental/2020060500/1591160699.algo-1.json" + node_id = get_node_id_from_system_profiler_filename(filename) + assert node_id == "algo-1" + + +def test_get_utctimestamp_us_since_epoch_from_system_profiler_file(): + filename = "job-name/profiler-output/system/incremental/2020060500/1591160699.algo-1.json" + timestamp = get_utctimestamp_us_since_epoch_from_system_profiler_file(filename) + assert timestamp == 1591160699000000 + + filename = "job-name/profiler-output/system/incremental/2020060500/1591160699.lgo-1.json" + timestamp = get_utctimestamp_us_since_epoch_from_system_profiler_file(filename) + assert timestamp is None + + +@pytest.mark.parametrize("trace_location", ["local", "s3"]) +def test_timeline_merge(out_dir, trace_location): + if trace_location == "local": + tracefolder = "./tests/profiler/resources/merge_traces" + combined_timeline = MergedTimeline(tracefolder, output_directory=out_dir) + start_time_us = 1594662618874598 + end_time_us = 1594662623418701 + combined_timeline.merge_timeline(start_time_us, end_time_us) + algo_reader = LocalAlgorithmMetricsReader(tracefolder) + elif trace_location == "s3": + tracefolder = "s3://smdebug-testing/resources/tf2_detailed_profile/profiler-output" + combined_timeline = MergedTimeline(tracefolder, output_directory=out_dir) + start_time_us = 1596668400000000 + end_time_us = 1596668441010095 + combined_timeline.merge_timeline(start_time_us, end_time_us) + algo_reader = S3AlgorithmMetricsReader(tracefolder) + else: + return + + files = [] + for path in Path(out_dir).rglob(f"*{MERGEDTIMELINE_SUFFIX}"): + files.append(path) + + assert len(files) == 1 + + with open(files[0], "r+") as merged_file: + event_list = json.load(merged_file) + + # check if the events are sorted by start time + for i in range(len(event_list) - 1): + if "ts" in event_list[i] and "ts" in event_list[i + 1]: + assert event_list[i]["ts"] <= event_list[i + 1]["ts"] + + total_events = 0 + # check if the number of events match individual files + for suffix in [ + PYTHONTIMELINE_SUFFIX, + MODELTIMELINE_SUFFIX, + TENSORBOARDTIMELINE_SUFFIX, + HOROVODTIMELINE_SUFFIX, + ]: + events = algo_reader.get_events(start_time_us, end_time_us, file_suffix_filter=[suffix]) + total_events += len(events) + + t_events = SMProfilerEvents() + t_events.read_events_from_file(str(files[0])) + all_trace_events = t_events.get_all_events() + num_trace_events = len(all_trace_events) + + assert total_events == num_trace_events + + +@pytest.mark.parametrize("trace_location", ["local", "s3"]) +def test_timeline_merge_by_step(out_dir, trace_location): + start_step, end_step = 2, 4 + if trace_location == "local": + tracefolder = "./tests/profiler/resources/merge_traces" + combined_timeline = MergedTimeline(tracefolder, output_directory=out_dir) + combined_timeline.merge_timeline(start_step, end_step, unit=MergeUnit.STEP) + algo_reader = LocalAlgorithmMetricsReader(tracefolder) + elif trace_location == "s3": + tracefolder = "s3://smdebug-testing/resources/tf2_detailed_profile/profiler-output" + combined_timeline = MergedTimeline(tracefolder, output_directory=out_dir) + combined_timeline.merge_timeline(start_step, end_step, unit=MergeUnit.STEP) + algo_reader = S3AlgorithmMetricsReader(tracefolder) + else: + return + + files = [] + for path in Path(out_dir).rglob(f"*{MERGEDTIMELINE_SUFFIX}"): + files.append(path) + + assert len(files) == 1 + + total_events = 0 + # check if the number of events match individual files + for suffix in [ + PYTHONTIMELINE_SUFFIX, + MODELTIMELINE_SUFFIX, + TENSORBOARDTIMELINE_SUFFIX, + HOROVODTIMELINE_SUFFIX, + ]: + events = algo_reader.get_events_by_step(start_step, end_step, file_suffix_filter=[suffix]) + total_events += len(events) + + t_events = SMProfilerEvents() + t_events.read_events_from_file(str(files[0])) + all_trace_events = t_events.get_all_events() + num_trace_events = len(all_trace_events) + + assert total_events == num_trace_events + + +def test_merge_timeline_s3_write(): + bucket_name = "smdebug-testing" + key_name = f"outputs/smprofiler-timeline-merge-test-{uuid.uuid4()}" + location = "s3://{}/{}".format(bucket_name, key_name) + + tracefolder = "./tests/profiler/resources/merge_traces" + combined_timeline = MergedTimeline(tracefolder, output_directory=location) + combined_timeline.merge_timeline(0, time.time() * CONVERT_TO_MICROSECS, unit=MergeUnit.TIME) + + start_step, end_step = 2, 4 + tracefolder = "s3://smdebug-testing/resources/tf2_detailed_profile/profiler-output" + combined_timeline = MergedTimeline(tracefolder, output_directory=location) + combined_timeline.merge_timeline(start_step, end_step, unit=MergeUnit.STEP) + + request = ListRequest(bucket_name, key_name) + files = S3Handler.list_prefixes([request]) + assert len(files) == 1 + assert len(files[0]) == 2 + + +def test_timeline_merge_file_suffix_filter(out_dir): + bucket_name = "s3://smdebug-testing/profiler-traces" + combined_timeline = MergedTimeline( + path=bucket_name, file_suffix_filter=[MODELTIMELINE_SUFFIX], output_directory=out_dir + ) + combined_timeline.merge_timeline(0, time.time() * CONVERT_TO_MICROSECS) + + files = [] + for path in Path(out_dir).rglob(f"*{MERGEDTIMELINE_SUFFIX}"): + files.append(path) + + assert len(files) == 1 + + with open(files[0], "r+") as merged_file: + event_list = json.load(merged_file) + + for i in range(len(event_list) - 1): + if "ts" in event_list[i] and "ts" in event_list[i + 1]: + assert event_list[i]["ts"] <= event_list[i + 1]["ts"] + + +def test_tf_profiler_metadata_file_read(): + # (key, value) = (file_path : expected result from read_metadata) + file_path_list = { + "./tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/" + "detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/" + "8c859046be41.ant.amazon.com.trace.json.gz": ( + "80807-8c859046be41.ant.amazon.com", + "1596659864545103", + "1596659864854168", + ), + "./tests/profiler/resources/tfprofiler_local_missing_metadata/framework/tensorflow/" + "detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/" + "ip-172-31-19-241.trace.json.gz": ("", "0", "0"), + "s3://smdebug-testing/resources/tf2_detailed_profile/profiler-output/framework/tensorflow/" + "detailed_profiling/2020080523/000000002/plugins/profile/2020_08_05_23_00_25/" + "ip-10-0-209-63.ec2.internal.trace.json.gz": ( + "151-algo-1", + "1596668425766312", + "1596668425896759", + ), + "s3://smdebug-testing/resources/missing_tf_profiler_metadata/framework/tensorflow/" + "detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/" + "8c859046be41.ant.amazon.com.trace.json.gz": ("", "0", "0"), + "s3://smdebug-testing/empty-traces/framework/tensorflow/detailed_profiling/2020052817/" + "ip-172-31-48-136.trace.json.gz": ("", "0", "0"), + "s3://smdebug-testing/empty-traces/framework/tensorflow/detailed_profiling/2020052817/" + "ip-172-31-48-136.pb": ("", "0", "0"), + "s3://smdebug-testing/empty-traces/framework/pevents/2020052817/" + "ip-172-31-48-136.trace.json.gz": ("", "0", "0"), + } + for filepath in file_path_list: + print(f"Searching for metadata for {filepath}") + (node_id, start, end) = read_tf_profiler_metadata_file(filepath) + assert (node_id, start, end) == file_path_list[ + filepath + ], f"Metadata not found for {filepath}" diff --git a/tests/profiler/ip-172-31-19-241.trace.json b/tests/profiler/ip-172-31-19-241.trace.json deleted file mode 100644 index 5a9c0805f0..0000000000 --- a/tests/profiler/ip-172-31-19-241.trace.json +++ /dev/null @@ -1,3034 +0,0 @@ -{ - "displayTimeUnit": "ns", - "metadata": { - "highres-ticks": true - }, - "traceEvents": [ - { - "ph": "M", - "pid": 0, - "name": "process_name", - "args": { - "name": "/device:GPU:0" - } - }, - { - "ph": "M", - "pid": 0, - "name": "process_sort_index", - "args": { - "sort_index": 0 - } - }, - { - "ph": "M", - "pid": 1, - "name": "process_name", - "args": { - "name": "/device:GPU:1" - } - }, - { - "ph": "M", - "pid": 1, - "name": "process_sort_index", - "args": { - "sort_index": 1 - } - }, - { - "ph": "M", - "pid": 2, - "name": "process_name", - "args": { - "name": "/device:GPU:2" - } - }, - { - "ph": "M", - "pid": 2, - "name": "process_sort_index", - "args": { - "sort_index": 2 - } - }, - { - "ph": "M", - "pid": 3, - "name": "process_name", - "args": { - "name": "/device:GPU:3" - } - }, - { - "ph": "M", - "pid": 3, - "name": "process_sort_index", - "args": { - "sort_index": 3 - } - }, - { - "ph": "M", - "pid": 4, - "name": "process_name", - "args": { - "name": "/device:GPU:4" - } - }, - { - "ph": "M", - "pid": 4, - "name": "process_sort_index", - "args": { - "sort_index": 4 - } - }, - { - "ph": "M", - "pid": 5, - "name": "process_name", - "args": { - "name": "/device:GPU:5" - } - }, - { - "ph": "M", - "pid": 5, - "name": "process_sort_index", - "args": { - "sort_index": 5 - } - }, - { - "ph": "M", - "pid": 6, - "name": "process_name", - "args": { - "name": "/device:GPU:6" - } - }, - { - "ph": "M", - "pid": 6, - "name": "process_sort_index", - "args": { - "sort_index": 6 - } - }, - { - "ph": "M", - "pid": 7, - "name": "process_name", - "args": { - "name": "/device:GPU:7" - } - }, - { - "ph": "M", - "pid": 7, - "name": "process_sort_index", - "args": { - "sort_index": 7 - } - }, - { - "ph": "M", - "pid": 49, - "name": "process_name", - "args": { - "name": "/host:CPU" - } - }, - { - "ph": "M", - "pid": 49, - "name": "process_sort_index", - "args": { - "sort_index": 49 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 376039232, - "name": "thread_name", - "args": { - "name": "python3" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 376039232, - "name": "thread_sort_index", - "args": { - "sort_index": 376039232 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 427796224, - "name": "thread_name", - "args": { - "name": "tf_data_iterator_resource" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 427796224, - "name": "thread_sort_index", - "args": { - "sort_index": 427796224 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 452974336, - "name": "thread_name", - "args": { - "name": "tf_data_iterator_resource" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 452974336, - "name": "thread_sort_index", - "args": { - "sort_index": 452974336 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 553621248, - "name": "thread_name", - "args": { - "name": "tf_data_iterator_resource" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 553621248, - "name": "thread_sort_index", - "args": { - "sort_index": 553621248 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 562013952, - "name": "thread_name", - "args": { - "name": "tf_data_iterator_resource" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 562013952, - "name": "thread_sort_index", - "args": { - "sort_index": 562013952 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 595584768, - "name": "thread_name", - "args": { - "name": "tf_data_iterator_resource" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 595584768, - "name": "thread_sort_index", - "args": { - "sort_index": 595584768 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 687838976, - "name": "thread_name", - "args": { - "name": "tf_data_iterator_resource" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 687838976, - "name": "thread_sort_index", - "args": { - "sort_index": 687838976 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 721409792, - "name": "thread_name", - "args": { - "name": "tf_data_iterator_resource" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 721409792, - "name": "thread_sort_index", - "args": { - "sort_index": 721409792 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 729802496, - "name": "thread_name", - "args": { - "name": "tf_data_iterator_resource" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 729802496, - "name": "thread_sort_index", - "args": { - "sort_index": 729802496 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 830449408, - "name": "thread_name", - "args": { - "name": "tf_data_iterator_resource" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 830449408, - "name": "thread_sort_index", - "args": { - "sort_index": 830449408 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 838842112, - "name": "thread_name", - "args": { - "name": "tf_data_iterator_resource" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 838842112, - "name": "thread_sort_index", - "args": { - "sort_index": 838842112 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 847234816, - "name": "thread_name", - "args": { - "name": "tf_data_iterator_resource" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 847234816, - "name": "thread_sort_index", - "args": { - "sort_index": 847234816 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 855627520, - "name": "thread_name", - "args": { - "name": "tf_data_iterator_resource" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 855627520, - "name": "thread_sort_index", - "args": { - "sort_index": 855627520 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 872412928, - "name": "thread_name", - "args": { - "name": "tf_data_iterator_get_next" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 872412928, - "name": "thread_sort_index", - "args": { - "sort_index": 872412928 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1375713024, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1375713024, - "name": "thread_sort_index", - "args": { - "sort_index": 1375713024 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1384105728, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1384105728, - "name": "thread_sort_index", - "args": { - "sort_index": 1384105728 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1392498432, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1392498432, - "name": "thread_sort_index", - "args": { - "sort_index": 1392498432 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1400891136, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1400891136, - "name": "thread_sort_index", - "args": { - "sort_index": 1400891136 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1409283840, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1409283840, - "name": "thread_sort_index", - "args": { - "sort_index": 1409283840 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1493145344, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1493145344, - "name": "thread_sort_index", - "args": { - "sort_index": 1493145344 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1501538048, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1501538048, - "name": "thread_sort_index", - "args": { - "sort_index": 1501538048 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1509930752, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1509930752, - "name": "thread_sort_index", - "args": { - "sort_index": 1509930752 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1518323456, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1518323456, - "name": "thread_sort_index", - "args": { - "sort_index": 1518323456 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1526716160, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1526716160, - "name": "thread_sort_index", - "args": { - "sort_index": 1526716160 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1535108864, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1535108864, - "name": "thread_sort_index", - "args": { - "sort_index": 1535108864 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1543501568, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1543501568, - "name": "thread_sort_index", - "args": { - "sort_index": 1543501568 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1895798528, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1895798528, - "name": "thread_sort_index", - "args": { - "sort_index": 1895798528 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1904191232, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1904191232, - "name": "thread_sort_index", - "args": { - "sort_index": 1904191232 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1912583936, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1912583936, - "name": "thread_sort_index", - "args": { - "sort_index": 1912583936 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1920976640, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1920976640, - "name": "thread_sort_index", - "args": { - "sort_index": 1920976640 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1929369344, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1929369344, - "name": "thread_sort_index", - "args": { - "sort_index": 1929369344 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1937762048, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1937762048, - "name": "thread_sort_index", - "args": { - "sort_index": 1937762048 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1946154752, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 1946154752, - "name": "thread_sort_index", - "args": { - "sort_index": 1946154752 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2097125120, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2097125120, - "name": "thread_sort_index", - "args": { - "sort_index": 2097125120 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2105517824, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2105517824, - "name": "thread_sort_index", - "args": { - "sort_index": 2105517824 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2113910528, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2113910528, - "name": "thread_sort_index", - "args": { - "sort_index": 2113910528 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2122303232, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2122303232, - "name": "thread_sort_index", - "args": { - "sort_index": 2122303232 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2130695936, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2130695936, - "name": "thread_sort_index", - "args": { - "sort_index": 2130695936 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2139088640, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2139088640, - "name": "thread_sort_index", - "args": { - "sort_index": 2139088640 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2147481344, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2147481344, - "name": "thread_sort_index", - "args": { - "sort_index": 2147481344 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2701104896, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2701104896, - "name": "thread_sort_index", - "args": { - "sort_index": 2701104896 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2709497600, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2709497600, - "name": "thread_sort_index", - "args": { - "sort_index": 2709497600 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2751461120, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 2751461120, - "name": "thread_sort_index", - "args": { - "sort_index": 2751461120 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 3774846720, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 3774846720, - "name": "thread_sort_index", - "args": { - "sort_index": 3774846720 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 3785520896, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 3785520896, - "name": "thread_sort_index", - "args": { - "sort_index": 3785520896 - } - }, - { - "ph": "M", - "pid": 49, - "tid": 3808417536, - "name": "thread_name", - "args": { - "name": "tf_Compute" - } - }, - { - "ph": "M", - "pid": 49, - "tid": 3808417536, - "name": "thread_sort_index", - "args": { - "sort_index": 3808417536 - } - }, - { - "pid": 49, - "tid": 376039232, - "ts": 116.457, - "name": "train_function", - "ph": "X", - "dur": 63929.222, - "args": { - "tf_function_call": "notTraced-nonXla", - "tracing_count": "2" - } - }, - { - "pid": 49, - "tid": 376039232, - "ts": 471.505, - "name": "EagerExecute: __inference_train_function_7724", - "ph": "X", - "dur": 63399.491 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 480.062, - "name": "EagerLocalExecute: __inference_train_function_7724", - "ph": "X", - "dur": 63389.109 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 481.926, - "name": "EagerCopyToDeviceAndAddCacheKey", - "ph": "X", - "dur": 856.75 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 485.724, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.316 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 501.633, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.001 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 506.904, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.843 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 510.725, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.913 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 513.33, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.854 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 515.679, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.98 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 519.199, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.872 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 521.43, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.947 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 524.122, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.896 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 526.322, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.993 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 528.588, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.861 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 531.747, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.954 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 534.076, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.89 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 536.426, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.863 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 539.632, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.881 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 542.005, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.879 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 544.149, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.874 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 546.481, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.988 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 549.439, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.952 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 552.578, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.892 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 554.713, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.096 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 557.081, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.87 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 560.266, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.857 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 562.443, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.926 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 564.686, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.859 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 566.931, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.918 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 569.16, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 29.434 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 601.365, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.891 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 603.647, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.873 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 606.162, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.893 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 609.722, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.979 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 612.266, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.874 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 614.476, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.873 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 616.668, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.378 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 619.419, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.797 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 622.484, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.819 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 624.556, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.948 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 626.9, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.853 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 631.097, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.869 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 633.303, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.858 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 635.441, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.979 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 637.775, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.885 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 639.946, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.02 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 643.439, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.906 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 645.603, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.822 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 647.905, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.896 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 651.243, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.94 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 653.672, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.916 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 655.864, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.934 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 658.488, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.907 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 660.745, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.941 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 663.926, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.917 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 666.077, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.885 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 668.357, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.989 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 671.635, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.019 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 674.079, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.911 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 676.253, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.911 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 678.504, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.861 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 680.704, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.792 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 683.986, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.035 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 686.309, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.865 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 688.589, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.864 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 691.621, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.9 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 693.837, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.016 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 696.148, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.848 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 698.626, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.963 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 700.984, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.86 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 704.229, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.919 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 706.898, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.919 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 709.216, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.96 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 712.349, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.844 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 714.879, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.029 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 717.492, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.881 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 719.769, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.874 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 721.913, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.848 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 724.943, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.892 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 727.137, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.855 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 729.424, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.955 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 732.494, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.907 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 734.905, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.892 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 739.856, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.852 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 742.096, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.24 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 744.61, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.909 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 747.721, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.258 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 750.326, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.9 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 752.703, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.922 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 755.719, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.871 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 757.883, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.089 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 760.201, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.877 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 762.508, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.091 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 764.897, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.889 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 768.942, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.827 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 771.06, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.914 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 773.412, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.13 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 776.601, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.854 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 778.95, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.84 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 781.043, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.915 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 783.371, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.813 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 785.657, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.933 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 788.859, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.877 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 791.272, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.862 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 793.694, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.885 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 796.691, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.856 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 799.168, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.872 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 801.331, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.846 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 803.555, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.89 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 805.689, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.911 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 808.987, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.915 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 811.274, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.912 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 813.616, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.773 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 816.628, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.875 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 818.851, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.935 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 821.146, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.016 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 823.679, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.894 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 825.871, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.848 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 828.905, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.929 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 831.292, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.859 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 833.628, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.019 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 837.183, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.877 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 839.634, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.871 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 842.02, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.971 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 844.298, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.874 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 846.452, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.851 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 849.41, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.965 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 851.82, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.818 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 854.111, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 857.31, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.877 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 859.66, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.811 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 861.823, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.834 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 864.006, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.866 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 866.227, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.93 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 869.899, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.853 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 872.062, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.952 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 874.237, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.822 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 877.115, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.871 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 879.732, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.887 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 881.944, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.877 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 884.179, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.893 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 886.337, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.876 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 889.33, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.897 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 891.462, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.047 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 894.018, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.899 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 897.005, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.862 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 899.384, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.842 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 901.469, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.878 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 903.615, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.881 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 905.791, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.993 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 909.14, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.892 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 911.341, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.804 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 913.623, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.307 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 917.113, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.953 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 919.376, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.879 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 921.538, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.9 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 923.844, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.906 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 925.994, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.829 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 928.995, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.865 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 931.196, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.859 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 933.503, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.997 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 936.518, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.864 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 938.658, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.871 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 940.81, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.94 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 942.963, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.902 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 945.169, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.878 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 948.145, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.877 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 950.233, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.072 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 952.712, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.915 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 955.621, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.902 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 957.768, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.894 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 963.112, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.007 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 965.406, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.888 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 967.574, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.895 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 970.336, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.867 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 972.488, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.97 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 974.803, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.9 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 978.221, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.88 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 980.338, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.878 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 982.534, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.007 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 984.872, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.842 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 986.9, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.868 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 989.784, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.863 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 991.852, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.944 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 994.333, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.077 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 997.735, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.096 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1000.062, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.91 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1002.177, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.988 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1004.422, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.843 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1006.465, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.899 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1009.455, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.92 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1011.662, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.86 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1013.895, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.857 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1017.375, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.928 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1019.604, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.911 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1021.737, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.971 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1023.948, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.915 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1026.313, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.842 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1029.329, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.873 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1031.633, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.213 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1034.192, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.86 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1037.001, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.899 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1039.148, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.873 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1041.24, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.851 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1043.338, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.845 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1045.417, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.876 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1048.711, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.856 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1050.825, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.796 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1053.023, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.84 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1055.862, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.886 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1058.053, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.897 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1060.2, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.919 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1062.388, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.866 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1064.505, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.922 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1067.667, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.843 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1069.699, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.87 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1072.09, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.882 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1074.982, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.885 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1077.105, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.216 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1079.607, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.881 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1081.708, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.903 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1083.94, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.868 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1087.048, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.861 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1089.131, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.873 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1091.342, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.879 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1094.258, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.94 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1096.521, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.898 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1098.651, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.92 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1100.858, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.835 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1103.037, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.859 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1105.918, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.943 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1108.189, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.838 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1110.685, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.049 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1113.813, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.84 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1115.93, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.893 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1118.08, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.105 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1120.407, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.902 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1122.501, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.844 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1125.983, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.869 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1128.095, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.895 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1130.442, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.005 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1133.508, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.848 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1135.959, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.917 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1138.01, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.897 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1140.31, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.891 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1142.439, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.891 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1145.398, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.864 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1147.67, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.874 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1150.049, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.934 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1153.01, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.886 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1155.217, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.89 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1157.377, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.28 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1159.929, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.896 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1162.109, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 0.972 - }, - { - "pid": 49, - "tid": 376039232, - "ts": 1166.027, - "name": "TensorHandle::GetResourceHandleInfo WaitReady", - "ph": "X", - "dur": 1.133 - }, - { - } - ] -} diff --git a/tests/profiler/pytorch/__init__.py b/tests/profiler/pytorch/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/profiler/pytorch/scripts/pytorch_rnn.py b/tests/profiler/pytorch/scripts/pytorch_rnn.py new file mode 100644 index 0000000000..b9a4827699 --- /dev/null +++ b/tests/profiler/pytorch/scripts/pytorch_rnn.py @@ -0,0 +1,67 @@ +# before running this script +# make sure that smdebug is installed +# set env export SMPROFILER_CONFIG_PATH="/tmp/profilerconfig.json" +# dump this json in file: +# { +# "ProfilingParameters": { +# "ProfilerEnabled": true, +# "LocalPath": "/tmp/test" +# } +# } + +# Third Party +import torch +import torch.nn as nn + +# First Party +import smdebug.pytorch as smd + + +class RNN(nn.Module): + + # you can also accept arguments in your model constructor + def __init__(self, data_size, hidden_size, output_size): + super(RNN, self).__init__() + + self.hidden_size = hidden_size + input_size = data_size + hidden_size + + self.i2h = nn.Linear(input_size, hidden_size) + self.h2o = nn.Linear(hidden_size, output_size) + + def forward(self, data, last_hidden): + input = torch.cat((data, last_hidden), 1) + hidden = self.i2h(input) + output = self.h2o(hidden) + return hidden, output + + +rnn = RNN(50, 20, 10) +save_config = smd.SaveConfig(save_interval=500) +hook = smd.Hook(out_dir="/tmp/smdebug", save_all=True, save_config=save_config) + +loss_fn = nn.MSELoss() + +hook.register_module(rnn) +# hook.register_module(loss_fn) + + +batch_size = 10 +TIMESTEPS = 5 + +# Create some fake data +batch = torch.randn(batch_size, 50) +hidden = torch.zeros(batch_size, 20) +target = torch.zeros(batch_size, 10) + +loss = 0 +# +for t in range(TIMESTEPS): + # yes! you can reuse the same network several times, + # sum up the losses, and call backward! + hidden, output = rnn(batch, hidden) + loss += loss_fn(output, target) +loss.backward() + +# F F F F F B B F F B B B FFFFFF F BBB +# diff --git a/tests/profiler/pytorch/test_pytorch_profiler.py b/tests/profiler/pytorch/test_pytorch_profiler.py new file mode 100644 index 0000000000..087a439885 --- /dev/null +++ b/tests/profiler/pytorch/test_pytorch_profiler.py @@ -0,0 +1,88 @@ +# Future +from __future__ import print_function + +# Standard Library +import os +import time + +# Third Party +import pytest +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +from tests.profiler.pytorch.utils import is_pt_1_5, is_pt_1_6 +from torch.autograd import Variable + +# First Party +from smdebug.profiler.algorithm_metrics_reader import LocalAlgorithmMetricsReader +from smdebug.profiler.profiler_config_parser import ProfilerConfigParser +from smdebug.pytorch import Hook, modes + + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + self.add_module("conv1", nn.Conv2d(1, 20, 5, 1)) + self.add_module("relu0", nn.ReLU()) + self.add_module("max_pool", nn.MaxPool2d(2, stride=2)) + self.add_module("conv2", nn.Conv2d(20, 50, 5, 1)) + self.add_module("relu1", nn.ReLU()) + self.add_module("max_pool2", nn.MaxPool2d(2, stride=2)) + self.add_module("fc1", nn.Linear(4 * 4 * 50, 500)) + self.add_module("relu2", nn.ReLU()) + self.add_module("fc2", nn.Linear(500, 10)) + + def forward(self, x): + x = self.relu0(self.conv1(x)) + x = self.max_pool(x) + x = self.relu1(self.conv2(x)) + x = self.max_pool2(x) + x = x.view(-1, 4 * 4 * 50) + x = self.relu2(self.fc1(x)) + x = self.fc2(x) + return F.log_softmax(x, dim=1) + + +def train(model, device, optimizer, hook): + model.train() + + for i in range(10): + batch_size = 32 + data, target = torch.rand(batch_size, 1, 28, 28), torch.rand(batch_size).long() + hook.set_mode(modes.TRAIN) + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(Variable(data, requires_grad=True)) + loss = F.nll_loss(output, target) + loss.backward() + optimizer.step() + hook.set_mode(modes.EVAL) + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(Variable(data, requires_grad=True)) + + +@pytest.fixture() +def pytorch_profiler_config_parser(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "test_pytorch_profiler_config_parser.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + +def test_pytorch_profiler(pytorch_profiler_config_parser, out_dir): + device = torch.device("cpu") + model = Net().to(device) + hook = Hook(out_dir=out_dir) + hook.register_hook(model) + optimizer = optim.SGD(model.parameters(), lr=0.001) + train(model, device, optimizer, hook) + hook.close() + lt = LocalAlgorithmMetricsReader(out_dir) + lt.refresh_event_file_list() + events = lt.get_events(0, time.time() * 1000000) + print(f"Number of events {len(events)}") + if is_pt_1_5(): + assert len(events) == 386 + elif is_pt_1_6(): + assert len(events) == 672 diff --git a/tests/profiler/pytorch/test_pytorch_profiler_rnn.py b/tests/profiler/pytorch/test_pytorch_profiler_rnn.py new file mode 100644 index 0000000000..d8cedde2b1 --- /dev/null +++ b/tests/profiler/pytorch/test_pytorch_profiler_rnn.py @@ -0,0 +1,80 @@ +# Standard Library +import os +import shutil +import time + +# Third Party +import pytest +import torch +import torch.nn as nn +from tests.profiler.pytorch.utils import is_pt_1_5, is_pt_1_6 + +# First Party +import smdebug.pytorch as smd +from smdebug.profiler import LocalAlgorithmMetricsReader +from smdebug.profiler.profiler_config_parser import ProfilerConfigParser + + +class RNN(nn.Module): + + # you can also accept arguments in your model constructor + def __init__(self, data_size, hidden_size, output_size): + super(RNN, self).__init__() + + self.hidden_size = hidden_size + input_size = data_size + hidden_size + + self.i2h = nn.Linear(input_size, hidden_size) + self.h2o = nn.Linear(hidden_size, output_size) + + def forward(self, data, last_hidden): + input = torch.cat((data, last_hidden), 1) + hidden = self.i2h(input) + output = self.h2o(hidden) + return hidden, output + + +def train_model(out_dir="/tmp/smdebug", training_steps=5): + rnn = RNN(50, 20, 10) + save_config = smd.SaveConfig(save_interval=500) + hook = smd.Hook(out_dir=out_dir, save_all=True, save_config=save_config) + + loss_fn = nn.MSELoss() + + hook.register_module(rnn) + hook.register_module(loss_fn) + + batch_size = 10 + TIMESTEPS = training_steps + + # Create some fake data + batch = torch.randn(batch_size, 50) + hidden = torch.zeros(batch_size, 20) + target = torch.zeros(batch_size, 10) + + loss = 0 + for t in range(TIMESTEPS): + hidden, output = rnn(batch, hidden) + loss += loss_fn(output, target) + loss.backward() + hook.close() + + +@pytest.fixture() +def pytorch_profiler_config_parser(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "test_pytorch_profiler_config_parser.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + +def test_pytorch_profiler_rnn(pytorch_profiler_config_parser, out_dir): + train_model(out_dir) + lt = LocalAlgorithmMetricsReader(out_dir) + lt.refresh_event_file_list() + events = lt.get_events(0, time.time() * 1000000) + print(f"Number of events {len(events)}") + if is_pt_1_5(): + assert len(events) <= 64 + elif is_pt_1_6(): + assert len(events) <= 85 + shutil.rmtree(out_dir, ignore_errors=True) diff --git a/tests/profiler/pytorch/utils.py b/tests/profiler/pytorch/utils.py new file mode 100644 index 0000000000..b0cf4c4c28 --- /dev/null +++ b/tests/profiler/pytorch/utils.py @@ -0,0 +1,27 @@ +# Third Party +import torch +from packaging import version + + +def is_pt_1_5(): + """ + Determine whether the version of torch is 1.5.x + :return: bool + """ + return version.parse("1.5.0") <= version.parse(torch.__version__) < version.parse("1.6.0") + + +def is_pt_1_6(): + """ + Determine whether the version of torch is 1.6.x + :return: bool + """ + return version.parse("1.6.0") <= version.parse(torch.__version__) < version.parse("1.7.0") + + +def is_pt_1_7(): + """ + Determine whether the version of torch is 1.7.x + :return: bool + """ + return version.parse("1.7.0") <= version.parse(torch.__version__) < version.parse("1.8.0") diff --git a/tests/profiler/smtf_profiler_trace.json b/tests/profiler/resources/1589314018481947000_1234-testhost_model_timeline.json similarity index 100% rename from tests/profiler/smtf_profiler_trace.json rename to tests/profiler/resources/1589314018481947000_1234-testhost_model_timeline.json diff --git a/tests/profiler/resources/1591160699.algo-1.json b/tests/profiler/resources/1591160699.algo-1.json new file mode 100644 index 0000000000..728011df0f --- /dev/null +++ b/tests/profiler/resources/1591160699.algo-1.json @@ -0,0 +1,14 @@ +{"Type": "cpu", "Name":"cpu0", "Dimension": "CPUUtilization", "Value":4.76,"NodeId":"algo-1","Timestamp":1591160699.4570894} +{"Type": "cpu", "Name":"cpu1", "Dimension": "CPUUtilization", "Value":5.30,"NodeId":"algo-1","Timestamp":1591160799.5570894} +{"Type": "gpu", "Name":"gpu0", "Dimension": "GPUUtilization", "Value":8.00,"NodeId":"algo-1","Timestamp":1591160899.7570894} +{"Type": "gpu", "Name":"gpu1", "Dimension": "GPUUtilization", "Value":14.76,"NodeId":"algo-1","Timestamp":1591160999.6570894} +{"Type": "gpu", "Name":"gpu0", "Dimension": "GPUMemoryUtilization", "Value":14.76,"NodeId":"algo-1","Timestamp":1591161699.1570894} +{"Type": "gpu", "Name":"gpu1", "Dimension": "GPUMemoryUtilization", "Value":24.76,"NodeId":"algo-1","Timestamp":1591162699.2570894} +{"Type": "gpu", "Name":"gpu1", "Dimension": "GPUMemoryUtilization", "Value":43.76,"NodeId":"algo-1","Timestamp":1591163699.4577894} +{"Type": "cpu", "Name":"cpu0", "Dimension": "CPUUtilization", "Value":54.76,"NodeId":"algo-1","Timestamp":1591164699.4560894} +{"Type": "cpu", "Name":"cpu1", "Dimension": "CPUUtilization", "Value":44.76,"NodeId":"algo-1","Timestamp":1591165699.4570594} +{"Type": "gpu", "Name":"gpu0", "Dimension": "GPUUtilization", "Value":34.76,"NodeId":"algo-1","Timestamp":1591166699.4570394} +{"Type": "gpu", "Name":"gpu1", "Dimension": "GPUUtilization", "Value":24.76,"NodeId":"algo-1","Timestamp":1591167699.4572894} +{"Type": "gpu", "Name":"gpu0", "Dimension": "GPUMemoryUtilization", "Value":74.76,"NodeId":"algo-1","Timestamp":1591168699.4578894} +{"Type": "gpu", "Name":"gpu1", "Dimension": "GPUMemoryUtilization", "Value":84.76,"NodeId":"algo-1","Timestamp":1591169699.4579994} +{"Type": "gpu", "Name":"gpu1", "Dimension": "GPUMemoryUtilization", "Value":47.76,"NodeId":"algo-1","Timestamp":1591170699.4570800} diff --git a/tests/profiler/resources/__init__.py b/tests/profiler/resources/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/profiler/resources/horovod_timeline_small.json b/tests/profiler/resources/horovod_timeline_small.json new file mode 100644 index 0000000000..ecba043b86 --- /dev/null +++ b/tests/profiler/resources/horovod_timeline_small.json @@ -0,0 +1,59 @@ +[ +{"ph": "i", "name": "CYCLE_START", "ts": 77567277433, "s": "g"}, +{"name": "process_name", "ph": "M", "pid": 1, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_dense_1_BiasAdd_BiasAddGrad_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 1, "args": {"sort_index": 1}}, +{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 77575718041, "pid": 1}, +{"ph": "X", "name": "4", "ts": 77575718043, "pid": 1, "dur": 0}, +{"name": "process_name", "ph": "M", "pid": 2, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_dense_1_MatMul_1_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 2, "args": {"sort_index": 2}}, +{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 77575718047, "pid": 2}, +{"ph": "X", "name": "4", "ts": 77575718049, "pid": 2, "dur": 0}, +{"name": "process_name", "ph": "M", "pid": 3, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_dense_BiasAdd_BiasAddGrad_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 3, "args": {"sort_index": 3}}, +{"ph": "X", "name": "4", "ts": 77575718054, "pid": 3, "dur": 0}, +{"name": "process_name", "ph": "M", "pid": 4, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_dense_MatMul_1_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 4, "args": {"sort_index": 4}}, +{"ph": "X", "name": "4", "ts": 77575718064, "pid": 4, "dur": 0}, +{"name": "process_name", "ph": "M", "pid": 5, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_conv2d_22_Conv2DBackpropFilter_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 5, "args": {"sort_index": 5}}, +{"ph": "X", "name": "4", "ts": 77575738499, "pid": 5, "dur": 0}, +{"name": "process_name", "ph": "M", "pid": 6, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_conv2d_22_BiasAdd_BiasAddGrad_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 6, "args": {"sort_index": 6}}, +{"ph": "X", "name": "4", "ts": 77575738505, "pid": 6, "dur": 0}, +{"name": "process_name", "ph": "M", "pid": 7, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_batch_normalization_19_FusedBatchNormGradV3_1"}}, +{"name": "process_sort_index", "ph": "M", "pid": 7, "args": {"sort_index": 7}}, +{"ph": "X", "name": "4", "ts": 77575763936, "pid": 7, "dur": 0}, +{"name": "process_name", "ph": "M", "pid": 8, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_batch_normalization_19_FusedBatchNormGradV3_2"}}, +{"name": "process_sort_index", "ph": "M", "pid": 8, "args": {"sort_index": 8}}, +{"ph": "X", "name": "4", "ts": 77575763942, "pid": 8, "dur": 0}, +{"name": "process_name", "ph": "M", "pid": 9, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_conv2d_21_Conv2DBackpropFilter_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 9, "args": {"sort_index": 9}}, +{"ph": "X", "name": "4", "ts": 77575763946, "pid": 9, "dur": 0}, +{"name": "process_name", "ph": "M", "pid": 10, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_conv2d_21_BiasAdd_BiasAddGrad_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 10, "args": {"sort_index": 10}}, +{"ph": "X", "name": "4", "ts": 77575763951, "pid": 10, "dur": 0}, +{"name": "process_name", "ph": "M", "pid": 11, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_batch_normalization_18_FusedBatchNormGradV3_1"}}, +{"name": "process_sort_index", "ph": "M", "pid": 11, "args": {"sort_index": 11}}, +{"ph": "X", "name": "4", "ts": 77575763960, "pid": 11, "dur": 0}, +{"ph": "i", "name": "CYCLE_START", "ts": 77575768923, "s": "g"}, +{"name": "process_name", "ph": "M", "pid": 12, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_batch_normalization_18_FusedBatchNormGradV3_2"}}, +{"name": "process_sort_index", "ph": "M", "pid": 12, "args": {"sort_index": 12}}, +{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 77575785744, "pid": 12}, +{"ph": "X", "name": "4", "ts": 77575785763, "pid": 12, "dur": 0}, +{"name": "process_name", "ph": "M", "pid": 13, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_conv2d_20_Conv2DBackpropFilter_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 13, "args": {"sort_index": 13}}, +{"ph": "B", "name": "NEGOTIATE_ALLREDUCE", "ts": 77575785783, "pid": 13}, +{"ph": "X", "name": "4", "ts": 77575785784, "pid": 13, "dur": 0}, +{"name": "process_name", "ph": "M", "pid": 14, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_conv2d_20_BiasAdd_BiasAddGrad_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 14, "args": {"sort_index": 14}}, +{"ph": "E", "ts": 77576058946, "pid": 1}, +{"ph": "X", "name": "5", "ts": 77576058947, "pid": 2, "dur": 0}, +{"ph": "E", "ts": 77576058948, "pid": 2}, +{"ph": "X", "name": "4", "ts": 77575785792, "pid": 14, "dur": 0}, +{"name": "process_name", "ph": "M", "pid": 15, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_batch_normalization_17_FusedBatchNormGradV3_1"}}, +{"name": "process_sort_index", "ph": "M", "pid": 15, "args": {"sort_index": 15}}, +{"ph": "X", "name": "5", "ts": 77576058944, "pid": 1, "dur": 0}, +{"ph": "X", "name": "7", "ts": 77576058951, "pid": 5, "dur": 0}, +{"ph": "X", "name": "7", "ts": 77576058953, "pid": 6, "dur": 0}, +{"ph": "X", "name": "7", "ts": 77576573764, "pid": 26, "dur": 0} +] diff --git a/tests/profiler/resources/horovod_timeline_traces/framework/pevents/2020070206/1593673051472361_88359-8c859046be41.ant.amazon.com_horovod_timeline.json b/tests/profiler/resources/horovod_timeline_traces/framework/pevents/2020070206/1593673051472361_88359-8c859046be41.ant.amazon.com_horovod_timeline.json new file mode 100644 index 0000000000..7e3fd103ca --- /dev/null +++ b/tests/profiler/resources/horovod_timeline_traces/framework/pevents/2020070206/1593673051472361_88359-8c859046be41.ant.amazon.com_horovod_timeline.json @@ -0,0 +1,5 @@ +[ +{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros": 1592860687560103}}, +{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}}, +{"name": "CYCLE_START", "pid": 1, "ph": "i", "ts": 0, "s": "g"} +] diff --git a/tests/profiler/resources/horovod_timeline_traces/framework/pevents/2020070206/1593673051473228_88359-8c859046be41.ant.amazon.com_horovod_timeline.json b/tests/profiler/resources/horovod_timeline_traces/framework/pevents/2020070206/1593673051473228_88359-8c859046be41.ant.amazon.com_horovod_timeline.json new file mode 100644 index 0000000000..615158752d --- /dev/null +++ b/tests/profiler/resources/horovod_timeline_traces/framework/pevents/2020070206/1593673051473228_88359-8c859046be41.ant.amazon.com_horovod_timeline.json @@ -0,0 +1,60 @@ +[ +{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros": 1592860687560103}}, +{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}}, +{"name": "process_name", "ph": "M", "pid": 1, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_dense_1_BiasAdd_BiasAddGrad_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 1, "args": {"sort_index": 1}}, +{"name": "NEGOTIATE_ALLREDUCE", "pid": 1, "ph": "B", "ts": 8440608}, +{"name": "4", "pid": 1, "ph": "X", "ts": 8440610, "dur": 812355471858}, +{"name": "process_name", "ph": "M", "pid": 2, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_dense_1_MatMul_1_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 2, "args": {"sort_index": 2}}, +{"name": "NEGOTIATE_ALLREDUCE", "pid": 2, "ph": "B", "ts": 8440614}, +{"name": "4", "pid": 2, "ph": "X", "ts": 8440616, "dur": 812355471927}, +{"name": "process_name", "ph": "M", "pid": 3, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_dense_BiasAdd_BiasAddGrad_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 3, "args": {"sort_index": 3}}, +{"name": "4", "pid": 3, "ph": "X", "ts": 8440621, "dur": 812355471970}, +{"name": "process_name", "ph": "M", "pid": 4, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_dense_MatMul_1_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 4, "args": {"sort_index": 4}}, +{"name": "4", "pid": 4, "ph": "X", "ts": 8440631, "dur": 812355472022}, +{"name": "process_name", "ph": "M", "pid": 5, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_conv2d_22_Conv2DBackpropFilter_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 5, "args": {"sort_index": 5}}, +{"name": "4", "pid": 5, "ph": "X", "ts": 8461066, "dur": 812355451642}, +{"name": "process_name", "ph": "M", "pid": 6, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_conv2d_22_BiasAdd_BiasAddGrad_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 6, "args": {"sort_index": 6}}, +{"name": "4", "pid": 6, "ph": "X", "ts": 8461072, "dur": 812355451685}, +{"name": "process_name", "ph": "M", "pid": 7, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_batch_normalization_19_FusedBatchNormGradV3_1"}}, +{"name": "process_sort_index", "ph": "M", "pid": 7, "args": {"sort_index": 7}}, +{"name": "4", "pid": 7, "ph": "X", "ts": 8486503, "dur": 812355426374}, +{"name": "process_name", "ph": "M", "pid": 8, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_batch_normalization_19_FusedBatchNormGradV3_2"}}, +{"name": "process_sort_index", "ph": "M", "pid": 8, "args": {"sort_index": 8}}, +{"name": "4", "pid": 8, "ph": "X", "ts": 8486509, "dur": 812355426399}, +{"name": "process_name", "ph": "M", "pid": 9, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_conv2d_21_Conv2DBackpropFilter_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 9, "args": {"sort_index": 9}}, +{"name": "4", "pid": 9, "ph": "X", "ts": 8486513, "dur": 812355426415}, +{"name": "process_name", "ph": "M", "pid": 10, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_conv2d_21_BiasAdd_BiasAddGrad_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 10, "args": {"sort_index": 10}}, +{"name": "4", "pid": 10, "ph": "X", "ts": 8486518, "dur": 812355426428}, +{"name": "process_name", "ph": "M", "pid": 11, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_batch_normalization_18_FusedBatchNormGradV3_1"}}, +{"name": "process_sort_index", "ph": "M", "pid": 11, "args": {"sort_index": 11}}, +{"name": "4", "pid": 11, "ph": "X", "ts": 8486527, "dur": 812355426437}, +{"name": "CYCLE_START", "pid": 11, "ph": "i", "ts": 8491490, "s": "g"}, +{"name": "process_name", "ph": "M", "pid": 12, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_batch_normalization_18_FusedBatchNormGradV3_2"}}, +{"name": "process_sort_index", "ph": "M", "pid": 12, "args": {"sort_index": 12}}, +{"name": "NEGOTIATE_ALLREDUCE", "pid": 12, "ph": "B", "ts": 8508311}, +{"name": "4", "pid": 12, "ph": "X", "ts": 8508330, "dur": 812355404674}, +{"name": "process_name", "ph": "M", "pid": 13, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_conv2d_20_Conv2DBackpropFilter_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 13, "args": {"sort_index": 13}}, +{"name": "NEGOTIATE_ALLREDUCE", "pid": 13, "ph": "B", "ts": 8508350}, +{"name": "4", "pid": 13, "ph": "X", "ts": 8508351, "dur": 812355404681}, +{"name": "process_name", "ph": "M", "pid": 14, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_conv2d_20_BiasAdd_BiasAddGrad_0"}}, +{"name": "process_sort_index", "ph": "M", "pid": 14, "args": {"sort_index": 14}}, +{"name": "", "pid": 14, "ph": "E", "ts": 8781513}, +{"name": "5", "pid": 14, "ph": "X", "ts": 8781514, "dur": 812355131544}, +{"name": "", "pid": 14, "ph": "E", "ts": 8781515}, +{"name": "4", "pid": 14, "ph": "X", "ts": 8508359, "dur": 812355404718}, +{"name": "process_name", "ph": "M", "pid": 15, "args": {"name": "Adam_Allreduce/HorovodAllreduce_gradient_tape_model_batch_normalization_17_FusedBatchNormGradV3_1"}}, +{"name": "process_sort_index", "ph": "M", "pid": 15, "args": {"sort_index": 15}}, +{"name": "5", "pid": 15, "ph": "X", "ts": 8781511, "dur": 812355131585}, +{"name": "7", "pid": 15, "ph": "X", "ts": 8781518, "dur": 812355131587}, +{"name": "7", "pid": 15, "ph": "X", "ts": 8781520, "dur": 812355131595}, +{"name": "7", "pid": 15, "ph": "X", "ts": 9296331, "dur": 812354616794} +] diff --git a/tests/profiler/resources/merge_traces/framework/pevents/2020071317/1594662614815222_905-b88ab1bb8259_model_timeline.json b/tests/profiler/resources/merge_traces/framework/pevents/2020071317/1594662614815222_905-b88ab1bb8259_model_timeline.json new file mode 100644 index 0000000000..fd4b0c457f --- /dev/null +++ b/tests/profiler/resources/merge_traces/framework/pevents/2020071317/1594662614815222_905-b88ab1bb8259_model_timeline.json @@ -0,0 +1,7 @@ +[ +{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros":1594662597687738}}, +{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}}, +{"name": "process_name", "ph": "M", "pid": 1, "args": {"name": "DataIterator"}}, +{"name": "process_sort_index", "ph": "M", "pid": 1, "args": {"sort_index": 1}}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17127479, "pid": 1, "dur": 5} +] diff --git a/tests/profiler/resources/merge_traces/framework/pevents/2020071317/1594662624558411_905-b88ab1bb8259_model_timeline.json b/tests/profiler/resources/merge_traces/framework/pevents/2020071317/1594662624558411_905-b88ab1bb8259_model_timeline.json new file mode 100644 index 0000000000..0a1d3402aa --- /dev/null +++ b/tests/profiler/resources/merge_traces/framework/pevents/2020071317/1594662624558411_905-b88ab1bb8259_model_timeline.json @@ -0,0 +1,667 @@ +[ +{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros":1594662597687738}}, +{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}}, +{"name": "process_name", "ph": "M", "pid": 1, "args": {"name": "DataIterator"}}, +{"name": "process_sort_index", "ph": "M", "pid": 1, "args": {"sort_index": 1}}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17127477, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17127474, "pid": 1, "dur": 905}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17127379, "pid": 1, "dur": 1039}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128432, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128457, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128430, "pid": 1, "dur": 33}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128575, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128573, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17127376, "pid": 1, "dur": 1213}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128596, "pid": 1, "dur": 5}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128593, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17127373, "pid": 1, "dur": 1232}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128633, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128631, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128630, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128644, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128656, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128642, "pid": 1, "dur": 17}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128628, "pid": 1, "dur": 33}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128684, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128682, "pid": 1, "dur": 5}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128680, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128694, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128701, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128692, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128678, "pid": 1, "dur": 28}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128733, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128731, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128729, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128745, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128756, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128742, "pid": 1, "dur": 18}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128727, "pid": 1, "dur": 35}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128799, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128796, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128794, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128815, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128827, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128812, "pid": 1, "dur": 19}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128791, "pid": 1, "dur": 43}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128866, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128864, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128861, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128883, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128894, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128880, "pid": 1, "dur": 18}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128858, "pid": 1, "dur": 43}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129087, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129084, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129081, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129102, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129115, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129100, "pid": 1, "dur": 19}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129078, "pid": 1, "dur": 44}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17127258, "pid": 1, "dur": 1866}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17127191, "pid": 1, "dur": 1939}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 17127171, "pid": 1, "dur": 1969}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129154, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129152, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129149, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129169, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129180, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129166, "pid": 1, "dur": 18}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129147, "pid": 1, "dur": 41}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129219, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129217, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129214, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129234, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129244, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129232, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129206, "pid": 1, "dur": 45}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129285, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129282, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129280, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129299, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129309, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129297, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129278, "pid": 1, "dur": 38}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129915, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129914, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129912, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17128428, "pid": 1, "dur": 1002}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129926, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129935, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129925, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129910, "pid": 1, "dur": 32}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129972, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129969, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129967, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129991, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130003, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129988, "pid": 1, "dur": 19}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17129964, "pid": 1, "dur": 48}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130290, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130288, "pid": 1, "dur": 35}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130286, "pid": 1, "dur": 39}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130332, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130345, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130330, "pid": 1, "dur": 18}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130029, "pid": 1, "dur": 322}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130383, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130381, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130378, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130398, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130416, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130395, "pid": 1, "dur": 25}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130375, "pid": 1, "dur": 47}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130454, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130451, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130449, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130469, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130480, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130467, "pid": 1, "dur": 17}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130446, "pid": 1, "dur": 42}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130521, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130519, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130516, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130536, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130572, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130534, "pid": 1, "dur": 42}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130513, "pid": 1, "dur": 66}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130616, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130614, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130611, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130631, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130642, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130629, "pid": 1, "dur": 18}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130608, "pid": 1, "dur": 41}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130683, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130680, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130678, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130697, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130708, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130694, "pid": 1, "dur": 19}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130675, "pid": 1, "dur": 41}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130746, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130744, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130742, "pid": 1, "dur": 19}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130768, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130779, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130765, "pid": 1, "dur": 18}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130739, "pid": 1, "dur": 47}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130818, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130815, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130813, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130834, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130886, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130832, "pid": 1, "dur": 58}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130810, "pid": 1, "dur": 83}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130924, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130921, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130919, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130940, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130952, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130938, "pid": 1, "dur": 18}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130916, "pid": 1, "dur": 43}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130991, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130990, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130988, "pid": 1, "dur": 10}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131003, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131010, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131001, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17130986, "pid": 1, "dur": 31}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131055, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131053, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131051, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131071, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131086, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131069, "pid": 1, "dur": 24}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131048, "pid": 1, "dur": 48}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131441, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131440, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131438, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131452, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131460, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131451, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131436, "pid": 1, "dur": 30}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131833, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131831, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131828, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131849, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131861, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131847, "pid": 1, "dur": 20}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131826, "pid": 1, "dur": 45}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131899, "pid": 1, "dur": 388}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131897, "pid": 1, "dur": 416}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131895, "pid": 1, "dur": 420}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17132321, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17132329, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17132319, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17131893, "pid": 1, "dur": 441}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17132358, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17132356, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17132353, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17132376, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133345, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17132373, "pid": 1, "dur": 976}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17132351, "pid": 1, "dur": 1002}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133379, "pid": 1, "dur": 5}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133376, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133373, "pid": 1, "dur": 18}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133398, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133413, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133396, "pid": 1, "dur": 21}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133370, "pid": 1, "dur": 50}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133472, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133470, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133468, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133484, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133520, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133482, "pid": 1, "dur": 43}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133466, "pid": 1, "dur": 62}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133562, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133558, "pid": 1, "dur": 10}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133556, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133579, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133593, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133577, "pid": 1, "dur": 20}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133554, "pid": 1, "dur": 46}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133644, "pid": 1, "dur": 5}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133642, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133640, "pid": 1, "dur": 17}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133663, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133671, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133661, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133637, "pid": 1, "dur": 39}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133718, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133716, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133714, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133737, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133749, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133734, "pid": 1, "dur": 19}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133710, "pid": 1, "dur": 46}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133827, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133825, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133823, "pid": 1, "dur": 19}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133848, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133858, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133846, "pid": 1, "dur": 17}, +{"ph": "X", "name": "dataset::GetNext", "ts": 17133779, "pid": 1, "dur": 88}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19735404, "pid": 1, "dur": 25}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19735399, "pid": 1, "dur": 34}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 19735376, "pid": 1, "dur": 65}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19735464, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19735460, "pid": 1, "dur": 23}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19735455, "pid": 1, "dur": 30}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19735494, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19735522, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19735490, "pid": 1, "dur": 37}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19735452, "pid": 1, "dur": 78}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19821254, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19821251, "pid": 1, "dur": 22}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 19821242, "pid": 1, "dur": 38}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19821285, "pid": 1, "dur": 4}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19821283, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19821281, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19821300, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19821313, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19821298, "pid": 1, "dur": 19}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19821278, "pid": 1, "dur": 40}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19905581, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19905579, "pid": 1, "dur": 17}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 19905572, "pid": 1, "dur": 26}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19905609, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19905607, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19905605, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19905624, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19905634, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19905622, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19905603, "pid": 1, "dur": 36}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19990380, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19990377, "pid": 1, "dur": 18}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 19990369, "pid": 1, "dur": 29}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19990410, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19990408, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19990406, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19990426, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19990438, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19990424, "pid": 1, "dur": 17}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19990404, "pid": 1, "dur": 39}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20075325, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20075323, "pid": 1, "dur": 15}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 20075316, "pid": 1, "dur": 26}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20075353, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20075351, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20075349, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20075367, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20075377, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20075365, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20075347, "pid": 1, "dur": 35}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20160414, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20160411, "pid": 1, "dur": 20}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 20160404, "pid": 1, "dur": 32}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20160444, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20160442, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20160440, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20160457, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20160469, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20160456, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20160437, "pid": 1, "dur": 36}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20245443, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20245441, "pid": 1, "dur": 19}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 20245433, "pid": 1, "dur": 30}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20245475, "pid": 1, "dur": 4}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20245473, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20245471, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20245490, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20245502, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20245489, "pid": 1, "dur": 17}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20245469, "pid": 1, "dur": 39}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20330161, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20330159, "pid": 1, "dur": 16}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 20330153, "pid": 1, "dur": 25}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20330189, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20330187, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20330186, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20330205, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20330216, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20330203, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20330183, "pid": 1, "dur": 37}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20415320, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20415317, "pid": 1, "dur": 19}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 20415310, "pid": 1, "dur": 31}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20415350, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20415348, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20415346, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20415363, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20415375, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20415361, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20415343, "pid": 1, "dur": 37}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20507302, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20507300, "pid": 1, "dur": 15}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 20507293, "pid": 1, "dur": 27}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20507329, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20507327, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20507325, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20507342, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20507351, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20507340, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20507323, "pid": 1, "dur": 33}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20592005, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20592003, "pid": 1, "dur": 30}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20592035, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20592033, "pid": 1, "dur": 8}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 20591994, "pid": 1, "dur": 47}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20592031, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20592049, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20592060, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20592047, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20592029, "pid": 1, "dur": 37}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20677159, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20677157, "pid": 1, "dur": 16}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 20677150, "pid": 1, "dur": 25}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20677186, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20677185, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20677183, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20677217, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20677228, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20677215, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20677181, "pid": 1, "dur": 52}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20762220, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20762218, "pid": 1, "dur": 26}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 20762212, "pid": 1, "dur": 36}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20762248, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20762246, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20762244, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20762260, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20762270, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20762258, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20762242, "pid": 1, "dur": 33}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20847251, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20847249, "pid": 1, "dur": 18}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 20847241, "pid": 1, "dur": 28}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20847280, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20847278, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20847277, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20847294, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20847305, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20847292, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20847274, "pid": 1, "dur": 36}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20931899, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20931897, "pid": 1, "dur": 16}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 20931891, "pid": 1, "dur": 25}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20931926, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20931924, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20931923, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20931942, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20931951, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20931940, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 20931920, "pid": 1, "dur": 35}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21016858, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21016855, "pid": 1, "dur": 22}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 21016844, "pid": 1, "dur": 38}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21016891, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21016889, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21016887, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21016905, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21016916, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21016903, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21016884, "pid": 1, "dur": 37}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21101733, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21101730, "pid": 1, "dur": 19}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 21101721, "pid": 1, "dur": 32}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21101761, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21101760, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21101758, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21101775, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21101784, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21101772, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21101756, "pid": 1, "dur": 34}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21186830, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21186827, "pid": 1, "dur": 18}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 21186818, "pid": 1, "dur": 30}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21186860, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21186858, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21186856, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21186875, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21186887, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21186873, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21186854, "pid": 1, "dur": 38}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21272093, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21272091, "pid": 1, "dur": 16}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 21272084, "pid": 1, "dur": 25}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21272120, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21272118, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21272116, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21272134, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21272143, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21272132, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21272114, "pid": 1, "dur": 34}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21363263, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21363261, "pid": 1, "dur": 16}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 21363255, "pid": 1, "dur": 24}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21363290, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21363288, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21363287, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21363303, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21363312, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21363301, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21363284, "pid": 1, "dur": 34}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21448488, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21448486, "pid": 1, "dur": 18}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 21448478, "pid": 1, "dur": 31}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21448518, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21448516, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21448514, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21448532, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21448548, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21448530, "pid": 1, "dur": 21}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21448512, "pid": 1, "dur": 41}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21533416, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21533414, "pid": 1, "dur": 15}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 21533408, "pid": 1, "dur": 24}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21533443, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21533442, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21533440, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21533458, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21533467, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21533456, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21533438, "pid": 1, "dur": 35}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21618728, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21618725, "pid": 1, "dur": 19}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 21618717, "pid": 1, "dur": 30}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21618758, "pid": 1, "dur": 4}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21618756, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21618754, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21618772, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21618783, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21618770, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21618752, "pid": 1, "dur": 36}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21703285, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21703283, "pid": 1, "dur": 27}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 21703275, "pid": 1, "dur": 38}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21703312, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21703310, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21703309, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21703325, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21703334, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21703323, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21703306, "pid": 1, "dur": 33}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21788615, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21788612, "pid": 1, "dur": 18}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 21788605, "pid": 1, "dur": 28}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21788644, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21788642, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21788640, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21788659, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21788671, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21788658, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21788638, "pid": 1, "dur": 39}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21873647, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21873645, "pid": 1, "dur": 16}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 21873638, "pid": 1, "dur": 25}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21873673, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21873672, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21873670, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21873687, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21873696, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21873685, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21873668, "pid": 1, "dur": 33}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21958390, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21958387, "pid": 1, "dur": 28}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 21958380, "pid": 1, "dur": 38}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21958418, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21958416, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21958414, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21958430, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21958439, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21958428, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 21958412, "pid": 1, "dur": 33}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22043487, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22043485, "pid": 1, "dur": 19}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 22043477, "pid": 1, "dur": 29}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22043518, "pid": 1, "dur": 4}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22043516, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22043514, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22043532, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22043569, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22043530, "pid": 1, "dur": 42}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22043511, "pid": 1, "dur": 62}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22128329, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22128327, "pid": 1, "dur": 20}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 22128318, "pid": 1, "dur": 34}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22128361, "pid": 1, "dur": 4}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22128359, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22128357, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22128377, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22128389, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22128375, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 22128354, "pid": 1, "dur": 39}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483259, "pid": 1, "dur": 4}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483256, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483253, "pid": 1, "dur": 106}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483148, "pid": 1, "dur": 251}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483416, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483414, "pid": 1, "dur": 24}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483412, "pid": 1, "dur": 28}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483513, "pid": 1, "dur": 10}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483145, "pid": 1, "dur": 387}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483539, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483537, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483142, "pid": 1, "dur": 403}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483568, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483567, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483583, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483595, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483581, "pid": 1, "dur": 17}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483564, "pid": 1, "dur": 36}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483619, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483618, "pid": 1, "dur": 19}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483642, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483651, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483640, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483616, "pid": 1, "dur": 40}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483677, "pid": 1, "dur": 10}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483675, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483695, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483703, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483693, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483673, "pid": 1, "dur": 35}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483722, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483720, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483742, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483754, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483739, "pid": 1, "dur": 20}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483718, "pid": 1, "dur": 45}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483782, "pid": 1, "dur": 98}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483780, "pid": 1, "dur": 103}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483888, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483897, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483886, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483777, "pid": 1, "dur": 125}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483925, "pid": 1, "dur": 4266}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483924, "pid": 1, "dur": 4335}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488268, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488294, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488265, "pid": 1, "dur": 32}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483922, "pid": 1, "dur": 4378}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483054, "pid": 1, "dur": 5250}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23483006, "pid": 1, "dur": 5303}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 23482987, "pid": 1, "dur": 5332}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488329, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488327, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488345, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488353, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488344, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488325, "pid": 1, "dur": 33}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488374, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488373, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488385, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488392, "pid": 1, "dur": 0}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488383, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488371, "pid": 1, "dur": 25}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488413, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488411, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488423, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488430, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488422, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488409, "pid": 1, "dur": 26}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488464, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488475, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488461, "pid": 1, "dur": 34}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488459, "pid": 1, "dur": 46}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488511, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488517, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488523, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488528, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488534, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488542, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488548, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488555, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488561, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488567, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488573, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488580, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488587, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488592, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488600, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488606, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488613, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488618, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488624, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488629, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488635, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 23488646, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 25568959, "pid": 1, "dur": 20}, +{"ph": "X", "name": "dataset::GetNext", "ts": 25568953, "pid": 1, "dur": 29}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 25568934, "pid": 1, "dur": 55}, +{"ph": "X", "name": "dataset::GetNext", "ts": 25568996, "pid": 1, "dur": 4}, +{"ph": "X", "name": "dataset::GetNext", "ts": 25732179, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 25732176, "pid": 1, "dur": 21}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 25732163, "pid": 1, "dur": 38}, +{"ph": "X", "name": "dataset::GetNext", "ts": 25732204, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 25893593, "pid": 1, "dur": 18}, +{"ph": "X", "name": "dataset::GetNext", "ts": 25893589, "pid": 1, "dur": 25}, +{"ph": "X", "name": "dataset::GetNext", "ts": 25893618, "pid": 1, "dur": 1}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 25893570, "pid": 1, "dur": 49}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26054116, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26054113, "pid": 1, "dur": 19}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 26054104, "pid": 1, "dur": 33}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26054140, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26216634, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26216631, "pid": 1, "dur": 20}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 26216621, "pid": 1, "dur": 37}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26216657, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26380833, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26380831, "pid": 1, "dur": 21}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 26380821, "pid": 1, "dur": 35}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26380857, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26547117, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26547114, "pid": 1, "dur": 22}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 26547104, "pid": 1, "dur": 35}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26547141, "pid": 1, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26710049, "pid": 1, "dur": 2}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26710025, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26710022, "pid": 1, "dur": 43}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 26710013, "pid": 1, "dur": 56}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26870649, "pid": 1, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26870646, "pid": 1, "dur": 20}, +{"ph": "X", "name": "IteratorGetNextOp::DoCompute", "ts": 26870637, "pid": 1, "dur": 33}, +{"ph": "X", "name": "dataset::GetNext", "ts": 26870672, "pid": 1, "dur": 1} +] diff --git a/tests/profiler/resources/merge_traces/framework/pevents/2020071317/1594662624718694_905-b88ab1bb8259_pythontimeline.json b/tests/profiler/resources/merge_traces/framework/pevents/2020071317/1594662624718694_905-b88ab1bb8259_pythontimeline.json new file mode 100644 index 0000000000..910c2ffcaa --- /dev/null +++ b/tests/profiler/resources/merge_traces/framework/pevents/2020071317/1594662624718694_905-b88ab1bb8259_pythontimeline.json @@ -0,0 +1,48 @@ +[ +{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros": 1594662605425850}}, +{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}}, +{"name": "process_name", "ph": "M", "pid": 1, "args": {"name": "Step:ModeKeys.TRAIN"}}, +{"name": "process_sort_index", "ph": "M", "pid": 1, "args": {"sort_index": 1}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 2664345, "dur": 9020725, "args": {"pid": 905, "step_num": "0"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 11993413, "dur": 87137, "args": {"pid": 905, "step_num": "1"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 12081597, "dur": 83633, "args": {"pid": 905, "step_num": "2"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 12166033, "dur": 84067, "args": {"pid": 905, "step_num": "3"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 12250791, "dur": 84227, "args": {"pid": 905, "step_num": "4"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 12335735, "dur": 84352, "args": {"pid": 905, "step_num": "5"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 12420792, "dur": 84216, "args": {"pid": 905, "step_num": "6"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 12505790, "dur": 84041, "args": {"pid": 905, "step_num": "7"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 12590535, "dur": 84381, "args": {"pid": 905, "step_num": "8"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 12675693, "dur": 84231, "args": {"pid": 905, "step_num": "9"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 12767716, "dur": 83957, "args": {"pid": 905, "step_num": "10"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 12852391, "dur": 84440, "args": {"pid": 905, "step_num": "11"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 12937536, "dur": 84276, "args": {"pid": 905, "step_num": "12"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 13022635, "dur": 84308, "args": {"pid": 905, "step_num": "13"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 13107668, "dur": 83931, "args": {"pid": 905, "step_num": "14"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 13192318, "dur": 84193, "args": {"pid": 905, "step_num": "15"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 13277271, "dur": 84079, "args": {"pid": 905, "step_num": "16"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 13362110, "dur": 84382, "args": {"pid": 905, "step_num": "17"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 13447213, "dur": 84480, "args": {"pid": 905, "step_num": "18"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 13532481, "dur": 84189, "args": {"pid": 905, "step_num": "19"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 13623692, "dur": 84413, "args": {"pid": 905, "step_num": "20"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 13708867, "dur": 84238, "args": {"pid": 905, "step_num": "21"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 13793870, "dur": 84566, "args": {"pid": 905, "step_num": "22"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 13879143, "dur": 83791, "args": {"pid": 905, "step_num": "23"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 13963645, "dur": 84578, "args": {"pid": 905, "step_num": "24"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 14048988, "dur": 84292, "args": {"pid": 905, "step_num": "25"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 14134035, "dur": 84023, "args": {"pid": 905, "step_num": "26"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 14218822, "dur": 84281, "args": {"pid": 905, "step_num": "27"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 14303845, "dur": 84096, "args": {"pid": 905, "step_num": "28"}}, +{"name": "Step:ModeKeys.TRAIN", "pid": 1, "ph": "X", "ts": 14388654, "dur": 84060, "args": {"pid": 905, "step_num": "29"}}, +{"name": "process_name", "ph": "M", "pid": 2, "args": {"name": "Step:ModeKeys.EVAL"}}, +{"name": "process_sort_index", "ph": "M", "pid": 2, "args": {"sort_index": 2}}, +{"name": "Step:ModeKeys.EVAL", "pid": 2, "ph": "X", "ts": 14686880, "dur": 2841712, "args": {"pid": 905, "step_num": "0"}}, +{"name": "Step:ModeKeys.EVAL", "pid": 2, "ph": "X", "ts": 17826321, "dur": 165963, "args": {"pid": 905, "step_num": "1"}}, +{"name": "Step:ModeKeys.EVAL", "pid": 2, "ph": "X", "ts": 17992851, "dur": 161036, "args": {"pid": 905, "step_num": "2"}}, +{"name": "Step:ModeKeys.EVAL", "pid": 2, "ph": "X", "ts": 18154299, "dur": 160071, "args": {"pid": 905, "step_num": "3"}}, +{"name": "Step:ModeKeys.EVAL", "pid": 2, "ph": "X", "ts": 18314869, "dur": 162014, "args": {"pid": 905, "step_num": "4"}}, +{"name": "Step:ModeKeys.EVAL", "pid": 2, "ph": "X", "ts": 18477266, "dur": 163838, "args": {"pid": 905, "step_num": "5"}}, +{"name": "Step:ModeKeys.EVAL", "pid": 2, "ph": "X", "ts": 18641605, "dur": 165686, "args": {"pid": 905, "step_num": "6"}}, +{"name": "Step:ModeKeys.EVAL", "pid": 2, "ph": "X", "ts": 18807697, "dur": 162694, "args": {"pid": 905, "step_num": "7"}}, +{"name": "Step:ModeKeys.EVAL", "pid": 2, "ph": "X", "ts": 18970776, "dur": 160098, "args": {"pid": 905, "step_num": "8"}}, +{"name": "Step:ModeKeys.EVAL", "pid": 2, "ph": "X", "ts": 19131394, "dur": 161450, "args": {"pid": 905, "step_num": "9"}} +] diff --git a/tests/profiler/resources/model_timeline_traces/framework/pevents/2020052602/1590461127873222_0001_model_timeline.json b/tests/profiler/resources/model_timeline_traces/framework/pevents/2020052602/1590461127873222_0001_model_timeline.json new file mode 100644 index 0000000000..d1a28408d4 --- /dev/null +++ b/tests/profiler/resources/model_timeline_traces/framework/pevents/2020052602/1590461127873222_0001_model_timeline.json @@ -0,0 +1,62 @@ +[ +{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros":1590461107892211}}, +{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}}, +{"name": "process_name", "ph": "M", "pid": 1, "args": {"name": "DataIterator"}}, +{"name": "process_sort_index", "ph": "M", "pid": 1, "args": {"sort_index": 1}}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057039, "pid": 1, "dur": 4}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057038, "pid": 1, "dur": 20}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057037, "pid": 1, "dur": 34}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057087, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057086, "pid": 1, "dur": 10}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057084, "pid": 1, "dur": 14}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057103, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057102, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057100, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057116, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057115, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057113, "pid": 1, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057134, "pid": 1, "dur": 5}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057133, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057131, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057148, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057147, "pid": 1, "dur": 5}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057145, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057158, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057157, "pid": 1, "dur": 23}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057155, "pid": 1, "dur": 27}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057186, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057186, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057184, "pid": 1, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057201, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057200, "pid": 1, "dur": 21}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057198, "pid": 1, "dur": 25}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057229, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057228, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057226, "pid": 1, "dur": 10}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057240, "pid": 1, "dur": 20}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057240, "pid": 1, "dur": 22}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057238, "pid": 1, "dur": 26}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057268, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057267, "pid": 1, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057265, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057282, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057281, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057279, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057295, "pid": 1, "dur": 5}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057295, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057293, "pid": 1, "dur": 11}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057308, "pid": 1, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057308, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057306, "pid": 1, "dur": 10}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057320, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057319, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057317, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057333, "pid": 1, "dur": 6}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057333, "pid": 1, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057331, "pid": 1, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057347, "pid": 1, "dur": 5}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057346, "pid": 1, "dur": 7}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057344, "pid": 1, "dur": 10}, +{"ph": "X", "name": "dataset::GetNext", "ts": 18057359, "pid": 1, "dur": 4}, +{"ph": "X", "name": "dataset::GetNext", "ts": 19981010, "pid": 1, "dur": 1} +] diff --git a/tests/profiler/resources/model_timeline_traces/framework/pevents/2020052602/1590461139923994_0001_model_timeline.json b/tests/profiler/resources/model_timeline_traces/framework/pevents/2020052602/1590461139923994_0001_model_timeline.json new file mode 100644 index 0000000000..fe7d5dd0f0 --- /dev/null +++ b/tests/profiler/resources/model_timeline_traces/framework/pevents/2020052602/1590461139923994_0001_model_timeline.json @@ -0,0 +1,7 @@ +[ +{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros":1590461107892211}}, +{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}}, +{"name": "process_name", "ph": "M", "pid": 1, "args": {"name": "NcclManager"}}, +{"name": "process_sort_index", "ph": "M", "pid": 1, "args": {"sort_index": 1}}, +{"ph": "X", "name": "kAllReduce", "ts": 31901600, "pid": 1, "dur": 130183} +] diff --git a/tests/profiler/resources/model_timeline_traces/framework/pevents/2020052602/1590461139949971_0001_model_timeline.json b/tests/profiler/resources/model_timeline_traces/framework/pevents/2020052602/1590461139949971_0001_model_timeline.json new file mode 100644 index 0000000000..9f9a46254a --- /dev/null +++ b/tests/profiler/resources/model_timeline_traces/framework/pevents/2020052602/1590461139949971_0001_model_timeline.json @@ -0,0 +1,62 @@ +[ +{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros":1590461107892211}}, +{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}}, +{"name": "process_name", "ph": "M", "pid": 1, "args": {"name": "NcclManager"}}, +{"name": "process_sort_index", "ph": "M", "pid": 1, "args": {"sort_index": 1}}, +{"ph": "X", "name": "kAllReduce", "ts": 31901629, "pid": 1, "dur": 130158}, +{"ph": "X", "name": "kAllReduce", "ts": 31901628, "pid": 1, "dur": 130165}, +{"ph": "X", "name": "kAllReduce", "ts": 31901653, "pid": 1, "dur": 130145}, +{"ph": "X", "name": "kAllReduce", "ts": 31901561, "pid": 1, "dur": 130192}, +{"ph": "X", "name": "kAllReduce", "ts": 31901650, "pid": 1, "dur": 130131}, +{"ph": "X", "name": "kAllReduce", "ts": 31901567, "pid": 1, "dur": 130239}, +{"ph": "X", "name": "kAllReduce", "ts": 31901628, "pid": 1, "dur": 130221}, +{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros":1590461107892211}}, +{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}}, +{"name": "process_name", "ph": "M", "pid": 2, "args": {"name": "DataIterator"}}, +{"name": "process_sort_index", "ph": "M", "pid": 2, "args": {"sort_index": 2}}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32047768, "pid": 2, "dur": 21}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32047770, "pid": 2, "dur": 18}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32047974, "pid": 2, "dur": 12}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32047979, "pid": 2, "dur": 20}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32048066, "pid": 2, "dur": 21}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32048045, "pid": 2, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32048091, "pid": 2, "dur": 17}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32048168, "pid": 2, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32048016, "pid": 2, "dur": 1289}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32047828, "pid": 2, "dur": 1526}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32047814, "pid": 2, "dur": 1510}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32048078, "pid": 2, "dur": 1308}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32048139, "pid": 2, "dur": 1308}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32048000, "pid": 2, "dur": 1589}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050229, "pid": 2, "dur": 17}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050206, "pid": 2, "dur": 92}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050329, "pid": 2, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050325, "pid": 2, "dur": 10}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32048099, "pid": 2, "dur": 2236}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050350, "pid": 2, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050346, "pid": 2, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050368, "pid": 2, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050360, "pid": 2, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050387, "pid": 2, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050383, "pid": 2, "dur": 9}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050405, "pid": 2, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050399, "pid": 2, "dur": 10}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050429, "pid": 2, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050423, "pid": 2, "dur": 13}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050446, "pid": 2, "dur": 1}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32050443, "pid": 2, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32051648, "pid": 2, "dur": 15}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32051636, "pid": 2, "dur": 99}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32051627, "pid": 2, "dur": 138}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32051777, "pid": 2, "dur": 10}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32051773, "pid": 2, "dur": 24}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32051770, "pid": 2, "dur": 39}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32051821, "pid": 2, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32051818, "pid": 2, "dur": 8}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32051814, "pid": 2, "dur": 16}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32051842, "pid": 2, "dur": 3}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32051838, "pid": 2, "dur": 10}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32051835, "pid": 2, "dur": 18}, +{"ph": "X", "name": "dataset::GetNext", "ts": 32051863, "pid": 2, "dur": 4}, +{"ph": "X", "name": "kAllReduce", "ts": 32057236, "pid": 1, "dur": 524} +] diff --git a/tests/profiler/resources/profiler_config_parser_utils.py b/tests/profiler/resources/profiler_config_parser_utils.py new file mode 100644 index 0000000000..b346a81302 --- /dev/null +++ b/tests/profiler/resources/profiler_config_parser_utils.py @@ -0,0 +1,258 @@ +# Standard Library +import re +import time + +# First Party +from smdebug.profiler.profiler_constants import ( + CPROFILE_NAME, + PROFILING_NUM_STEPS_DEFAULT, + PYINSTRUMENT_NAME, +) +from smdebug.profiler.python_profiler import cProfileTimer + +current_step = 3 +current_time = time.time() +good_start_step = 3 +bad_start_step = 1 +bad_start_step_2 = 5 +num_steps = 2 +good_start_time = current_time +bad_start_time = current_time - 1000 +duration = 500 + + +# These test cases will primarily test the various combinations of start step, num steps, start_time, duration for +# detailed profiling. Each test case consists of (detailed_profiling_parameters, expected_enabled, +# expected_can_profile, expected_values) where: +# - detailed_profiling_parameters refers to fields (if they exist, `None` otherwise) in the detailed profiling config, +# i.e. (start_step, num_steps, start_time, duration) +# - expected_enabled refers to whether detailed profiling is enabled (no errors parsing config). +# - expected_can_profile refers to the expected value of should_save_metrics for detailed profiling +# - expected_values refers to expected values of the profile range after parsing, i.e. +# (start_step, end_step, start_time, end_time) +detailed_profiling_test_cases = [ + # Valid case where both start_step and num_steps are provided. Profiler starts at start_step and profiles for + # num_steps steps. Profiler will profile current step. + ( + (good_start_step, num_steps, None, None), + True, + True, + (good_start_step, good_start_step + num_steps, None, None), + ), + # Valid case where only start_step is provided. Profiler starts at start_step and profiles for + # PROFILER_NUM_STEPS_DEFAULT steps. Profiler will profile current step. + ( + (good_start_step, None, None, None), + True, + True, + (good_start_step, good_start_step + PROFILING_NUM_STEPS_DEFAULT, None, None), + ), + # Valid case where only num_steps is provided. Profiler starts at current_step and profiles for num_steps steps. + # Profiler will profile current step. + ( + (None, num_steps, None, None), + True, + True, + (current_step, current_step + num_steps, None, None), + ), + # Valid case where start_time and duration are provided. Profiler starts at start_time and profiles for duration + # seconds. Profiler will profile current step. + ( + (None, None, good_start_time, duration), + True, + True, + (None, None, good_start_time, good_start_time + duration), + ), + # Valid case where only start_time is provided. Profiler starts at start_time and profiles until the next step. + # Profiler will profile current step. + ( + (None, None, good_start_time, None), + True, + True, + (None, current_step + 1, good_start_time, None), + ), + # Valid case where only duration is provided. Profiler starts immediately and profiles for duration seconds. + # Profiler will profile current step. + ((None, None, None, duration), True, True, (None, None, current_time, current_time + duration)), + # Valid case where detailed_profiling_enabled is True, but start_step is too small. Profiler starts at + # bad_start_step and profiles for PROFILER_NUM_STEPS_DEFAULT steps. because + # bad_start_step + PROFILER_NUM_STEPS_DEFAULT < current_step, Profiler does not profile current step. + ( + (bad_start_step, None, None, None), + True, + False, + (bad_start_step, bad_start_step + PROFILING_NUM_STEPS_DEFAULT, None, None), + ), + # Valid case where detailed_profiling_enabled is True, but start_time is too small. Profiler starts at start time + # and profiles for duration seconds. because bad_start_time + duration is before the current time, Profiler does + # not profile current step. + ( + (None, None, bad_start_time, duration), + True, + False, + (None, None, bad_start_time, bad_start_time + duration), + ), + # Invalid case where both step and time fields are provided, which is not allowed. No detailed profiling takes + # place. + ( + (good_start_step, num_steps, good_start_time, duration), + False, + False, + (good_start_step, None, good_start_time, None), + ), +] + +# These test cases will primarily test the various combinations of start step, metrics_regex and metrics_name for +# dataloader profiling. Each test case consists of (dataloader_parameters, expected_enabled, expected_can_profile, +# expected_values) where: +# - dataloader_parameters refers to fields (if they exist, `None` otherwise) in the dataloader metrics config, +# i.e. (start_step, metrics_regex, metrics__name) +# - expected_enabled refers to whether dataloader metrics collection is enabled (no errors parsing config). +# - expected_can_profile refers to the expected value should_save_metrics for dataloader +# - expected_values refers to expected values of the profile range after parsing, i.e. +# (start_step, end_step, metrics_regex) +dataloader_test_cases = [ + # Valid case where start step and metrics regex are provided. Metrics collection is done for the current step for + # the given metrics name. + ( + (good_start_step, "Dataloader:Event", "Dataloader:Event1"), + True, + True, + ( + good_start_step, + good_start_step + PROFILING_NUM_STEPS_DEFAULT, + re.compile("dataloader:event"), + ), + ), + # Valid case where start step and metrics regex are provided. Metrics collection is done for the current step, but + # not for the given metrics name since the regex didn't match the name. + ( + (good_start_step, "Dataloader:Event2", "Dataloader:Event1"), + True, + False, + (good_start_step, None, re.compile("dataloader:event2")), + ), + # Valid case where start step is provided. Metrics collection is done for the current step for the given metrics + # name. + ( + (good_start_step, None, "Dataloader:Event1"), + True, + True, + (good_start_step, good_start_step + PROFILING_NUM_STEPS_DEFAULT, re.compile(".*")), + ), + # Invalid case where start step and metrics regex are provided, but the metrics regex is invalid. No dataloader + # metrics collection is done. + ((good_start_step, "*", "Dataloader:Event1"), False, False, (None, None, None)), +] + +# These test cases will primarily test the various combinations of start step, num steps, profiler name and cprofile +# timer for python profiling. Each test case consists of (python_profiling_parameters, expected_enabled, +# expected_can_profile, expected_values) where: +# - python_profiling_parameters refers to fields (if they exist, `None` otherwise) in the python profiling config, +# i.e. (start_step, num_steps, profiler_name, cprofile_timer) +# - expected_enabled refers to whether python profiling is enabled (no errors parsing config). +# - expected_can_profile refers to the expected value hould_save_metrics for python profiling +# - expected_values refers to expected values of the profile range after parsing, i.e. +# (start_step, end_step, profiler_name, cprofile_timer) +python_profiling_test_cases = [ + # Valid case where step fields, profiler name and cprofile timer are specified. Profiler starts at start step and + # profiles for num_steps steps with cProfile measuring off cpu time. Profiler will profile current step. + ( + (good_start_step, num_steps, CPROFILE_NAME, cProfileTimer.OFF_CPU_TIME.value), + True, + True, + (good_start_step, good_start_step + num_steps, CPROFILE_NAME, cProfileTimer.OFF_CPU_TIME), + ), + # Valid case where only step fields are provided. Profiler starts at start_step and profiles for num_steps steps + # with cProfile measuring total time. Profiler will profile current step. + ( + (good_start_step, num_steps, None, None), + True, + True, + (good_start_step, good_start_step + num_steps, CPROFILE_NAME, cProfileTimer.TOTAL_TIME), + ), + # Valid case where step fields and cprofile timer are provided. Profiler starts at start_step and profiles for + # num_steps steps with cProfile measuring cpu time. Profiler will profile current step. + ( + (good_start_step, num_steps, None, cProfileTimer.CPU_TIME.value), + True, + True, + (good_start_step, good_start_step + num_steps, CPROFILE_NAME, cProfileTimer.CPU_TIME), + ), + # Valid case where step fields and profiler name are provided. Profiler starts at start_step and profiles for + # num_steps steps with Pyinstrument. Profiler will profile current step. + ( + (good_start_step, num_steps, PYINSTRUMENT_NAME, None), + True, + True, + (good_start_step, good_start_step + num_steps, PYINSTRUMENT_NAME, None), + ), + # Valid case where step fields, profiler name and cprofile timer are provided. Profiler starts at start_step and + # profiles for num_steps steps with Pyinstrument (since use pyinstrument is True, cprofile timer is ignored). + # Profiler will profile current step. + ( + (good_start_step, num_steps, PYINSTRUMENT_NAME, cProfileTimer.CPU_TIME.value), + True, + True, + (good_start_step, good_start_step + num_steps, PYINSTRUMENT_NAME, None), + ), + # Invalid case where profiler name and cprofile timer are provided. No step or time range has been provided, so + # profiler does not profile current step. + ( + (None, None, CPROFILE_NAME, cProfileTimer.CPU_TIME.value), + True, + False, + (None, None, CPROFILE_NAME, cProfileTimer.CPU_TIME), + ), + # Invalid case where step fields and profiler name are provided, but the profiler name is invalid. No python + # profiling takes place. + ( + (good_start_step, num_steps, "bad_profiler_name", None), + False, + False, + (None, None, None, None), + ), + # Invalid case where step fields and cprofile timer are provided, but the cprofile timer is invalid. No python + # profiling takes place. + ( + (good_start_step, num_steps, CPROFILE_NAME, "bad_cprofile_timer"), + False, + False, + (None, None, None, None), + ), +] + +# These test cases will primarily test the various combinations of start step, num steps that are unique to herring +# profiling. Each test case consists of (herring_profiling_parameters, expected_profiling_enabled, +# expected_can_profile, expected_values) where: +# - smdataparallel_profiling_parameters refers to fields (if they exist, `None` otherwise) in the smdataparallel profiling config, +# i.e. (start_step, num_steps) +# - expected_profiling_enabled refers to whether herring profiling is enabled (no errors parsing config). +# - expected_can_profile refers to the expected value of should_save_metrics for herring profiling +# - expected_values refers to expected values of the profile range after parsing, i.e. +# (start_step, end_step) +smdataparallel_profiling_test_cases = [ + # Valid case where both start_step and num_steps are provided. Profiler starts at start_step and profiles for + # num_steps steps. Profiler will profile current step. + ((good_start_step, num_steps), True, True, (good_start_step, good_start_step + num_steps)), + # Valid case where only start_step is provided. Profiler starts at start_step and profiles for + # PROFILER_NUM_STEPS_DEFAULT steps. Profiler will profile current step. + ( + (good_start_step, None), + True, + True, + (good_start_step, good_start_step + PROFILING_NUM_STEPS_DEFAULT), + ), + # Valid case where only num_steps is provided. Profiler starts at current_step and profiles for num_steps steps. + # Profiler will profile current step. + ((None, num_steps), True, True, (current_step, current_step + num_steps)), + # Valid case where detailed_profiling_enabled is True, but start_step is too small. Profiler starts at + # bad_start_step and profiles for PROFILER_NUM_STEPS_DEFAULT steps. because + # bad_start_step + PROFILING_NUM_STEPS_DEFAULT < current_step, Profiler does not profile current step. + ( + (bad_start_step_2, None), + True, + False, + (bad_start_step_2, bad_start_step_2 + PROFILING_NUM_STEPS_DEFAULT), + ), +] diff --git a/tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930988000000_4487_0000_pythontimeline.json b/tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930988000000_4487_0000_pythontimeline.json new file mode 100644 index 0000000000..4449549742 --- /dev/null +++ b/tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930988000000_4487_0000_pythontimeline.json @@ -0,0 +1,7 @@ +[ +{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros": 1589930987655184}}, +{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}}, +{"name": "process_name", "ph": "M", "pid": 1, "args": {"name": "RotationPolicyTest_file_size"}}, +{"name": "process_sort_index", "ph": "M", "pid": 1, "args": {"sort_index": 1}}, +{"name": "event1", "pid": 1, "ph": "X", "ts": 1008335, "dur": 1000000, "args": {"step_num": 1}} +] diff --git a/tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930989000000_4487_0000_pythontimeline.json b/tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930989000000_4487_0000_pythontimeline.json new file mode 100644 index 0000000000..545cb3559e --- /dev/null +++ b/tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930989000000_4487_0000_pythontimeline.json @@ -0,0 +1,7 @@ +[ +{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros": 1589930987655184}}, +{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}}, +{"name": "process_name", "ph": "M", "pid": 1, "args": {"name": "RotationPolicyTest_file_size"}}, +{"name": "process_sort_index", "ph": "M", "pid": 1, "args": {"sort_index": 1}}, +{"name": "event2", "pid": 1, "ph": "X", "ts": 2010841, "dur": 1000000, "args": {"step_num": 2}} +] diff --git a/tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930990000000_4487_0000_pythontimeline.json b/tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930990000000_4487_0000_pythontimeline.json new file mode 100644 index 0000000000..f4f1493eba --- /dev/null +++ b/tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930990000000_4487_0000_pythontimeline.json @@ -0,0 +1,7 @@ +[ +{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros": 1589930987655184}}, +{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}}, +{"name": "process_name", "ph": "M", "pid": 1, "args": {"name": "RotationPolicyTest_file_size"}}, +{"name": "process_sort_index", "ph": "M", "pid": 1, "args": {"sort_index": 1}}, +{"name": "event3", "pid": 1, "ph": "X", "ts": 3012558, "dur": 1000000, "args": {"step_num": 3}} +] diff --git a/tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930991000000_4487_0000_pythontimeline.json b/tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930991000000_4487_0000_pythontimeline.json new file mode 100644 index 0000000000..87409f6bd3 --- /dev/null +++ b/tests/profiler/resources/test_traces/framework/pevents/2020051923/1589930991000000_4487_0000_pythontimeline.json @@ -0,0 +1,7 @@ +[ +{"name": "process_name", "ph": "M", "pid": 0, "args": {"start_time_since_epoch_in_micros": 1589930987655184}}, +{"name": "process_sort_index", "ph": "M", "pid": 0, "args": {"sort_index": 0}}, +{"name": "process_name", "ph": "M", "pid": 1, "args": {"name": "RotationPolicyTest_file_size"}}, +{"name": "process_sort_index", "ph": "M", "pid": 1, "args": {"sort_index": 1}}, +{"name": "event4", "pid": 1, "ph": "X", "ts": 4017124, "dur": 1000000, "args": {"step_num": 4}} +] diff --git a/tests/profiler/resources/test_traces/system/incremental/1591160699.algo-1.json b/tests/profiler/resources/test_traces/system/incremental/1591160699.algo-1.json new file mode 100644 index 0000000000..728011df0f --- /dev/null +++ b/tests/profiler/resources/test_traces/system/incremental/1591160699.algo-1.json @@ -0,0 +1,14 @@ +{"Type": "cpu", "Name":"cpu0", "Dimension": "CPUUtilization", "Value":4.76,"NodeId":"algo-1","Timestamp":1591160699.4570894} +{"Type": "cpu", "Name":"cpu1", "Dimension": "CPUUtilization", "Value":5.30,"NodeId":"algo-1","Timestamp":1591160799.5570894} +{"Type": "gpu", "Name":"gpu0", "Dimension": "GPUUtilization", "Value":8.00,"NodeId":"algo-1","Timestamp":1591160899.7570894} +{"Type": "gpu", "Name":"gpu1", "Dimension": "GPUUtilization", "Value":14.76,"NodeId":"algo-1","Timestamp":1591160999.6570894} +{"Type": "gpu", "Name":"gpu0", "Dimension": "GPUMemoryUtilization", "Value":14.76,"NodeId":"algo-1","Timestamp":1591161699.1570894} +{"Type": "gpu", "Name":"gpu1", "Dimension": "GPUMemoryUtilization", "Value":24.76,"NodeId":"algo-1","Timestamp":1591162699.2570894} +{"Type": "gpu", "Name":"gpu1", "Dimension": "GPUMemoryUtilization", "Value":43.76,"NodeId":"algo-1","Timestamp":1591163699.4577894} +{"Type": "cpu", "Name":"cpu0", "Dimension": "CPUUtilization", "Value":54.76,"NodeId":"algo-1","Timestamp":1591164699.4560894} +{"Type": "cpu", "Name":"cpu1", "Dimension": "CPUUtilization", "Value":44.76,"NodeId":"algo-1","Timestamp":1591165699.4570594} +{"Type": "gpu", "Name":"gpu0", "Dimension": "GPUUtilization", "Value":34.76,"NodeId":"algo-1","Timestamp":1591166699.4570394} +{"Type": "gpu", "Name":"gpu1", "Dimension": "GPUUtilization", "Value":24.76,"NodeId":"algo-1","Timestamp":1591167699.4572894} +{"Type": "gpu", "Name":"gpu0", "Dimension": "GPUMemoryUtilization", "Value":74.76,"NodeId":"algo-1","Timestamp":1591168699.4578894} +{"Type": "gpu", "Name":"gpu1", "Dimension": "GPUMemoryUtilization", "Value":84.76,"NodeId":"algo-1","Timestamp":1591169699.4579994} +{"Type": "gpu", "Name":"gpu1", "Dimension": "GPUMemoryUtilization", "Value":47.76,"NodeId":"algo-1","Timestamp":1591170699.4570800} diff --git a/tests/profiler/resources/test_traces/system/incremental/1591748100.algo-1.json b/tests/profiler/resources/test_traces/system/incremental/1591748100.algo-1.json new file mode 100644 index 0000000000..6038d2ee23 --- /dev/null +++ b/tests/profiler/resources/test_traces/system/incremental/1591748100.algo-1.json @@ -0,0 +1,32 @@ +{"Type":"cpu","Name":"cpu5","Dimension":"CPUUtilization","Value":1.36,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu6","Dimension":"CPUUtilization","Value":1.95,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu17","Dimension":"CPUUtilization","Value":1.7,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu24","Dimension":"CPUUtilization","Value":1.63,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu3","Dimension":"CPUUtilization","Value":3.84,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu2","Dimension":"CPUUtilization","Value":2.04,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu11","Dimension":"CPUUtilization","Value":51.1,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu19","Dimension":"CPUUtilization","Value":4.17,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu29","Dimension":"CPUUtilization","Value":2.19,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu30","Dimension":"CPUUtilization","Value":1.55,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu1","Dimension":"CPUUtilization","Value":19.3,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu10","Dimension":"CPUUtilization","Value":1.78,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu12","Dimension":"CPUUtilization","Value":6.27,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu21","Dimension":"CPUUtilization","Value":1.45,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu23","Dimension":"CPUUtilization","Value":1.89,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu27","Dimension":"CPUUtilization","Value":1.91,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu4","Dimension":"CPUUtilization","Value":1.68,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu9","Dimension":"CPUUtilization","Value":1.38,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu15","Dimension":"CPUUtilization","Value":1.4,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu16","Dimension":"CPUUtilization","Value":1.78,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu0","Dimension":"CPUUtilization","Value":1.6,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu14","Dimension":"CPUUtilization","Value":5.92,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu22","Dimension":"CPUUtilization","Value":1.76,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu28","Dimension":"CPUUtilization","Value":2.75,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu13","Dimension":"CPUUtilization","Value":2.1,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu31","Dimension":"CPUUtilization","Value":1.48,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu25","Dimension":"CPUUtilization","Value":1.49,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu26","Dimension":"CPUUtilization","Value":1.35,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu20","Dimension":"CPUUtilization","Value":1.74,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu8","Dimension":"CPUUtilization","Value":2.85,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu18","Dimension":"CPUUtilization","Value":2.76,"NodeId":"algo-1","Timestamp":1591748109.2321954} +{"Type":"cpu","Name":"cpu7","Dimension":"CPUUtilization","Value":1.66,"NodeId":"algo-1","Timestamp":1591748109.2321954} diff --git a/tests/profiler/resources/test_traces/system/incremental/1591748160.algo-1.json b/tests/profiler/resources/test_traces/system/incremental/1591748160.algo-1.json new file mode 100644 index 0000000000..841d8fe52f --- /dev/null +++ b/tests/profiler/resources/test_traces/system/incremental/1591748160.algo-1.json @@ -0,0 +1,32 @@ +{"Type":"cpu","Name":"cpu21","Dimension":"CPUUtilization","Value":0.83,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu6","Dimension":"CPUUtilization","Value":1.6,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu12","Dimension":"CPUUtilization","Value":30.35,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu16","Dimension":"CPUUtilization","Value":0.6,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu27","Dimension":"CPUUtilization","Value":2.01,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu30","Dimension":"CPUUtilization","Value":4.2,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu8","Dimension":"CPUUtilization","Value":1.69,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu22","Dimension":"CPUUtilization","Value":0.99,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu24","Dimension":"CPUUtilization","Value":1.38,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu28","Dimension":"CPUUtilization","Value":1.23,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu9","Dimension":"CPUUtilization","Value":1.35,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu14","Dimension":"CPUUtilization","Value":3.81,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu25","Dimension":"CPUUtilization","Value":0.85,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu29","Dimension":"CPUUtilization","Value":2.3,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu4","Dimension":"CPUUtilization","Value":1.31,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu11","Dimension":"CPUUtilization","Value":55.99,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu17","Dimension":"CPUUtilization","Value":1.51,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu19","Dimension":"CPUUtilization","Value":1.83,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu10","Dimension":"CPUUtilization","Value":0.87,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu15","Dimension":"CPUUtilization","Value":20.91,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu0","Dimension":"CPUUtilization","Value":1.4,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu5","Dimension":"CPUUtilization","Value":0.87,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu7","Dimension":"CPUUtilization","Value":1.05,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu20","Dimension":"CPUUtilization","Value":2.15,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu31","Dimension":"CPUUtilization","Value":0.56,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu1","Dimension":"CPUUtilization","Value":0.98,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu3","Dimension":"CPUUtilization","Value":2.91,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu18","Dimension":"CPUUtilization","Value":1,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu26","Dimension":"CPUUtilization","Value":1.03,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu2","Dimension":"CPUUtilization","Value":1.08,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu13","Dimension":"CPUUtilization","Value":1.29,"NodeId":"algo-1","Timestamp":1591748169.2323716} +{"Type":"cpu","Name":"cpu23","Dimension":"CPUUtilization","Value":1.05,"NodeId":"algo-1","Timestamp":1591748169.2323716} diff --git a/tests/profiler/resources/tfprofiler_local_missing_metadata/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/ip-172-31-19-241.trace.json.gz b/tests/profiler/resources/tfprofiler_local_missing_metadata/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/ip-172-31-19-241.trace.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..bcd63810e667a3d04fabec3bc52f66f96dd73cbf GIT binary patch literal 3690 zcmYLJc{~&DA1Bw`b2M`{Oq3(DkT1f>6+*eiL~b_ZR_=z$HH0b26*)3@j+D(kS0u6I zOu4>OL_r>;5?J9DM4iRIM|wrxc%ofyh11frBqt>1&Uj=I1)^c832tTqWCQ!6?QM6%L*uB} zqF+6`dl>xlO;StruPq6U&R@rvrC(Z62g%^Ay%x+6p0c~w=d8QaP^9D{2>=hC;;#Su z(tUbR7^6-C0f?~;6X~I24`9@ZBV^qFz3^hxxlhRWgM>5c#1hQ^Ah#KH5((yikYL7G zZMzI{kXWB$cO1tfqoX@QPMh0Y0;9*rlst&1zGq$8{Zg3bcPAG1X_e_$ z`S_M??z+8pSOt}4@!%59=q#g4nuNV;l;@XdX%t4_p{99Phng=+6?3#%#xy_+ZLx9S zz%^VlxPi(_*0FtRZ1$!2k)t3k=#oKRp&J62o8`TnH#?YLNK)e!Gf{#8d^idlqX<@x z1p{h@k|t&+J5AZSWwlPypV4WGY6A(7YEygf^FA&AaK8C?B44aGqH|^X-b;^ww1g3y z2@v5Y?tZh~Y-Ca7){fJOyQ|_JWS7m5K+tD8oOC`Up=PDOWACjdHYjNzG9R2^cWdy$ z@7pKrM2H;7$|#w8o87v1S!9Igtn|qVUbw-7F!h7L$ksW(XpNY>!XC9Z%TT_!3>^Og zKSV0kgQkj_7uB}ElzmMtGw0Wb7nhF$JW}0J%roG~9|#?=a)83Qx>28D3E^8P-jvt2 zryfq8D%Hc@MuBDkkU(GQ6v42b2h6Xwvv_bf5|PtyS72LMbmlzRhDa`{ttwKr4$|h z26!js6L$wkh0;fGMk%5SIQVEN4&*7O zJ@zw#`^V8|9AC9JU8_pAPJtZ{UT}q8+fk@fsdEn1X+K&BpQi*IP5Hl~9q+C}e+=-v znIdnktV|tKwpvBp{OjIq|0Fdi=KPH&9^%PEo13ay{&gmT3XdIyXcwAWW+_v9I}fuR zQ;3JcVMbNiIXp=YPt4D(IvWt9DZ*uj?@g2A5lFaKR2VSrMbYMbS9I5}rGUQ<*D1Tx z)6=6WHg(O>GhIcb_r_A^fH8^7CZ;O~As(N%ubniV^-U6~Wwu9aE3!=M-uKTG>=7>7 z-+yC?ysP0@erKNJ*vs=OlAtxZ=&7r{=F6IMQI72QB$w}$Jj>ezj0*h&Yr%Mx@x>J6 zWG)B2!?VF2PG+>XFk`i#)=e>AT#~kM{?1Os6FUeJ9uT!g{4{x<_`cV8fWGED5kk~2 z)09K?wABEoGxDHXsTvD&w>E4B`A3GKOS_;`@8W*K#RZ&rAl~>%2_Y8T*vX`Xq(9UL zFohIkkmnxTi6j`XtGUg5Z`O2VN#f>(SRk5*71RH{wr&eFf6V+~99l^!7-A-z&$<3BG_1icrx-jUC zG@cSSiOB|S>Q-^phwAvJs2_{O&3O;KSQ6HJ>~c;LWGdC zV7>8U&rk$12P$6+JYb~vu}iz8x0r!Fwr6`JEQ={2Ew`819X3UUTQETFt3jde80;~} zz=+RZvpVt94je5xahLRv;8za9W$+`^zATHOiDc)2hgWBl3_Whe@D)55zy}T-EVY#5 zs^v&%wxD`e$Ib0dyYIcDuhCX0@`gvYhn=1`vx~ERVJ(lj(?2T9?6++})n-rjp4{-e z(;p^o%h%M{D81fA&r9)^y4(;)HxFj3jPitQqLaEIoXWfr9SaM>Cm(~f#sh_LW48e%sDjXwlwoR1;}NOb}$e|KXTB zX8eQovxqr8X^s!?D$Fkn)IA4Ds^2+oWJ?AqTLkwH!$kR?y8BJ_d_YtUf2c?j*?I(d zE^1#S5tujTE%@4xJ=vK0Y+Sp$pe?vHskbq*5vhX+LURH8W?S(Nm(p5JGzcsk1L22) z;}Nx?4oZ>h0iEoE>Zo&Swu(3Se6F6P4&%>RYg|$g#Z1y$diozN4!9}_F{sRne?xZw zgJx~M6o!{Fqj(kP_Ux8cPSa1k@qVSG%;GNY0Jk3?cIbvHqswQ* zt8co9I)dG@s;5j|FWf4%G6XuwlGZGm5@W2et6C2&j?Ypzy4HiYY~U@X<0V#{6B$VB zvAMWDkZuw^pH#pMkd?}Lci|v^I>h?xKmij#!HO}-bgjostil8`Qd~K7-@l*vQMpNY zK+x=VI?Pkv7?~cNn;{XGGCE<~e^hcblC80dQ94y~zqUn8bv!A`gjTM}-N&9!95Xi* z*$RQtMwMmi<3Awmi8=XMy5tfjJ-M<1K*78Sh(*Bm+ltRks4()9DM$hKo-s zEvW#KxUA?!5FRn4@cuTk{Eh@YflkJ)Hh!8#^BaB&uMo#xpMF+<_LovH4YfLC)ZY?#j=Q zp^`+6I!|YeJrD}p2KS-Ni*Rsfu0+}M;op}<^-fK%K zjQXk~=o%bUJX~~@z4o;q6w9q${0@hf>C|yO2s6<(LL2dG=wpE^K;7A*$(=k&y3ARb zOC5&sU(S3U6M+3iOarG|OG;s6`G&LeUz2Y*?(rZIl=Ha+^?iW(LAky47nlG>&Eu}Z z2)%F5vp6@)^Z>7ZC3w6n5qqZb@h);B?t4U1U!1kcv)Md1>Sjx4IFqZuPhF&#&Gozw zwhe_7l1A|wtR86jiaUb)eQR-h;hZe$%wHx{TiZ~H&c~%Lr-Ev3X#AAp51RNHgcNfi zL&WlWsPy|Q1Bn|*cx$fZ%Z`InYMvlDTeHQ0F^K!{utI$w4a*uh!Es?~b&M|qHaK?S zm!m&a(bF_obC4ZOUeV{`mVO_8|H*?`f-W-6>#a)bksVw5W-6GxSrRCA_27K3oZj`D z>5u6Qw;KduZz&{Q%EtoMIK0T?8#;a<<<|0cF8sF~@u{LhpOxuQg@kE+wz!(O@$sJ#+OEf_!PdI z=3#$)RdTW4Y4vn=-RzdvKkeNJo?j@`ald}@g&+XU% literal 0 HcmV?d00001 diff --git a/tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/cProfile/1596585363775342.0_1596585373504789.0_151-algo-1_-1/python_stats b/tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/cProfile/1596585363775342.0_1596585373504789.0_151-algo-1_-1/python_stats new file mode 100644 index 0000000000000000000000000000000000000000..55ff377e8e4d72799ceff1d98a5ba198ddb074cc GIT binary patch literal 420137 zcmc${2b^R@wLcDQ&b#cA)3StR2s`BLv}8nbkf7M~%=Fy3?9B8~-Lrv_rzoO=3IgV| zDh3P>@W}&F5qVD(9tvh@K|oM!1A+u$LHU2b=Tx}2Z}*+q+4cAL^RcL%zEySV)TvV^ z)v1>|CavB0KN=h1e`|L>e6U>WukQ!v`z9BLn5Gj>!!`vyqvOB6{CnZ&~w?yKn096+m;^ zM4;i=Wz)H8pww9{n-y_SxwC(uVj(b7Ip8!D7<@kKv%=s{yTPCC!r;fP`{0VL&XHO? zAV?_HIO+2+LVa4PTC9~<)iCT_4$HBldw%fLCc8{|q03h=?CT~xY#A$KFzyh?!NBp= zlR+3W4~$9`Vtsz`^m1w7 zxN_|%4mVV*qHm10k=ZgeuVa2X^xQr>oLabHlGz+T#NT4u$(7;q5WeTKn{D|0++V(W z$bBE4R*25uvT@8`|6?7Bwa{u`#S99@<7Jy*m`y=@rQTx6n9}NE zKdNQZF1;)?e=XuuT%;13&h4$ zWeXjXfLGO{DlVxzT7JWIK-5)ZXuC=JkD8z!;Fl z`SzCkN)!XS;~`8po(-lUMKrhkr7LcD?u>%3V6Bf_$MF-g$nvr&k>@oY<=DFwdvFBQ z$~br_tyGIRA#}B|D~mmqfdNQG5K_1TJh${*$Z!py@>ftl6^i3osT zEYet8LR)*pM8*n};q3oY28@S#21a^Y9Zre$2EHdCWm!R|G*YWXrZn8IBv%4bYpV#r z0R&)15$9yBY>Gw9nJ0u4o_YETKsaHXb51O^1wn4uo(7QakZCMOt+=;{8)-+jxAKkxDtEd3iM3|h^U;vrC4Cn`P5lNF-qDxzp} zh{u%?112Kh09Vx>NJnGR$i(s0TdI{*KpiSqMdbJo0)i?(?RHyCPS?+MP0C1O-^CH8avv8!;3x8^;o$1w)kP2I7m$#qvOT5QH=q zQcMZ$8aJ}>mc>3(8MNEJ$g>8*)WEO^fH|v;$(%aj$@Ccrf-5U#c_)djDBFVM_z~g1 zrQY6RPidG_#5{60gXzZ<;%#|YY!SNB84a%%QbrfK)6gzPC_Q=pJ2GU?yRjOtJ$nP- zfSiHbuVyL(;QqrRh|cBNI~`nMRa_IMki&Gb7N<`bj_l6sE>%T&*@|eLue(jlQuhFs zq_Gf4H#HP???305z;l$}Ck{#=3*}vU=94hqM^I)CmJQeuHyqX_A0Nj|7$2gL+Wj7X z8>?W7m?>tx9cO(pXRYeszq)({vp#YC2E?+Pa*vn+xUH9AnEIYY?gaM>_>JexLzNoX zO%Y_%Tj>Em0KWuA@{h`i?^eMGRU%WWK`8J?^-ctGQ86%|Bru}5vxpA;zy)9a=q-y1 zz5;AbTyYOH0%4V;v1DsNu`Tph>>y>0LE2QI6>E1e05O*n8gtrSORl(QR>4;~CYep~ zL!*wOK6!YgO7Zpte*YeRuZr4&g0DbA<@fPVxIwo8pNMO*^4@_;6hmsniqVk5!+^#E zwIIAQc*@nkehL-@S8?6^oRqx?+kX)9b-5S~j}$AzR?uoJX8ZS}@J0_`aqZ6nb2De* z6ATx0B$dljlCilAAH`?UsIYBJQDX?80%t4ArdAvxE$OBkUq2tfuw+l1KG{>Fi=bXd zP;aVr9sc^OfiYJBs515xn2+jP6jzi?Oh%#tCkVCAXuM02OPpl4(6gYL#O12{BP?Q=5ba*(4Z zTTc1x((>s?f!JKdwcKJWziN$n{Co{R&lm-;QzkBOhMl4WS*#A1d$?oXOv>F%NUkyj zVFAXrO3`tV#+M1s^#n%^Qi}_|0&q@RKQ>qE!|t$*gnaIY7^gmUWN0M>V=Q9+S+4|4 z5n)CZ0;?#5Xe5?NF%hcGq2W}~22Z0U?E$XzEH8`g?-3C@u#O-s@Y22tdR;>_u&`_Q z=PqACuY0!bwLDmYL9?%1?iIt7;Vi$8ZeY?GDz7XKmaA3B)MFAXMKkj*wwnQI6Be#W zv>_h4dh%{D^Vf5a{mZjmk*k2TXf|xT!bY=-#|-tJUhX*>5-i5b*;!^Ye2-7UajIBM zDfBtUPAI7$D?#Joq^p@g<6wEPyKKM!8*>Qn?I^4QlUf^d8-9q7n2c83M13Oz(6}RZ zhn6F3`5m@A<=*QrKX9XM3ciAwO3P!g=OMmU<7lWvVNu0CcSvGRZ)P7XqK)7Et&2C? zYP*83U=7b~yM}4&PF9Z!JF7AbdhUbN+-+2mkM|%F<`-eV8CVTY?q%u5`2^;F2+U`9 zUHg{n7Hm=Q6-^8;AwEEnG zUmH)c<;;;G*Bh4dWVX5=Mf9u9_Pgsh@0efk6)Z!0JNj^`)(^~*WnCsaRCfZb)DaKY ze2)JU0#hI`v(!S2l9u-bkI87TfM((fnn+3NtSOS6L4zVD9}p0VkV$GVX{dgm136Zx zqwNYQZ?P^QI7l}#BswqO_)3Shi$6g@X<9Ie zLjc)GG&^sA@D;4%>)Rr6tj#TEu};Ww1KqGhP{cGp!cg_itu(I2IsZGb-ppOo7Nd#F z-3m#0D2j*5okQgs6=tsb-ju!OlA?-gsuYfy;NTgn9G2%v%ptyrEobQR49bZc4o}_D zsbfwXox?`70uQUAj&n>v!Oq^DYw%YTR&?3eiOCz&D|NThK8s%)Un107i+LA>1(gX7F(`Ik z^aL(vvX>W8STcUXDb-IU3788aP8iC1keTQQ39tCYoE^VZPHSAz6 z(AYaNTwEy=V9Q{D)1e=AUav>VrML|mzeiDDsJe^To&rfHjyRQrUUJgpORjl*9?`Zy z5TFjaBK?{iJ4}1sgecfCSDSJ~!n%O4%`8?2D&j3=uT(-*6VVeZO&RUaPizwF-ERDh zz0QU)X&BrbSQdN18urVgrEoSIvyAJU>f}C{H$kCPd(8s;CFR$>FafG*yDBR7NQLH?z^S_PUVZ#B4oF7;#B#*l3ndvC{0DY_>?9 zNupar^Ig8uiV#u{$z;FzJ^HtjqOXl4=JeEmb0y#J~<6nq7{ zXGena65GsoZHhhpR8PP=;yGL*7SR#6Z8G!u=in@N727smgkQ0BXyF*7x!N~we1pW* z!N&i%>Yaa_fAErmueioPV&jFfjBr82do@sm(X20C1$=~^*NWi~?s!YeCUZR8s^>NK z;ke5~sFRf3ZVp%Yv`C7PnJ2n~q)WW)qOq zf@LXeH?z%L1q~Zv7#=^W7`a>^2ff3UY7FjH>nE2voimtD!zBNLIqb|?AH_K+>w49( zu2*@uE^E7JAuPd}y2zvV;6iK~?4Ir`3^aoS9nXO>)X~n>4-Fb7xYn-z1n*a>e6q`u z=aXHd<`9enl!fyl^G&-k5%knHyrA(2=kR*Y;S2ZAz3o3gazr6|!@Q5Y>%^B%m9V#o z3$)_HVvGSRlpm}K_g0?d^Q?4f$}0_3PE0IEaRB9=1PJ5%*mU!BQ|#2CzIZGh(OigW zVvsQN2rpIyeOkYR6}Je z#eCwH$CSkE{7c-I1DIcZ8U#fX_i%x8&Tg~2!k9w?=Fj>`&|zAg;36xYNiv!1giR5E zIX=#Ht3$RrHRSA{jI$6Ymh!3PLzR_7WHtF7U*%LVSU4=ny zXW-|?YxvDDiqxWKSR^*wxf9-Sj?h8~L6hqECyZa;&RW-S;y`E7FSwZyexN4H?Fni> zr=uZ%aAdfOd8^JKNBHVc?~s}dCIM-ng4eOf3)$ngJ0Jbh+c)2|;45xdpUz#qa}nRS z-r07$s~sln?9Ab6KA{kXoqf)~-txwq&V;eiRosw!(gk6cWDOx?;bjhA!Gvza4~?Iq zfCfPb+uN`cf@_+NLU?3P1-8M~l$5aKXADfE+0CB$COu>b@-~RJf{5B?(16Vh4;VGn zn?SO8OlTGsHbvDL)h`X#j($ra%&HwGF2=a(YyQf4KRs=3A56aLxZI{NYCfny1xZOU zfk+ca59ZcpxV^^a4mps)llUn94Rxb!Ky~VWv}e@_ofP;Z)(UfL7OoR2wOy75T@Pl2 zH14G5=oj~$H}m+H=QVMZZ4amUskT=Xl{^bPpD`t}%HM0C!;(7XYx$A-M3Q+#B<(FQ%DGY(+7g|BTY{AyE2d zK4}7!@t)x>8WL9%7`kqFM?oqUBt8FV&=y^(K8um9_BQ~r4OgFqr$Ar92{qmTtg(RL z=((?dKHD}|a_+vcxH;3(VbvF%eFl2=IHozSixtE-EzfGd-H*sI5 z2)g1Bu)chP{vH5SpUNfeV?_U9oXO?fF?$}h?Vo>l;LL)r0Q!DHe>oM;c`g>pyOI(D zzY7A7Tm@jCJmC|RqPh3Zi&vhc_%G-*^(|dgs{dWWMJtSuoq&NVELj5)xDQ$0tbnytEai#RD&Vs?UG7(%s; zI8Jy;C4xl5NHV@oz$qHgA0W8aH|0JGfxIAwJ^=d|XUW2zLjQ2u@;|y<>sQ&@#%v_Q zM{;CrOjE{vU>;h|vW9BC$7H>CJO`8rQ+Wh`$N4e_xve2T5Ts-udJ$5YkN=KgS zf>udhm1BRGOeTto)z#G^^$FN<#$2-9+R}sdxI2Sz{0YETpN_xE^{N&fCxjdGkm4wJ zFQ`9WZt%>41OcZbZ0#{Y&fk3E#($ZpEHJp0 z)CmtG7o)K+Nyy59`P=5Kwren!1y9Xb;(6{w_*EH3EANNaAUa4zu-)45Px-v4ngs9` zHjmqD$@sywfIp^7$xVW~lPXZ&T<0p-Yoe+R;uFL>NysIP`8@-rYE^C_Sa*Cr|1jdv zdmZ=d^j>UOyeX+j-Pm)aKOt8;K{t09z5 z9W)GHs45&k^ob@-0(FUIcww>W*W*_||e_Mz^1U^GD&Z31hO$^EM6NHek=M4 z=;;^&K(V7fuUamdo_-%&j;4WfGspr-Y?o(m?23QNGmSB9SRe5<%N5AR72L0nV!)`l z%MRa~bsbo~s{owe;)ljf_@ur`51dpDS~$!T{QB9Et^W11Ir!RD(hB|3U`#33tcucF zuNTLw6wync_}2NIC&8}gDx;30uHm>WvZ1)+us?N7Qrccf+MceYEuq9q*zJ>?jeb^jy2Rpj~&1i8Y_EF3S^c&B@q#Bh35;w&<*2hv6oU2ZP!1}9&Z8EPgV(4(js_lDT!TSE^ z(Ua&*jdJ@iWxE`v3TK7rxl+Jv9TNfIGbp4Hbh^+fK+N5Yw(GLVbS~U zY({zF%eH8MxdbMVUGl1uRNg06Ob)BwPXTtgOK2lbUS*9*y4aERd^ zM4Qv_&FpVq(!}FifBK$(t%khlDr9@>nh+#mmzCgbn~J3I5LbF9+5J5uaC8icHs!o9 zY2KE*EaUclIQLruqA(1_s~&Ol z;>|f{7ce?;e^yAjL61>qX*k9-VrrplXBxusoLk=fUbh^&l%^gP2eq) z@?&^)Pp#8lmGd?vxzh!PQx%W6*yau=I$2mOCZzoi%WoHY2L{4N)&0c-fF86qZZxau zG?udSV@SEWab^^JrQJ>#8KyeYW+!}v{T;{tj{n>yOMi1GT=lNPov=q+yt#NuBpVvu zf4_ss*ks~xx~GqZ%<&1DlCZxgW?^W{G6K#r!V#cY&wKd;rHc_I1RI9j<`Rhnp};n; zQ?UH#@vy6NeC|pS^Rt*u<1Z-c(;$%y=nXnr0n`+r>D#8bJZ@9yah~|CCFjgZT*sV@ zp=7(#`fELixwult%V(vTS{Ws%7Q_`8j7pAk3ie>1!-@WibU+Q-tw_L&F5u*XWx10J zW-ri7sz+V4gidT;L{}|7N!U2dD@rMWakQY694D3B5ruzKG7jy+C7P?0?r!ek>_3sS zdLyY@A;djsS4r=dty@O4P@h6Pi|HX=SlGR`mD_Y%6-Ke8#XR^C{0yq1YZ3F#X^|c{ zi0M58Fygh|rHQr6V4B+7k(g$@DW-`)10+E*l0F7zmNKtogmurfO@k8jNV?YAK$5ds z>vD1?l!T>&Dlp$l{i(@idI>?lEieHl4RH$84V#l+145NPc#ZeK4uYp<^MW1-e*Qs@D<{JC*uDOf;^ci7!l2s(;y8;Fn;!6 z>WcK%KqMA2GdBXUm_WQrYaK3=4)v<;9u_~-CJo0jj}KTuG?cPhEGj%2n~GTPBu*Mt zjAsV8w_H@4uZ)^s0~w$_{@mu9^zb|Q;c)>$M{OD}z6@PT5xwu`g>QcQ8HGYr`rKC^ zIq}5fLQGefZid#)eo7(1VB~`+KKl0^zqIvtW|9paNhF@YmE2LqwxEAXf(2%kV^TG{ zw}Kf*(Tl6`Q|zT`vq;AapD267#V=<-HizjaVNZE~v~wGEOfshcqy}Z|`aE8_QVB(F zs{z5GsZKa)M|uO=;FCN>_EKG@uxm0b*bHee&C^Jz!#MLMew{$hJl$!<7&$y}p-OV; z-N^}0!h|t{URhG%|yQa8BxY0vNR$$9thTQKic@ZOLA=%x0y@hZnq$DK+PGN3mh z^kxhxK)Ptf_aQQ-J|CtCCi*M(m60`3ll0_~*{urD?F5L04;J^8j=c1!xCANAZ%{u) zs>j&PS<+==SJp2E$wZfZ^Wp!y#Nb4B6+pZiKQvC`CnHK#(v*ew8Tkqbo#%aH6$;q) zYkh+&Q)!BLy#ZJ2n9B{AYS#Xe$GkOMy~uG{c(u$|Oap&LFJcFhYsNnLt*bbtZ~>@M z?UmCY+y}P?Bw8yaH?f|?sJhJ) zfehrl$76;uW5bzM&Kjh3u!2wsna zQT`XXN_xUUJcLkJS<#w48~dZNxsse=ikv=)WJM$bvIdD8bXNt)MA~o?w_StyuU#f0 z9Mcd6sS%=Y#b!-zUX(E3%(+Yej|BXUH*ykVTaePD2mBX7`d;l z20oUvO#+x<9)aDFceIhxK->jOId9r~6cx@*mepS$P#xU{M^x80j=7aW?=#q5=7X85nI9xhwSd~vC^ zc<)sQ98lV~Z=e19_U_$(Z{9t4Cx($Cy6&Af-Z*kYso*Qn)sTx{gE+gC)9oM=j_LlA zBKmUY>km8N!-o}o1+$@(#HO{hSt)HAnjEt3umbt}Icpsy!CPy8o%P3e9(HyiautlS zn4|0^^1BVHZLty?mQ#@=5iPEPaKWT?2_VVB+=4jGXSvP?B|Z;gq8DXd1sItQWHipk z=eW=kP6kayMGtPhHUnm8W7Mr%&eEm_44hmni|(YW(IgTS z@rVme4jP^F=pMH}|FJG8f#3V*+onx@EiEx?k?D6ruX!MlVyKL*fOvTbYf!RG8E<~1 z{^j1&AccdA43q}DdrLGthxJq}BvdevJZuJ<2HM=L=S(2vR-falOZxNRd{=5o2QMH5 zf1}20QQ+c!wJ3HBSiL^VPHw?)(Xk)8@8)lR1x{U80gl?|mC)2w!W4K=tpu83Q0hm| z)e&PV$nqjLx^vtGW^WAL0w)Q!1)PYva>y{s+%JLXJp8|Tc4UPE%-^ho=cb(r6GIlqgDrUkBb-n0_J5O&kLzaKP?rPO8Q zq;8`;FyAiS_?PT1PU%GOp87m(n>V5KI-U3LSSQdXXY(&2c@BtZ9aiW2oXW>ZeY4SU zob$jQe?4|zOs}+hJM9h3d@?mBU@&|V3=vbTHP4&%Zdjk3la#+{TxE`MJ|!h07b4<; z8_6&X4~C}D>Kk$fzMRv={1~$Xv_azO{25He^7_lm1Xl{wNXswp8K z8Lm^R>x@j~h&+s*{lpsn!kM2re+-uo0|l*BrKyBckfz`u{)VTZnhE$&I)zw7FTdvI zlRx;ka|_X=4?gp^{g%B6R)rzuBzZ7d2SMH^mLG-5w?chC*Zyu4^_eOL9>^9=K1w1ksM-kAE*lMhu_QpNHW_1g<>p;)Hf;J=6%9@BY>irIj6VxLVTp# zd$xnZMib<9Z8uiHfxF>n0|?=N$!sjA?nka-j&24pqxXUwnG-^i`%9`byvD+P2S@5R zAOJ}bef`qf&GF|?MJ5MZv8{yJ*BA7ZdirhlXiX{#Md{ zw|j5};66|cQyKEwwU+F|R?P4rD4-e($GA(S?~x%W?lFQGsZ_dL(21pLg^7m&bo?s{ zJhKpiXBDF{V$##I3CAzIX#?vqkuAa8aOm+~Ja0l!+s&DOJ0KF-VwzZJV(Vm>1?|Fe zDDBU^Ge3DK*#jx;yG-LWC}DLqcV?~BZ36?_!;?85Mzahq^G$l#aE9^I+%PFMKFMP{4+&q5d00zf{gwy1 z$gBekP`oCdV=8rPhQgK2Amt1n@gF&-IWf&1?t&&D4pxWEx*MB6DCi9?p?@;7{}Jf6 z@kh)J=d|Q-1*WMtH*@a9S(YZKUA#e%M~6g^x1=huRjR=<<4>cNY`lpyhL=a)O<3*J zEM&_K_DTJr(!qZVns8eF1}ipMGgi? zm;;&>G9l$;#!F)_a7AcdWgBVL!~o(WIS`Ae|FADSb?>r+abnmCF}HhLN(hP?Wpj& zYycARF%J`sX5Vwq|9WH{3@j3&)5fe-?Z<`o~IgF#Th)%fa z;7`2ti){DdPEXxd6EiM;9=I|+qf*$)?y_so&tITAI~(g4)3FfCX~j!VL(Y_K_j8@#iJ$8?0V z;XFde!8bRhMJ6UWa{Jk6inKw4z|=D51-7&_xS9>7Q!>yDL5Wx~g+5_6aBV)uHq#*M z+a^I;$iS6Gd>}K^p3LH(gQG|hbzS?Dp@ki2q1+|j)Y#8t_T5U)YY`cyy+(;C!M#`T$l*kZ-b?`d6w4bW~_nt5r=HTCJ;~*>4FQwPw81Cg8aQ4hT($biQ2f{r?jm85R*v z&Mp5~N45Z0F;D0UhEX)!wiBWA5_j9#6s{50LNG|*MH}rZ(2a|L^`Wqyh{Kufga-dU zfq7UV3!h|mW130=Piw!kBt}yMH8TdD7C>cAGY3>)D=iqDlGzWjDCSF8^A;4AWZfI% zJABeS&9p?k6Z+A2s-|=!|H>F?jN0L4gfASm)*)t-%2;Hdg@I<4pNV}1T@H%K7!sH>&Y#N}vaoi+ zja=HcjksY2!MmV$`&xmshz}sVPu?!gVG?O{CpJjaWA;tLMEoaOlvzYyy<+&<)5_-* ze1$k4?|Z2I6;oMJ>FrEhW=muhQ^P^#npP`5O&?n0BqDbTwc-6zRGHZd(2ph_IvK9{ z$(31PoB87O))m%IOl%8G!u0 zKsLS?inK}=92h=+$5mV--bONS0CIq+KF;)PBFab3yNY4_1HcICT2>^uQz7kUFD&dK zfdq+qT5M4ORkpt(*hA$ro#cqA+?=qUvw^UR=K|_@1wQ9PD^Sq^g8Mv|vOm#k=>Xkwbsm$l{XK0SSM8#D$n7VKq`x}N-ZM3Ps92;vN30pk?3MYASr zfjt&*hyA0xC@pn~WKM=upRBHdh8(#U66ZV3J9_i@cev2#DozXy+Cjz;<~$d{W|ukt z0j9`$ZvjUJ6(W~167bQ@0AM+;6kf*ml{EJcaq_s7Rpq2!QBnr&D9)8ZV6Lx^`KB@UW34(=>-0&6m z#7;ARNpjcBeMO}iSaVz@J^!G!tF5f%%*`m+x)jl)bK)}}?cJl`E0pKfJz6y{cC0o$ z(ux0vPbgfk0k&NBo#y#J;Le2yy3G5V51yD~q_|&=C1`OR}ijFFUI8%+o`KNFO_To4oDR zyUuJm&`dVEln%$vDj$`_-Ap7b;=kKd))dE$b@8_>nB$eQf?tS%rS5WT^tAv;A)8a^m`Li9Jb>H-J3LsK z6bZSq&NK+}LIB+{tI9nih%ip=3%@Phjbso#lgy_$OW4J|g)2XQ{2ickO+LAl82z_I zlp2XBO+gWMs;1?D?@Z0qqNZ%vU14LcQ#S;fcSVp^XTP2r-Z8 z_T(Xw z@rTekGU7nwp}Q)qg!;HZie!SCsR|1?`r)7Y&#T*87JyGSx20)>SpNu2>2 z$CRx-Vm`4jxWS=eHa)#BDxq9qZxcM(>1k$1z!A+nHUk&7EF(^oyhU!SVS0 zOn#qYR&F0zgWEs(X4#$1(s96mBX6IOWP{T=ZvP>-2U1NA54Sit0-d{*N6{;Tr(FH( zr{f?Lc_`E!Q!Z50*=kBqPOv!=dkmEm)M^gTp zax=_f+4LAdaFX(%69~k5C@R)?(OG7~Uy)LqhYsj+J1KbyZ^NGMj6T5KPTqD+twDgn ztyFRsZCDElq`)?EPJqH=LN}2p4(t97MII%zl5qkNIPRJ}e>UviRVxF1N~*gyHcKcf zVejg4AoA*zd#}Izz>T&cHE!60&`M9OcxwPd)g$DtF0qE!mh2dX@CqgC+E!YLYZuMLT5YTTb2g zkpIDT<};gV&CAJ;U;fBRxBlhSLN-Z&zm|4z--FlQ`G*s+ImBOP=FlCtmAc~?5CNAN zUOlVgt`Sq*(FwnZyqa>0%@u}gIN!hWojaPfcQrZAwr@W>IQuxQNf=CuXv#hh+|s$` zxI(mO@l05mE)-|kk>gw}N5K-ER%IGkZ%Sx9&c&Ik?t1_q*n=V|eeGlgKGtU_=Od=B zZ)5=a8eRlox)!tz4tRY*7BPux5L-|%*Ptan37&3u-zD1d#%b67?(s8-V2K+Of9K}m zC93M`!EU6Spu?1bHVg>UsIW=@b>s)0dUF4QMf36GUlUJtN6EEhOlh?hTH}w<*te)y z?XQds$Yt*$!~B@B0Z;^Uzh~~wsEc{L8wA(*GZ8xzXNC2xqE!$f^lU<2LZ?-mrxZb9 zcY6G*l=A={5MGZ|waH5~cyib6O-(YgC10zde_3&jS0(gaJH(>0)on*8NhSFT*M&$VAV2$NY4 zWiz~3#>iG&$S5;cVElhvvn`k(T!KrmYP$CkzT9-H&od;qAq*3?1L78Z|J^@(-GATy ziA@Xb?tS|3B@)kiKhSHF)`mb2ibdxU%hP&Sn!mR!VX62YpaNT;dyUQSMHqjo#u5~l<7Y5LyL;!W+OPgG`F-mxM!}Oy5 zdrvNW>B^%E(cM>Ued7N-eOeP6+o`?m;`S7yeSXaTc4vQ`r=EVn#wT_Ze1+k2=kw=} z5~Y7@UIAq?iMT<~$;30Y2jZ^|;`%wcpacTZC0Cyr{)SZzB6KSdCVokP0*239jkm;N zG2p9(EdB(A4ZeR7LrBJpTTEla(>JWJr$|=;J|-T95%6JmyR~@4Ori@HC!{+Sh6Z9u zNE%|1mDE$02-kxtaoA3I425d6lS&%rf-dTFL}S29aV~B)Y#`5yT*W-4^m9K6iiHno z_7#xZW%wapwH5l_0)|@fH*jCvod)l7AhcM2&#LaEgG9RcByK^}r|MxcW{cZ6=!Ynv zY+3Y&mM*f}Dd^)p1e}FEbnrN^rfFS3u0L^utjc|)D}5x~h~66jg(u%KNoEor;b478gmVUzL{T0WA~I%AGj z)zgpHwx%THQMZ?D;<2B1jhg3FAPIak^G$lVB{_@`Rv0L}=^_7e^2$%G`Z2<(TxGOl zQeH-1c~VC6ZcmX*V!%ZJ&UNB@8#iz;_3~Cq8o10{N6o3t(qHDA^pH8|#|2{!*AL2& z!ivhsNUZ9>KG4LlF{EJfyw6$bHAc z-qhT|J<>{#jCn0iuEq`c73qx1NO+=qD769_&w-@%u98N_4(-Wq$E$9x*^a>& z6kC3dB9Ek`;?o{QT)Duo=~O;mKz5Zh8gp14;IUjK{w0z|ND)1B+|Wt4?{yIZy?_6w zAME`?phoKXz@UO*+gXta>2t00KS9Ju<&|} zJ?Zcc-rIHa1Zb3vrX)LK&^G>G66`!0Rr)Kv;eooC7xAR9h5+2JRz$-*qG}s?qhctWB0jv&Rfv6JXa?mKv2c_6F+|{6-V@*|X!8ZM zF=qg3#Ocrq0fQWp?N<<|2p(X22utULDto*=*kwMi$PJ%l56RP%*1{(`)kod6nMb?K z@IoRL#4aG3%ypiT_U6a?LanCoc)5 zFZ(78u70z-3XmT_dyZw)qKWJ<&Rsg|vXU0SXHoMeb4 zv%H9wrVKkI?~w(Pbt`7U{T&xv%r`fJZrY(F1F4w<+@s%c5KeJdlw8I2TSvd{m4^Je z3m6BIVQnkogCrLx)6qNQNbxFSNeUAg$go(_&v)1l?zmfRLQbnQxwl!^gK}oUSIn+5 zq4+IaXco~4=kNRe5B%i_(2lLRp0A*}*SlZn(B_SHTK7|swOS)F&R@#a-QH$LlpBA; zNXl9Rq{Fz&j9kM=n1-J6w($Z5me(s|7SRI_6u$TUE8tsn6<}ZV+{Z-E6?E!FI-UXH zn90z(v8i!$_ABr}Pnh`%X$=^Ll*rsWEuWl8NO|t*k{l)5{1pARKqAB|cdnVUF)9++ zGRw0Rk%oz3om>T?@UCs$k9OnQ3>b2gRE?nVi$o@i=#m|u`1Y$;9aZoZ%vkJ^{1wg| zVQhLKU(Q6VR?JibQbfO5^THbsI}HhuTxI+s>OiHtG$2poHm^{^+1bKtteYUg=V6#R zsVxe|cZ*!C{$4eZbH<#an?8p&#ywXE7fu|%12(IeMO0ce`Krp_F2*&tE4I1yBcC3H zmTBWrayA2&d*Y@~=9g6V(iy5YKT2H0nzoZ163E6@qspMp#?5G-gf~`rHf&3^xDS{F zP3Orw=QlG`J?#>2`kBr2lhBJ3+N^cpRhj_LAb-0U1#i9E2N8N|t1x&kK*^-B)$&vRW^M8#cp_|~lJb}mFuz42=cKm7PE6!OK!&*K2< zs(NVPfwN}ia8kA6<0UkTKgCEB*J?&4rs#k))JD%vRfrI4}=i|FfmbI~R5mK%(s_Xs8zc#Wsj z(}RF5S-Jpb=HS4rM5*Z^Ru+gw)*A4|jtv?NGKd$OmCkt|*SrlVHQ(W4VB~=q4ti$2 z#Ov(p0}Ug$k_0c~dUQYvgTK(jjR(6$Z4sX;#y_0Iz#_O&T{0PrFI}8A4_(Z%=|AX- zMYDGiFcwySgEXnyG2?u-*&mUG2nv(+;W~@GI&*P0>;sfY5$SD5WNL5~*Td#ue^51h zD$X9`8aLj~;lGaEZ#7ul2UJydM`BH&K7ft4@jE^#N4RdxahyC08}P0&QaU3+86_LmWp(&9Ar*~U>Q>T`u8y)uhs`5>2tMfAS= zdtW_#4x*u4#S8=CxE%KbBL7=N{({F3xbaV4MXqdD$*Q`}`Rs!!u|A?tpTe15i>}b) z-`L}~+2gMsy!x$u^O670RWR$*@ny`H2kO(H(q&@OTAcGnk~y5jCYy;C2v-Q(NkE%T z<&bk;o5dIrM4ofL0xZ#^cOc`?StPcyf|OL=Zw4$adQ3`!)DrPYILumclUwjn{3i@r zpVEi8eJVhqfpU<9*$q%igD+6toyiQ1M}c|B*%4ZK-s&pmR)r)SNeW3q+(O5jV7X+? za3tBcv;j=tvxR*E{u?dl!vZtRx8DAv%E}*N*{S}9lL_p+&{Y@h9 z&nV2tbdg^q?XmFhy7>xbc_4mh^m43u7sta&Ll9jw(kX{N%mHXK#)40aB0@$ydp+f08_D4kJ$R3BJUWdiqor77KDny^ zd<5G?26|xoCzxkE$fnlb4kZ#1eI>-dFtey*lA#k9J`v=C%(H#1PR2SGC+-9h)MqgU zZMd|Wmh2YGl@MrcYPCf28J$7W6!yrNi;g7{3)LWqj~3&BnrMi@UZUV8h1@+`Gou9S z0Ooe$jP4*`n2b!qt)`Ox{z-l>kG3rdp_{r??u~9ByfT6tq#`p0F(qRvaiNfb(sb*m z2NXltiA4FPAd05IMPNP*plk>a2q}JpD4y}khEDEb7hm@8 zt@{6L)YHc~9TwJJ{-`_j;kB@qvz$X21S6KS1tf3@WrFX%2TAZXJx~BFgGpyR%jZ0j zR`J)EC6c(2d04K&?Ae+^+uw(B6--zxTcSD6w>B*a^RjKlgUuq+B#_&=Y>?ZLjHK;{ z$>8UThcuZCJG6A6hmot8E6^b%@V@}fQbc%1v1>a-`sQewS!s2 zrQMadW9E=({1-`Rt&(D(72RG5<;BoUjExdLaYC!T4=c~joz2ep1_Bf=QE=o)OGI-K zJywNo-g5rror9&^kP&+}fL9DJZl;eBP!|4@sIP#IKFeR~ynk zM>Uh1DDLLkDIOcAvi&5m_(%@`Tpe?jG$C!*KF%WU1sTQfXV)yEs~-AT?Tt&2!rfIy z&Aae-EJ7R49 zI`jAd!pkDcIynv5l9QWE-IMXs@=NdhZM&_^02Ugq9|IE*)qQDGZBzP>m zPuy2bRWTUMkv=@wRm`8!61}{iy|A#>mRGubCB39MH?ffabO?rvp&mxM#nr_?6^m%_ z%6+%#z2?$F^rgE#{n(GTey>Cqy=k2lguHvYc<(Nm%vXh_X_yYC)_g`KL~m@w4f~3< zz_DVVPJe@Zf>5p8@YqQ~l}b{ZOsg)m+6Hk60Tqb5=-&?_?mVaPC!C-SaOqY3cLQfr zD~%U(Y2b$b&6s0yo{2#=`u_m34M2M9b;rKQJ(}7K6UWzjF^|@OOb9(``nq6!Rxt~e zCLc&q)c7~z28Lz5_T6|d%&8GTo;F`!KE7UgNwP~Zv0c4hNe{9>tUray1^sv#1kn0C zNa}GPvKoz7Bs&y`dQ3k?T2~EnoVRm0L%>`%9T4bM*&}ts!xL(w-?XAwHPt6uRA8K zJ$9V?Iu&sYnJgM8*Aka%ku++;p89v^pMcpk3Hq{q%Bjd8zh>ee~%EBjau0@3;yZU1GC zKU?gYeg58j&_>*JlT>{;HH-hjO4jFyUBbC{fv}v0AkKA7L6892-}ZWv<1T^~cQ9t^ zQ0`Cb5N$y!32D?EhlVK#GZq~Ptm_)E4@WxPGDv)hyJ-@zPZJt;__n(FTd!8f{z$mwuV4>D8?n0b}^k%f)_#(mN$ z5<&QKmpNIHmot|RHRzZRpo9AS<}2jr{>_{L3$J?mifj8tf;20Ua{|pq*o=iWA0&b~ zK~d|jAv$}I^BSe!;-DECuZ)xi%Gew6=Q%bX@=3wrLOi_r&9n&?!kt}b&UC>-!E}@{ zC76#j8=$hEWG=vGKRE-`0&sISqWPo_6askE3cay2M{i8y)4I#g8O#F3JAN`F{zO4JjbO*5(Z6}!3!6~+Q_gZeAxy+iCe|<9A5#)K8ehScBT4UX@$^(ks;RL z$-ka0Ps4BaIwTD2S3dgIJs!MS4D4*#-~m+Mrdo@8mV?bZ`#l14dE3&yh09kG4U1@| zzBZ70t&sM;_$SlYjc`97Tf&pw-`PwpLB>Xdc}O=qV=FuRS|E)K&&B?b&;JSUKw6{; z21Fm~5K@b*$1}RTe~cGN{Z&^yhhQ%sOY)6L#kdbM%w-2__iB4Q@}w7r*_jAUdp-79 zeR8p9kf(Mti%T-ie<}GklSwM_U_EXbyQ!0CDa+V-uiXOREpom;iiGllFurI3ZbCRR zYns8$$RSu>iF;${E5Za<$7SS@aZzhwni9Vz(_jh zlOijL&kN8Hb6bP|Na0fyM4gYUz3I1KI;LPgt0)gVu<5985DEt9#3#)L6!Y9QC8XgT zz{5=FJ#5AdhjmR~MpL(Un>WG(UDhM;tRLy|dExVB>P}9tM`3U7G4V-<4OSE;IF%&Y z$NL0fwi$L~%GxOt7#K+`uCqeCqN|t(L44>XFq8}z!o>T5^v36jdFvDi{;^in$V7i~C`k)%pM%U-6K_dlGyFIFZ4w!3>+c+hflt{vIR2Ru26z>5ZMwseJ>9{oIrK-^3#_}nT^ z6yYiiA#Vyz2s8JGg9|!vy?iMFWBA;=wcMRUGoZ76kxS3Q2Gk)Y)_n_xGVzQX^2A%G zMu1d?kS9!;t_Dso(7z84lzPhjm4RN_ph24Sb8{6Sv_d!;TFDs0rh7L*?3p6cEpB7$ zRcc1tuIIVYgJibI#{qyP7LvY2h6L5;_f()&+Gmq8iLNsH6*G|F;jeNAi$$$P zP>Ld?)6lxe;MNryG!~sgBC-kaP5hA!!UtnbzdNlSB2FhW2sVYcR@?GD*o^Tn$Qu^e zyxOqe=-prL&KdmC#EApJZl9t<%IGDpMExrV9pV^LB3~6~3TW*nV$?Jg)KkYYd}`ap*em`4(%xL4pqav?{~Y^%jvMaIQRFX`s^=Jv~6d z@Q_)>j`d+pdaA&Ni^smG&o{n6K&IQEjIsCs8qi_m=FkQjY>SnFtmhawXkXgg#x^r# z9BTy1!8sa92*F?3Xf`!b!IEY*khlUC))a%o%AHzhw*o&jN^Cc;*KJQPLx+o zn|b`pg@UgDX50pX)riO1h?ToY1*!7x9juO>?+TJLh~bR|!yKob;FB zqj)AMXhDKGI=0yO2VuUOH2?KWYd6QAN5%|SF;_SD?++T~MWSa*|KDQ&OL3XGY=&UP z$qZD40nIBASdDOYx$vJP(My-df}8O=BQJB|Mi?Y3w7ZQ`%g88eI1Lt7 zKTB{;2jlr@RiDX=JpMguxi`qd9tHDV=92jZ!~6e$o@N*gQ_YB)4xAJ409<_*WK5(P zkUXFHlQ!FG&q4Il^E{dR;=d=$;9GKV7XGO66*qVfe$JxKla;Hu&!=n^K3~J$S!8jB z^ZUfMX27v-Jtr8Rq++EkaUzD13#=|XD>9P;O-BLuQdkja6Bli`Nx@zmOAxj@u00Oi z#W7wh)4OHgIDPsPn;wV5$yKoUya{HG<91?l4@MvB*kQ*wB-!)gK_0XuYr{mK;+9Gd3(Mc;IYu|HU~a&lfsJr4Ri~Qp{urC3C5Oe2 z?*iISF{OAbz=ijKt0krkyclmw;SC~DNOP#&7^TUuojux|XT^ZHIFF!VIV(fL+@P1s z?nGzx8CZ`KD-oWFN23>s9~O2MQi`ND`XCo^(m*A`=Rt~|(jr~PZS3slD0uZ*W3kaD zteAqH(8sXi(ngGPY$JJ)(B^$yLKZ=o(8e#3N~8@B!uqthDo+%pW|A>O*(MDM4nTqy z=Y&A3eMzP){3AWSVq3+cWgUf2^5*HTCj#cGy5deObRKBcqubH-4SE$YehcqSOu4vhDwHU; zBs47i4)+zH;CuX4)@OF{b=4E$i*Op>RYV<&=v%A5eALk8+Z24owc}+tcuh}&zy5VY zX%)hgv8w<$j@9@y3ftz4mMTQT0N}>?3`ul`jB{kA!o=7?g~;$PJCM7o9_Y!hVzmBikX3EVc*&OD$f&iMq#_+jff4@4Ez;1S+$IZy2~F#^-7F;=3| zePN3QN*qx@_WuFTMik=dkcE(ZYJ?t+9JY%#*I`j5MV9l}Zy~br_~a6EmGhK5)E9+n^EMo#M^wnASDq^LG$wQzhL7nOe>O<^gn%PXdd` zP$F!o>zjT?ArTU#>iH&&>1LCIYOKsTpvw65+&Yu`2eLA?D9*oWEm-or@fDLdoFLq)U=5d{9-X{<04h?JW zWqs$4F~8AI!Yv#3+>3qou~0A%>nGvF(OwV}rR&1i?Mw|+5HI6NU^RviLZy2kb;o~g zJWj^0TZ$3SF3?BQg%8FIl10X`9^CE0^IoaY-&8!6Jec9Qb8z5!Ot{K=DUAx0f4Wz! zWc}P1XX-jOFEuu;ZolW~s+arP1Ei;!@<2>K%&)BKw?UEaVj*88(x-`B z9GOp4U^K;{{HmWg2SN0M(%x@<=Y3^5&?oLm)}7mpFec=XpoH@tfK}+jaG{Yd{)1iK-3c5@ zBSvqCT`C=>5SCe6VwXF9?^-e%6T@M4&i5A#dY-+zifMDFK7jyt>AiUi;(Z#oxv)&z zx8?dqB;0wViJeej`n*Y@X`y_6GFLIw15+4Micc^+%u{aIUk}YHedkrqC>B0zny-xS za=lzSJ{lBTpH`HM6>t?L>_2|OR=|c8<3}-r3^VOq8pb1aNu98im?~cXf`M4_pYTS)N-Kcn^p_Po5 zSbg(NK!A}YO9_i;>n-M9*1Pde1z!QIM-aou5C_vWOG2401zwGi!Ybr*3%&w0OPecY zC5I|z5D^`;cZ3=0N`;CT{au zU5#*X`a{W8%p?p6;BwfOIeN@BHP7g|zyN2Ba@Mz4kwMW_vQJYg77_~rZ2y*&@fl)a zj+$<9wW_}?ne}R%Cks#izTz@O>;a6#ETZpUwa?1Ec7&tNRe)j1cQK2r!qj>4q)D-R zbqxv2d$~Lz1BKfD5TWfV>u`mW4WBO?zYOJ-81rz40Bt`SwMpzq=59 zcGjQX_=nniuo}0H$zyQrzozZ6pLWJIJ=x4)ecJKfh83@GL4gdMQ^UdJVOPj!wRAf4 z3@Xh#bU{ETuu+hrGSq1f(FF+vti2>Kh8=w=$(UWbNKL~mWT^}gGG{>Xx_xE6d5+P2siJLeZf#M`-Z-umf_9((Qa zhyryLvzg8|K%EiavJpesSQ$0j%YCYZzsOHmc(K4& zFhL1)_zG}PpD78}<$2r`-gO2E!Nav0BwA+a3ujcEp=HExBmP;~N8XPw_zIdGNKjtJ zVB-jWaG9aQra@!*U*sz2k!h;U*%)se8@oe4kC(8`$An(;xX{XSMU|HM2DV@?hOkYjfVUM8kV+k!SL-LtU&Vw(nIgWp=k`OPgjbOkw+m4 z#Vn%f?|pvGi9=@=dj5cNx+sqm<19;5K#K`9YfJ3u~7Mwim&Ci{M;AUGfe59xa zG)a=N{^1-`v1#bEgDG-Kvx5%s{QyJ7+Y>W(F9t@{%J|F+O!4nX0!YLR>6XuC^_7~Q zgLJA?LZ@1LM%(R|HYBw38r$6M!9honjloy-t?dmM{fR(PjHWy};BkLLknJmfd09Mjys}G^VE)D=BO#U{v{P!ld9@J?UFj zU8_8#Ef%E3%-thJH20UU9&+D@A?LV?S*m!wmw092vwQmraIzylia)Mn4=yiu$2Cnt z6@ncy5W7}iWUFb=wFfl|%6m8v3vVEgZoTBHwSV61Kq_!D&`oTz1+2eA2uH-mt>Av$ z028TX9_YzPd1iOC#RBB*h&}a_$AA>gSnn@{xC#IfXO|?dm?~EmXcXlIE-CyqAYXAU z=8{9HY8yX=aioa0{LAIY1A7>xOIxv@$J=wJlks2`$TBAPmBQNYag9V1VeeHiKUbIo z-v19MQbC7(}coSNhuYyve>wbO9cOJe5LCrt7{kHc_eR|h|IYd55 zz@ncVUWguk>i1oT%sjPV4%dO2QJgq~oFgxMl3l1T>PZ$p$%>@E)`4mSmqk{vyt;5*bG3!FQ@E0+BNvA~yM<>QlaGaFol_ow7m zR*-d1xar_eyz~o@u&bE=P#$~$7nwzL!%3fb=x?X)U+@+4HvNQZnd)YZ;x+d!s^TV^ ztGL|I9{|4FzpX4{#vBgxs?uunxtW%Gg{hp1`G_%MW!x*yg}umDSVX0L&)R$U8{k5A z6-=MUk!jG3W$8Qc`wY(=KjvfcyYL1>%p$sb;GQo}Kj_?ouZ)@@#ZfqLb%Q1%o~&`m zTu2aEL|-}Yr>D*B!#zn?$zu9LzyI^TuY3`2g1L%$hYs`<=g1=3?7b)aY4ZL@6nw>L zcR7S<#-ZU@#vd${8OhPyBCy1_ktA6}4^H0lLxo3NC7J7mkQB;zx@llyy4hu;s0WEHMTfLZL7Qg3|PY_0VI{kaL3O*F#^ta z71mku+#9t%>no&GZ=)wqf74knoPSy&`tv~xfA-FE-p6Q5`TQ`h?63e3f0uQ>FBUz1 z`6DOY`j=A+(S@(y?uq9wFopDlb9Q{iNap)dbQQDkub}zLX!fYfvdhd-mu0&492@3H zETUt-vE3Of%gC|sDruk>x6jr;467rTDWUf|hq?EhkD6zcg$ELKsP}6Cq&@*u#zXkB zJ{P_$@wT?`V`=!=GvOuLG^jZQh39d_y&aBBpXtq2MzaAi?kArXE~)DKZiBc@5XAs-3-+9w`_N&uhn$%d-l6)Hm;~K!1d7 zq|yW91fJ!Iv+}hfGjWJ@Ofo+tGI=c47ug=Xpw7WksL51_1+DP51w%I>dXXcJg;j>|=Bm25I-8EQ9MM@5KadX|1RO{aUBBPnfB*Wf%=us|qb^b; zZ)zXJGUpSX`HRl}Vp0VQ>(Jb+;43cDD=fLgi;X1%f_~F_prle_y@LVVEaA-5UV_Ea zoVM4JEABxy7gurj90P4-U`eyf9O(hJnDklW*WFb{O)sbq2nmdLHh<2a(@6&@XZB3t$g;wE6_fPL~Uj!q- ziue5KYnOfvZ$G=rs5wFxG3T4~xLF|+2?r6#Kd_9q*{bRBbSu2e?TL3mYsrk4`6fMH zcH381SC^GnQ8wwTaJcJgve+PW-0BjlLGWnLmBI&@_zEfD7XB=n{>_*Iwo_2n6uhTh{FuDyJMs76a8IMg zxcE-I@l_e&O=&KeO3Y_KPpnHA9^)G2emfjv6srMH;PY90Wh`;}sNaiSk*lQfwQYuK zi7%*8NY+DS1*?jelw1Y;Mp-!4#>x>{nP&arp-F$$GKL@m9=T2+MRdr_`BV1$!qtW7 zy|@4Jw_p4+!hx}PKLIH52SH$X2No?-$Wql4Hx6q{iA6N~ z9JAS#j~rd_l~J=?xpvMs>G879@7eL}nkNh@iq*u;Hy-In5ExNYup&@D8GIHnfw>VC7CZ-=s%tw~}B_!}8Q%;%v<1wH!4+RE88*JG=kC0c2qQ zfoF|zWM=ojBEK@M$7SpsQgl#bvUsOK08_sd<4F;6f@^Bu zjS6NQK8mhpSd@Ab7E9Bi0|l8VBQ~3irz6Xj*D56iUv`EkWG@wEda-*iUWN6kPb+oH zseS9AfCx!K*z}=_rl*2LXk7d}rZ_>l=?1#mD*Y<|yPS)tyRg_x?1(8~EHJ?c_Fsyc~t z#tqy&v$aOe^k!C|J2Vjt4HCbDi}YK3S)U=7{Q4^rqn*R3(!lQKftAn*59_HgML{mZ zzdb>i?*si(L_3dO_OTt$La>mlv{)O3HX)z5%!t8FxAD8bb@66f!KLjg=Ev%^SczE~ z7SWV_9=N4*4OCB8F<-|rVSfcry%Iqp)p2&kmhXP$UCXXOhC5fuA~(Zw%qvQ50hJm2 zo6P_sUnmrW}Ip3rQ@9=|H@d&+acHR1|);cIw zm81Xh)#cxS1xlFYhi^)90g0ncy zOiMpW984hlWhBK8V84zxAvhVVLO?|3SWP1$ut3_UTg!>Eu$wIgx?2wZL5)zV-a1WE+*~ufjJImYM9ISFFh~k&~D=LG*$CCjWWNim$GG9D$!r4){*%HyWnoEP}m=U8i@Po7$|gL-4S^&P32Z*5%Z3K3C%2totP@! z#O@CS@b?7}54IqGeO44ql`yB0p2Lq0F>$QN2|qI5(cQTeCsG66+TPmA3L+4P8&vXn z%nwd>kZX)N)VfZMslgjH`z{L79M0}f-nc8v9;)$GlBIoBz8z(x1GaRDiK#8%7f6&3 zf6qyIdICR7wljGGOEo@9vXn983QL+M{V;}P@Uaha!ohQ3NYk3N%QKPp+9((Eg7oYLaL&ttTSNi!sF4V3PFwO$KcIW9-3}l@Ozn zx&cnp{lOkKh(}vvfQA=~vZO&`!G$DKz7Wi@=?M$+vlh&sek6rZizV0}6T3W(UI{Aj zn*9O@B~3qOL&6u)y%`{Nu{isfqse00{>D~uJ>fruUJaO}NVf6D4>K|*@WBydu9@MZ z^q+ytfA-z!K~7&dVy4iT{Em;0pYI?&$sFODYoyuO>qLG63=C_e;;pur!UsQAlHt$N zAf8%oiJb|V^~}z}L2H-h zO78fT63yw}XGtFX zA!K;KN$?h*1$)p&Pu%o`d*LVmI@;0mV3PFq^z<_?C%3rUfc&BM;)>`G$z6J$ zXBv_}?F-!_<2_8F{`ECqgAvz-7Er%PGLo@%7%>88g3&9t|2v&R=0o%F^ra^pk!`c? zw%T&!FyKYmh zVol6EIB9zW2DG=qkTDyGmQ@!X@D`>39|6R{2@taduYo-rC3=?ziP>SRA8%%2^exFF zfP(_Lm9Di8Y9_`vlN{-k&I=_B!z6Km625cQ4#BSF$f? zFLgUKN4EO2xN~_$dyzK?eT!p;4_)`@i?vfOB)8eOrd;;=%Q$XfUo775P!on(O@KU2 zeITHBHqJ9z)nu8e4W2AdNz`dwVB8yt8Cq6 z>_!||VeDbuIq@24ISD_Y26mt$)<-D2`JNQSz&q^QTTw`=9rd3iH7G5Mn7sxEYsXcc z^7FXHt}DWcjB?7$w0BqB9~s0Iv})F#jt_h4bbYJ3A1JD^6%>MHFhsLf4M_tURb?;(pEh?03+?&CUDJv33m-t&F*= z)XH7ZY`In8XW%)(iwB?NWiQ9e0@&_i*Q~h{notkCsC-OpX)f-wWS^qp!)*dc(~uNy zPmj=?-$_2c$O)#eL6X5q(&HP$4~A0C$BS`tF2^UQZu{zA4_w=*@nGA|uWbn1{F{%foDT*ZV7J4w@*qRcF%>rI(26?&*$`vl* zqq??OwF%FF7nNYv>X8=oO)~2?%gL&z^Y;!k*C!2d~M> z;Fai7)QQzSz=%u3u(0}3k`cGvmSkiK%hIGcap6Oo-=~RwJ+^*N4VWZhH>AudCMSX^ z-7T1Jip`77;ikAKVzHLZ?rDAL(%)SUzbbm^iR@@Cm}(3dW8*Wxab-6CTq=N@_tc@b z1&pXNW&??Zo_v+DD}ktj&>LI=8M}KmwSFKDF~g$pU{bP-qi*bpo!lk0rrA$TL1(eJ zENKL+95b5X%%tFton!3@ACx(u224_p4a!4GgQ9-)>91bRQMC*^e5PUctpmOb9(G3O!NR=2KIB8)6n*h4^Vnh%r9VEs+wI@m{JSYN zb}B?QUMCh?Fl<6TqN0Q-C?WT`9R?OX6|$WeB@I8)XjRA!XkW`(#iw8?3Q-#gMI35l z5{Kq|wGk|_*{zzx7)xhLcA1j}OJS^&EFNQ5@KGxNd(Brj`!%9S_NB)*)M>WQqSsLW z(79(61j`y5(pi&AL36E0S@(sQ)o0CJ9CA;YNfpkvrVkCE5}%wz*v-Yh-{w5mr`Trp zj$J;fs28%=>`N%Y`IwT*e2fLYeMw5)tvQoMtU9|+b95@OS6gx_SO`v~ZsI}cn1pN4 z*Oqj$9|GL0819GVt)N0yp&48EDd$IxjTL;7eY60Yb$!fpUNcQ zMC{F9R>ZJK(cXiA#BQihvgjuKpc7ektJa0bwARMt<^&a;KS`HDIby z?+S5QBsND}Uu9Bv!txAlPgoE$n(oTj8r1imMLaPxnq<)RWOBa9WX=WSJTF0+tI+_< zTn&&Ub9<^klrnvp+&Qsih|XJ8crZ!QF2j#$Ytm_~FKJ3s;-AjgzRN{3A6isLUwUjA ztQs&$vMyT=y9@1|NLz{&CY!c1JQ=!ri+F!EV3M@mLJ9>0G*1r~zhinaHpaEtSt?Pl zm`6#Ay@mtz30SjA-`NZe!DiR^BFTkYMPYkKWi=7|Cs;e0N_IYzV-`E*5tbOSTe<-wT zz$9tAae=gzwvQuSW8<0h1){>_t<#Bbs=s_KzEpo)zi|?}5}4leq_zr0moa z+hXMvBJZe1L^Hk9<6ss(}PLUJxGlZlFKM>p&U;xn;T+vTWp>d zG*M%nex{`wTUBwGm@Vc_QJS8G&YO|_!RttBe3RtsK;l?nTiar_ zR~f{@Sjc{~_lP;e?e!I!vlLn!RQTO+xLip6$DO}TsEyi<9T_ytjVy2Supd8XK9=*G z$&a_;u;e3J^v7Hy`!NsRRK{Y%-$dBWjT>ne(z1=M94waANi12sq)l1OpGo%60op^c zVY)6IR71`^>u}Xql8ag1zJV(;ig10`(sKI*)O%Zjbwr-q+&HC)l++1F7Q5!c@SDR( zvv@C{IlIZ@_aPrqrN?nLt7e>8S$vi%a5{!P&YPNV(C;-qNisUWK1LT5yXGT%7G)L8 zCEl~z8S+B1`PBrN+$!_pER{7$>r?&xP(w=7TTLzMW&P6n-}6{Pcr9Rz`kbf zg_<1pr6)W~n5zMkWUjeB<{-i&A+Duha&C7QR5QkEXT)n~T%wMl8@B#=zvb`aFpz!8 zVA}qqFiED5-7uyXmSXkU8#TU+c;3X|k;!G>{&k-rd^ z6_rCXo@N$XVFuXsJV$OGdTq15=t%kxOapQnn(~E?Krd*l@nMqj`iGR&hZy5=@w&Va zyW^?%A|TTmMhkYT227I6BR5#Qb33LLTPV_)Z+^l%mz@6!9F?Wfi;DCt+}4JuHj|yo zJT1Z-Q)_&eZ5Wm{BwD(WM8-uKCkLV_{SBub~loU#@LIq zW^=mt46>!E#FK5Eu_0~MxsL61xf}bv*K+MPzg>_k*_WR15ixQ#V3KUjshch3>6q>| zF?41|bCm7b{@5XZ-lzfvsR5JZu1@a&CU^#pF?23f9+PsLG^geMZ@h(#QTvi1H~W*q zBsn{}ZnDWM`_dIf^V*wusEkL+Th#i;P&Z)M})&LH9knPS-&Zs+}vzDaCV4W+H$@a zb}*_j*%wvg8Zb$g7S=t)RB~_@{IT{H%{L#VWi$WWw0bIpc|7N+-*7@X?|>dlwYC0G#U*CX6^uU2#_&b&6ppPy)d=zF2DrsnigYn`fu9b+P48rMlXnXNZ7arOj6?Y z^8{^Bo>p&5!94)lQR2{moW5lEv=S7EANG$FrA97r_;qQSbB84S_m;i8`>oVoTK#rg zbZK_5BrJrhWW7O^LdS1n9$E)bUwZs*r2&VMbVPli)bHb6V)F5_`h? zMRqk{lH4`cr>W6ooMcBsZuUjJ-4{5NM(9w=uLw|43}Z1drtDWy-I^U!d*Ba0b2m*ncKEx+5cR|Aeo+LsKj>`w}lK zI=XfvU3b)VIGivD!(p(wpunc8VkFe~GRg7Wy0y+S>cSP#l67-hJU-ML)0YgZZap@I zN%EIl5B|6|?3^&GbB^o=EeeV&C`FTP%tv%Z<>onkfsKYxBCMf*bCHz(T0!cmkhh?wfv_%O-q z`b}5DE2hziUGnJW_M1NX`-$H^XYdgv`=UIuYI0=+qGj0RFz5Pl6;yDhIowj?^CYM1 zH|-0j^H${-7CMTUT#>ZS^Ph9u7n*zg$oMuKqI_t^tiFKpwKM7t*T|X5E)Zs(|BpbS zG6aWhI&eM(rS+o&4K=7I+r56v)O5RNbuIwqj-t6Iz$HA{0?}G28W$6Yme+tu^0j_b zwa?e|wvH}SQe`?!K*&ewD}ye+eDTX$<@AMNv!gb!A6U49yYuXe8)4`wqw{UuxCUK1 zgPV5Eft)6{8>))b0$mzX6`f~(>5pgq_J*Y&HkRxQm($OnYr;JdSFQ|;xQ<5v zYJ8kzyB@oQ5i|pr#-no8(&;OUO}NRA{JN#{O)ro6!H5U@9#Oh?kAME(D|jKld5Agq zI(CaHgWC$rFygVzS-}0ln3LUb=ZbgB-+c}j8b&Yn!>9S-KxX;5j8;Y=e+mb) z@ohE2SIoVnZR-x)7+LyM{^Z+#)1u$+Jrds!Gl!N?*)Nl}?5Uy5s7!FX>bF<%+rh|T zDHSt6objbc@45euQ&F<~YlrXl^3PAz5A&n&;Sk(7Qbfrfrr?X5oXa!l<*_YGC+&-y zel|P&ci%1SuU~$5OMV$u>TzGXj@e|(=3l?jjoXM!EMmJ8^~*DH+d^gFnL&2P$fa4^ zTfbuW+-;%=VJ9Y|%IHFSxqVezxw})fA-{ZLLLpX<4=?=wA;nzji5qS_`@D;aO1DA; z!%T6aYMmpl;XycHICA`BFP(8Ph;P85_;N>Q15zgP=$Gbl8^Ue8w zl|azA^0nVw@Usy0m!lU*qyB5qZD*j8;8Lp(gGYmtf#{w9ry_ZoOM<=>e+Pa0g5mHE zTYe|U-)3#yybIF}*Do*M_pz?KI&&rag7zBl#pAe;z=sL&d-gm3@n!!!FITcJ7TjEN z)(ss=fUi3I>QOsf-;pcX7YJj{2CIfi^0U%-kV~*WINZo~xt(M&mW!;_T$GAI9J(%U zyJlUe6LH+EX`F^O!#rair4t8!>&eFh-1lK$I1PQ44xTwqy;*m_gVZ(PlQZQ}^|2NC zn)JA`9ZfO+iNi0E+vVIr4<^al`mL^2Vq|SmG)O0#8sp*;x}Nfqse7QY#`7>Y%zYi> zF440de3H+4+(WmpnA>3rzOj?t#N6@W23cB#1hsL(~8T3rsBNT zLQ|19Ce16i22IA3b!bMFw^P4U_WAxx7oYV3&a*@>9LvHor_XI; z6KB37LWGN*NMtCgp-xGcTMm)ii2a{xJ z{i50tOWXi+j3v8Y=*9L*s`3g?W#$y}5p55RwfjV45u%EuR0F0L zIkLO)j@sidocLOy0f+YNO9n^wCxuCJw0@hcGOx`k!~ZMuERE46j9(mJo7LVF8oNB@ zV{J0VF+*(TW|+#rfFJT+rX{=*-i?8evD5Vg103Vn3MA7J|Ebyu-lk|z*adzQ+<7ob zv2GZ5n2=?g?8riQskm=)V0e^!!ZX&ByJOaSU`)?D*-b&QTf-P72op~BTw;anG{uix zMBN?i&1mFivS6+OpA=+0)3IZfJCByzlo-96N;Aah!6bQGzx|8!*ojmy$)A&ptqG!s zZB+?tz$E#rXI74;Vons9Q=D`!vJC8tnjANB?N17mWNXrTXrJB2pzT|L(#X`c{b-T- z(5qYZ7DL^@aC<7^FK?QQb~`rBg+G>Umz$U{%17z%ixZ>v`-<5G*I zb=u>I7E6}j*nW#+MUH_cZnyBZfIOI_?AB+RwY)4~^=77=m&0j@SW$%C6d@m)m#r^7 z;mI=fyQPu0)o$h>iN@|3y&t6MftLI|%5H*0T-v>wAJfak8SSs1N zI|{6i6SYlSs@S2sa3y*J5_g*I`JrK}%(iZiw68g9fEw>oA3I%Ny&ijK$1I!lIVuPr zr8m2`{_wYhQDn}(^jLmxAPgUVxp2$xt&H065mkdjj9SLiz6=?!!neVX8RQ(EUlkNv zo24p!H%{G@ArGP!{JuOkg&hHi3u%1JQel1eK5cK)8*lFFM)~xle*XG*yRMu%78L~S z3m1uOi;Sotj8-!DQ@#b4&Y0LQ^wZ=}I%2&|)(o77MSMRt^6#a{`H1c^#d)M{iXXHr z`N2FWV`M|!Z0>nZXbO$P>VM<0cEF@o%*nhpsS(YnrNXBH93U>p>3NL)g|Qy(aHxcM z+-W?$hrqss!v*AB(32ZK!~nc$i+#P>7%oHO!Ha9878F`|+|nJt^w?|$nCC!&f?;yf zP!DFg7E0{l*~aLOy=PNdbL&61L>&c93eJVbn@8!ErpNdD&u%B>O5g74`=<-XU1V&t z85O23$Q&coTJTBrS)Vvu)k%c9?zS0ar)LOkaogNWpMHIasD zi-OCU4s6=88hp5FLP^|xS24ADr$hT(+~?3-$-W@uvG%ySW8Ch@MVd0UkPq%0>2nU z+zBJq9(OUPLmmEOCFbt<@!)}f?HTK>4ek#(k=B5n+1Ou-D?7?z zR|BuOU_DaG``XObhf%k-jJQzwSb!AWA*mWKnh~8!Bv@yY72d5;2 z4j}$3Xm|L~cy-CbD}EkatMr}9i;WU};DvX3!ij0FJs8D$JN@Mm8gnYNPG`+h)N+Wc zkxXha?=FLc2csltVG<;!A&n@TM6AR~RT426CBezHG6-viVxdi{;AQY1`*6LGGq>F$ zpO1l-DV%zdo1(D;F&kiJz+OzaD#5re811$}PxvP^HA5qDQr^O-RE)FWjahXZuKj57 zc8C0Ce3q7bFu=KXey@(xQWFQ_m6-!kbZJea-V&EF5Iq<+`#P94F@IjOJ^ieTGp6Tk z_WGwF)G|*jna~auVn&8620Tu6VKCr+le3f6K^_cSe;Ec^xDXPp^q

8!|X6B-BY{ z5DzK(row|!%s15NE9}NK%$tAOCuV1=li-ZO5a$)-b}9$%5X0^4!jhty$Pz5;2(T|1 zL)ZQkMW+60{91!Pn`MDVB(LZR37e3`wg*E!xsrNf zl2H3E3=2U!qn8O45$&qO02|E5a6@qm7HX)1IA#tqbWe!KU@Xf44RvK`p2-xGyB6Rw zx|!(j48~*Mt{47SiGQa&a}Jgkks?|~5fet6dEqsd&bIVKFH&sfL_Q!NVMaP+ulNY; zg(t1=TwT7U$*2$7yUX336u9l#3_5ixP?g!oma8hQ&R_nkQJoC1cuuefL%H;4jK;*S zys$+~2JtmPHtXU!=maL^|u%BWDGVBhB${;=Tl>cno8&oU=M~kM^?w# z*97KqQSEgN|2g$jPo0{j03Hl+j;@YV2mHaG-SdE+8&I>>=hK4$&b2Ex!pKU+3yLAG ziEHmFv?g>d+d}ACX$`N~+_ojF9Ucs2b2a9Wu!$4N7W0oeW)2ho6V}~Zv}epv9*jjC zFzK-}2#BZKSV;o8NVbltgIoujVY;pi2J)$`OrUNFQfp$5t@UBO%ETlGN)WaE)U?^3 z-}THBvgGW+P#h*=b}^Sl@W#v*t__)GNaRUdpZUowXG5eGgCYJeQzBo(UKVzIkc0pa zta)KrEOzpKiwp`evzs9mp%8;n3SBVul`(MwRfO-MST8bTF|AZa$H~kumXJh`LQxrw zsY8K`xbE?&^s<_gN)<)g_mm5Nze54L*7vqFU;4u>GG{Xn2HjB4NiLj}DmE-bk;Gza zXTp~B?&Q7p`1Q7l4GGqkh~DNAhAM7MM#H=5=$kVi_uvg0<#yN9Xjqq?NuLJZY}*@8 zC%zl!!E>mj;NvTl567MT&c$P==4wr19t?3_M_po_n##a=SfC?akH$JN8oZs_j4|iI zDDEXRPTL&YjS7L-p+I_jH;z{@yuF~XyvQAi1TAg1eaWC@f70^(O6w4|V`K)c1PzbZ z3>qGc(kNrruZ%(g#A0zeE?R0s3(yj;A$89|rbF7z1Q(}~Xn_|L%4mTHqbQDg~Tn%xAnXktS6b@!~HHV-7jAK{U}DJqS#VY2!ZN%`&4IXl%Z&n{$)3J*qwIEgty%x%eN0GuK#{S2Ja z$QX>pS!Isa%M|7C!Q?D;^Tv&zV%;q`h(FM2Ra<5KZ~f%ti%`Vm zsqGtX{@K*yvsA@{p|o$Ls*ESgBlEL(j~s-uaNcoIwsB%yin6@c7-iG-2Mupy*n_b& zg4byJQ&-YDrQDE z9Cq4t$c;xmR(n&z!Vu?AAe~?rcEczx^LXD#)yc4K^)y5-2a)iBw9WNKdxkJP7-jq} zqd%5$6@F^2@PwH`!-G*8ACks!k48d68ZCF`&f4&nN8_JaiBfggxKVgCS0{o<5wi*f~G7fcq`YE*YjP zZh90ukK@cWv(UM8Vek3hhU8_nN!r7@S-UQE3}Qp_2PBD^%xtM4t-Xj6EWNm`W@!9{ zG=}Ewy5&gRL6Mq`?Ii4~?Nqa?NVT*nquNu(VcQb6drq;fj0*A5`Y3}=-I{CK!2=nU zvH93ya-)$tkCXpa(t=sGqJgY-K4V{q_C|=@w4O4^rHt zam#IYGv>2-SouOXi_KL8P>0DgR8|s=B%72Zr;e}q1P1)3rVps!PrZ})1hP@S1G*m8N5M|8ba)@vnGGD^3B`7U3r_j!G z(Y+%el39H($qZdwK&hPOa;ewx2M4f=N1>8nQ?aGHb-~i^uIBQJ_TJMYHX^u&M24Ae z;Gqwcwqv=1gp}&GK8C;@Brwu)KN|-Jq+@z%HZU~qBaIQ3#^l!4UMEq-Ym`eC`{qg) z{&v=Pe{kKztkK}XU^J{}*V$*sPpC7O+Fw6n^;z5aul$7adJgCXyC5}h&hLvUw?DTF7rGZF(w9Y6b$;Wpc! zw2H8zH@881oBA4=voVhh=g@;8&To=s)`hDKHTHHslMl~aUQQ*$C_i#gcXNoON_V2R zfrU~0*>i@T`F-b=!S7slQd5?gJs99zd+K_Rl3F30E*x9lm^&J4^BaL(%v!Chx`L(} zS)v-dwld662@i%{b~jbxT+U)!&N6lFFaXW*eO(78)LBLXB6s#BLpAJA3Zryh03Ck> z>Me!~HeE7e!i)Q6sfGtboF@8VJlg?PA`+^}_9cT|`%_}{oeQx9mvE$vpnnym?R6D9 zs!Y9gNW_rX1#~K7uo2zGv@Y#|YRj1Qca=j!<^YlzWtK|3R1pqoGp?4Ol8W^61Auo`OaKbY-vh7qXuYnhIhWHK$@ac_atE&u_@?;_ds0PYA?v& z%Ua^A+FKZ^sKI#!vjK*p)B|!lX0%s7nUg56g; zs%)UxAC60zl6_$>H)kpdjg<(s3${=02o@pTKkZ&?5Mjt4K04<G3LVaXlFAo|z;y za0(Y82$iBjR-)@SY}|Da_aKG&P-ZFE51|ia3?IFS2>=lGBS# zS-G?I8jTl;e|Ur+cCUIMxn2sJ4h4mNh~Mz8#Dv(rX(dB49*lPBEaDv-;a!aD5I6(T zbc@O+NqGY|gOmrOq#h=zL1u2qLat`bzGNV`KP6^-)5C8N`B3=wW|@+O-(xHTzXxNP zud*bQ0`#BEmHs|>pEtLA|H>?n!h<1B82@BGiaTK~JYd6FI_*VF0hsT2_^Jks;&eGH z@f;MXrGY|I!^ez14J%s<4XC!t>S!A=2Ps6^d&1%ob(-+rJpNdd<9C={FDE`oB|lEMI8;9Ta?zWCq`Jh}qd?1Hl*EW`x|9Q<1SAVH3OTHcq<=eaGj3W7>@(ym_-iTrj zzYP~KTw*-h{Nyk!teOGWYDO;w#fW_M525l1d&I7h#*KftETos|gv*#IIv6Iny}8g? z6?+{k3$`K{!LZw~{E<9o#9(e!0(GpAOds4pJK}{vRMqxBa?x2^e=##^puG>&-OiPz$b6WvLzl2ac*55r}A+BjyoUl+LF_=aC$Hl z?8&_wut8)tevGJh6MMt&Q+scQ6e=U9+t{zT{JnQvku-uMIjo1KXe9jGx<`j;L-y#< z8#9hNW#Yw`{WCMq%Y&g}{D5*{>8#2~*~oG z%)7$QRrz7FD*(gv+CA(vl22B;;@l214fp;E-Rui)4g9kBoQsN$8;Q~*Y{@ndm26){;kX4j zEQ4^5Hy-M?7Dj3-XzTWO9|{}eRCPt!~dki_o=)22?&P_;T@DMTIY zz&0Tk4y?y@;3}8m;$l!*+M6johRmikT_d8@bv>|_%66-}cMJ(L#r+J;Nk~U6v<-;W zvf>to*2>^~iEIxSwlx-);##`4j-hTSG;y^vG({oNf(^reEAdAysFt6%VRD~M|M2H& zxeM1=e+cr-{5%iF%Dqa2Nl%1bci~xWAG~#T*39F<0O#6I_o}-YWu&*7a~LSFhB{cd zh)(NZnGE%f=#Q@L`2W(HxI^6q`{OSMb$f6^`%Ry?zi(y^h6ls$=-t!9AJZN2cJ%VX z;3srwSjdmNd_15~oPm5N6wIAkZtZSMhArwIV}^)vGzJ(e?H^K>Vf4J+9?m?KU?%sxACIYZW!w;Xn?y$EO{E(+I(BcwqavzS$9!n0 z^q0;siyTQNn_dW8SA7h%I=mlJt1{0>3=5^L*PeCy{asfaoYff~3_7Ei(`J^M;#3>a ze*Jwf1pL!I!YUN#gZ0OPwuFrOHF&S$CuhF>9;+z zy3~W=_}YfvO?KrF+01&su~egShzCQQ+f|RK>B4#8g-8DN@2fwR#is{D)hlcar$Bw1 z8c9&%Gk;*iB)oXpi&c6s#;I z$daxHLmhjBW80MW8#)E2_QF->Y!s5RljR6yN~j}=)Zot^Ym$YI8v47Qvfq0x*PfKc zvj;<-e@&iQV!YDFV^qu!DgaAD-5XiSKpu=D&1TfrcP%~n`lUziP^Fhc4C!$Gp1V)M$u=ptY=q$xa#()?HBBo#j*!OoKHBGqh%oDBWZQ;*jZPW ziC$9LN6#wp&iO@Dmk5>@r{eC?lJBb3Eh0n595?0xCwsN&)K*+qjT9(vR^Ko> z_cFM!h0U=`xa5Mk4lmF&;23V74m)~IvC!Tb@s2&i1cAC$)e!iY(KWk#_Jpw$+iQ+4 z3&Y<1EqnI@8idV-=>)5F``ttvjHO4sZANF#?U+$$f~%ISS1fX=BdgK0AzLo84T;>5 zKfOnfI5UsjgMnP?Ntua`jM>|An?!Tb`_V12a)FGuH*L3eOTS;W@crk8AD7j29t^wg zA&56LryBd`Q7)p|^12RdMa;oSTC0JIIWu-xFYYbd8-hqvMVLyG2MJhHXf zMoTm12-SECVV~1e7S=TUMOxfe zf|29%*DS}!gR$bSGK<+;PfB0^!fkJjeEh3fgT;d(&Obl`A+C)_bVvIyAz)4axVLGS zVRyZHBrtn9)gFw+9Q=Wsg98G1UXwVxdxHC zCqqLdyF)ce=GG%xN>3e@Sv1pwVTaCJ?**^zP;M!hT+%`pZqSg5Bh~xS4v&%zK4h%O zJs9?5h5cxJ)WJFidvD{}V86+X!bH81UZ#-UX*%^Ec*0HF|KC%mWa+jCQ{O18brP-( zr&03DimPhGQ>14Q2^QVWJm>Afu=oCq-V08kyRpT>LgzBxGMO;Vbx%$PaRG?KZ=&}S zIPGG*)~eitvCLMPoNNrgB>j;Ys`Fro^Dk82rFbC0b7SVl7&-@W>9g}RwY1H0jI12G zMVB8#l`g1bQDjIiVA$}kbQjd!tp;cIXpt18cN?R-NVm-zEglRivF!ZXnn>` zWQJV57^DY7etS1+8u2Ut%eMwgXrqU7;CRawjS-Cg;!*$TvwtB=!X6Bj{CO(*%^c;U z7Vtbld&%V9;ZR-?(S%;gr8PuMJ+ZNJvT!4ldV+r6*2lGXc6YE%oji<=vpqE$RmT`& zqgKBS{i$rZV4dmIu?{i9dx72L^}6gZ3`hCXWmn($?`fHZ{XH1=`rnA|PD-h7-V54^ zPxf+S$!>9G_{AQK#TvX#zim(P+cb`84nLJ!y4ngWr-q1DO~t}yu_z%K#$EcID9_+F`jF%D$p|7zGwihBjP3#s z?3xq%RhnHV-73?bdLJY9y6Lhf_R5Ssc`y{^%hU~?;i(L9$!ABy$4K;4-P6S1iT*O8 zA20`3WCcxCP(Nw&57xbEw?Umf1Bucg+%HR8+}Y;uDtRy}jclAc2IQq*7ETWaID5IE zWpb7%ccZ3AS9NDzH%gxW!IBv(dydFr+Jm7WjzYh!evIaB+bLnW(A-q$>^icwyrj^o zWx>2?ut>BHy(Y0>G?ip@KIRk6wYgCVRTS^Fj>Kw1Wg4k$6Q?p6M{K&=vT2Fh*P)w+ z)=Q+dRh$-z0yPJ?mSAC_x4df-3bPZe&c6PP7mk=YCRf5m+hcFo|KY5@_h8ufuMp=8 z?E3+%EtF9@zV4njsLhReLyke!m@4S&wl)U6Np~GGW8S_8ZWQ#qR=YCLqVBa!UBh~| z<&h;2?gtd}!jtBnf9k=|@=RODkipV&SYBRkZoth`xFn#ZwY;JM2TyIp{9J0n;Cw5r z0KL)>yTOS8sigOR`ge$)Ut%uE?1$5#;q6o;fVW>86FDZceUWHyO|3 zlxM0UZJuvp$;?DYWSH$)&;^Lmb(ROi-W@`%%^rjz7$mCL6yc19dN9OkZnVrkR411W zjm(wy+4~6k6e037ETX_oqWuO$T8Ym-FPX5bzFiH(?Rbd>5a%}??$Sk9Q2LI?-A?R@9GJZ z!=|N)gg7^s$LCSCc>kX9(q*}=F08JIu$&f#irBc03^lPHYe&$utaV)~U&!o=Th2;X zl|Oxaxhsnu4~AoHB*GZsNqp>tFiB$8%y33W05CBpdoUJfU^dye2w1xar`R$OCqhwG z5XWE?@hJ?v3`aT1j;>YtrebSrp1HS43U#~bMjWrvvGIaXS3donA3b~dwz<*==k(ir zSYvaRh&>qg_@p{|ya~&Xf?3{X&nJ35q;S)n&&{6A{bJ8E_FyQ&CP**%6qdQ#o7B1GrS}Z#^RKF;60(~BreP}Wbx_2P-yGxB|%_q<@UtD&6b>KY)r8-BIe?o6 znljBp;a4^-V-vA1;zBQ8H%Pcg4bpcH{NR~OMim@u{5IU4tBJm`Iz!UPHyTUZ{9)mk z^H1TpiF{JiL40T_rsx>QD6%s;y^wM)tQcQh4{_O3pP2RZ_nI6Fo3fLJ3Jc@eDST)O zh`vxdrnzpHp^secl?`vDzxo-VtBhm+<4Tec^_lZaP&{x2DynwoWy}Ukg4O_*33a;@ zqf0y8zSL3-3uEb4*$vT2NKU`h&LOujz`1s6Rg;wOT-DiR3K8H?Ya!o;g>QTJ%KXZM z56mC9|K8>wdzad~xe77;;f;9BQn4=?YyG%m^2c;^Y4vOA{pO zj@ucwdzjs{P{wu%ABF@jI?Y|~pND9;L#)goG(A<|WFu$ZbXl;g? zw>Kw5?tN$Jm{6KOkxJMVi@)D*@xkmLwPzO2@L+&*?V(k*wlnA|?%jcrh!NS@V12oD zbt@B_Ro*me$6{op9x!1C(=~%2LEAOXI0^idHW7g*cn^UPA+j1UR%F2%+R(!YAJ@|G z*u8iD!XpE7(XluV^xbLX;wIgLQJmc|KAj4wTH5`9o^V`R^F0{FxiS-{`d#+J44fW} z;`}sm?k*$L)v2?nN_O$CIBUQt&Sk{8M|#v-RMB)jG)tZyjN)W{@o>*HPUTaq^0Vl7 zua7(!#rbXGY)tpN=+g7Hpb@uTJe(ek;(V7l7o|;yPM$W*mx0rRQJlNdiXM^XQ`)7+ zVKVj8gRwZnuL528$+SH61j%)4k9gzp8*ipL+JIazCPu?_VL!Gt&p>Y!{A10%z$U~l=OZKMsW_VffGE~VOSMs3`TJ>R_nK2 zwfd0XY`xX06By0&+U~(voMv(f$I+U_E#9`fPJ8dXF(YuN(9RcZ_wdb#t^4z`2Lqhz z9L?)&Yc4M7F2zItyl$wq*i~+Kr^`(2cS2e;V6dPvjQ(MSKhM*s z2|Mk5OiAnw66Q#J=n=CWrA{)5eG>8t9dq${z=K!(Jl+!STeUHwN^4AG_NVDE{=C^i zh~roOEj;nR&c>VF?t#^6$XK(e0rrgLr3a(9vtz!(0@fmOtZN%+NtO9K21A@{$nO)h zJLkiLO=#ZZFP!*Vp<<srdXkU;5^0Jcj3`VtX(aXK*z-rZVVgtPMJ6BFQa%%ETM4 z8OS{ti(ER~^SC8l6@iHfzu0To;o~z;nRqb7nd;>D@Y5WsYd5{-E7Sk=OXCa9q`W53 z1b9l}!7Kj50ACrcTRT`y=8lQ36w@b7g0o+oGnL!t%rN9G$YglyArFE*P9{9c@1;i39p(WAyFn zcge)?JnDw!o0bN}7Pmp=KGusm_D`E><=M8qCp?@maf9K(D9%Z=GW%s4Bm@k^17pQq52uie!6;5@ zOL&0rnc;9-&!#6lFwLioJ4}1l3WW!MEQpe&ZiNR4*|d)hzexDqREJSu?id`A$7m?D zugaVJ<|V~az)(w#&z0P*5E?QpsLIX7mO^(c)DqWfp&~hk1vfuWyd{@-GW6_^|5*6PN1+$5enO$P&R73YoCWk$sPhphKHub zX_c&GkLvb>C(yl(uVZIhhIHvhwCq8tzZwQ2ctO8rkD>94R(e= z#y?JSFFNO62bQm!2Anz4jawQIMsaRRksp*E71pMdWOaTZa88D*C))185a$#Qh~M$y zB?HLFM7Ef<9vL`27{$pOM|_-OIJU}_9^CHh3$}k2Ge@|OwB7Z)2ctN@M)lbtZYjIU zP@qnjJ>CI~sRJ`dLKW3wy?x%?&Np`EVf2fsu5q34U@X1B%ud_nJC~NbO=!aoq5P7f zPy{Vz7np~JtMo;OQVkfT)yTFDvhx^62#}{uRBUy?36Px^Oy-B%n!%2GkMV(?hjb{I zL~A|&A#6_z!~;<#Nh_R{JxF3Jf}UzPV=#*IB;Z6)uND5#ItBHrup1MNqCSf3mf5_x zc~W2E&u_jT+;r&P(1~->BJf}o=VyuYsH!3m**$>eePl+RUfZ7%p(4|TMn6WU`Lsog zO5b=ozkKfYGr*^3dpsEBlY3_2X=$9o1MHc+Y?6i3gHfEOtl=PWPcjCZeE6hYJu;a}= zAL{t01gFzq<)PnW-|X3I1a0Xr)7|UA0O#7n*D*3{7u`*`m%BlHV9La7xO%gb^<+yA z^l5+gu?3u&4@>KW2g4RF<|YtxE&bU^I`sL*(tyf~Yi5l@?LBG;K{@k;Gt&4yc*Xw^ z;9-A zj{M;U^USy4a*${ZW%A)TLdK^T^6I(R7xwicuo&LV26<~(3l>FnR+~|uf{)KKvP?tH z#gk9w9(?EvM;Tq3Py9}w<6Y{(Q@{rQ_S8L3|J7@;aju?laazD039LwK4##NXtb!y;#}$J=}&F8!)Jrs4 znCR4PlU%X1CQMxQJs1jdT~j~7=uhY@HlhYg*rhIF$RA^cSUTb{4KD;J6-MAX2tSx!vII4_6)G# zc*ijoq&+8(AL_< z{2II1!)cpb`o}BZAN$Pz5~tU84@Pl5M4>)RS;gC~vzN6G9H>TCj)%A$Jb2<;&x4d8L|{BGt^z{$cHTQ2JyQkjW}tEqjsNW7sWRKJQziPC0G{pC_Y|P^lk;@ zNwX)6(AR)b^iG1l#IfeEL#`^$7z}YX)1EUow=x7fNga_5!zN`yA+>|7CbL6lqM^ZY z{B#r|ux7Q1Of_5{g1}Zf%IqOf+O(lzk-ZmhkBArh6xP3^hAOY&>FM6~;DLYb(d%jI z-QkM{uGu-?$oUuA`fN=Uc_5#4!2MUuy`;@l1WEJe!LaRZl=}=AtIEJ8Q)bY{E7CfU zaL%g7E3ANt`(RE2FT_TUasK!yNi`#ERQ~?a69>%v?+&yqYiF&eoXpDFZ3X1pa36OR zc`%ok3~vfiYaO!F-2x1q8ayb^=W^&^0}NZ5#mKe5;Uh7?e|%Kqa;85Y3I5QO8Ze4;G1NL0k3?rj z3A{QV%gbA^Uk8DCpsoRf87E%P$QVSVUu}@pMfPiWhKvg*JP%%5xF>CbH`Cg%U5W9p ztrs4K7J5;C4@PnBO`N>E7gBS&C`y6j6SA-v=d{W35Z7G~p7gmAA zD@e0(S{ULyf^F}DepE)9Gpq(<;X4DT z2+`cHK$y7qqBt###ku+|s`S=e$3%5uxL2QVDTfkNP4uMhd)aVcMo)S$mQpYt#H!*AG1S*qq&xa4TeG;NX^Lu6W?3 zNyZM%BnFop5?3x;Ppx#`86B&yep8Z3=Aq?sK~tYLRM<;sh4%sb$`D{GMA#^n!Zp`+ zduh)DU+Vt(PDi7AyxCYpf@s|(bKcnHtcQ-$b~)LLGQzjCo|KX`i3*21TbwTJGM)j|ZbTnUD}pPvaCGVEdOZyJpZY-lT8n z;q+h>=jp_GuBe0KQ)IPiu5?EEte;K$%>G$8Js8FLHgWzKlRf0=a0(B1$Ko^?$uC0c z!LbS-x2GPw;{TBFQ_^zKq5R`_4qAFwBUCsVX~7Vx(h#?J@RkkUr%H;ShUaCILirL= zagef2`7T*A@bRVy{usRC-;?-HOpD4oMWxEa*MGhB_|HK6%g93#ks zfZOzyS=Zj)(@dNf!dFS)^k6JbDaqeoasksfOsYAuFByq|_9u1PfO(xuL)%ECjfpiZ zUEa)vU=t=$?w*8Yq*7q2D;dwrtc7Ww0!>W2PZ2b8UW5{S%Wr)qJD7LS|g+|Z^ z^QmJe-glzf>HR65WHB`skH=eL=%_W954k^e0%77fJs7gTm^j<%lAf5xDX6eaxAum^ z#lY``HcJ%?3Gt&?rEyjm#z>I#P$)^F2ro>lpyj^w+54XT^`}Fusr-<=2Sc3GIKO7?PA@OXRsegbFC*xzifUn~RA&6REz`JC>G-rSI^LcDLw>N* zf<=qyN6Z$k!Ee;!BpJ`narBg2@tf`aL0o>&T1I)oPM_pMHAYV|PuS2gyYs{Ccc>lI zWUOq4av^{g_cP7cq#9n5HUZiyS>f_z(WU9$_h3}APEv8nYEL+v-cFzgqd1NF_G2s4 zn^($*m;r`Qs`F}L6fdP0UY2gZfMNS%>O%*gJ#qjOf<0b67~-5uM`;*kx>kDrOwRIr zM8`c5#+S(rCkN1jfRxRxunjQivwO)8AEgJb7&GCHyZh%#7yfqEcYko*h@8#-a`6hQ z(nNggxX6u-UhdIRMM5?qL zx7-QR^EQJ-dhoUEBx76W7JVWotX(D5e=D=-Gh~@tnhK}YTx5)7U&I3^>hP1K8n#%+~cG$-#`a@S>{N?ROU3RB9N(tunK{gKeQaawE*K5JyaTvtQfxuKqEo zRr>nqT9+YaOEK_z~yJv(oMGU=-(4_R&!Gk>AXdZALHIVjL1@*0S6U7}j$fQgYmoG()2c_uoc z@z#3iY+BW{B6@1|KyltpCH1yS1Ps%`$TiD$?f=u!&hIh4`jsLFH%oc&7XRuOfxoH) zBA;Q&ZeM!B_aNtl17d$t7>hbw#_8!cvJ#s>~I`%)EW3`TMOm)-15!ZO(ULhNBvNr1-YWqE6-`#GHI`a0VC!8Unj2bYC)2skD&*QjGK9AP8Pc*du z(;H^r^vFYdz_8pd5~=~CXpaO3;Y_j-$0=GcDp%Tm-!t29+zZCa_Lx~p-I2!c!CU;n zScqb6|4F4{JNDZ`ChPHxp?O5d!y0n>(i1)|CZz_9VjfB%N5kLg!fV&HoV)yU3v;Cv z_g(m}pWj(>obx7A(X~EC%|zEcc*TD+@S}dVxkw%fon1IsYxnWciswn4k7(0vhn&7> zD{8rR*pM9q6=Nd=JdsOa!?^t z{qjw19f&tV=WBeUH26(;7ZoGU_pxv9^_8JdOw5(uIb;7L7oCM8EVhds-yRHcmVqge z(Bhm=9h&x~$KUU?KPA>G1}D>1{G2~Br_T(-(DIsMW2@0z zI3O2#WNjn;)^@P9XW)U+=Zvp(bS%4`#?)waH(C@fmuA&~N3-fE;A}4`w9%UeZq6=4 z7^k7c5`y{VL2(6FwI@>BqM_)P`(7h4Le_b2IF?Ks)sFU=h-%FTJnvH(&Bbh(Ux7=0 zP&dmY-ZBk2o)hUY#LoU6WEV_eY}#vpSbo&-v@bp3cSX{!k@hErQJmx|{6691F%U6h zxMf_JVOGaKB-re5dN9O!G|&aNQ2(Q;)+w>Vi<;~ToMU8xQ3HlJe+j6tlaEITAJ;ft zkm&RseijSO>_v-(QJiL8`jy~x_FkGCTYmRgoHbw+=YObVzZRU%0n$E|wsSKl;siX(A$C_G+cvCv^sjvXWJ9?O!3IAkO@cQp`@;Fmah-bNo* zb_APebO(s6+X6~?lxpp8|AZWz*QwK&RbWn`L>6o^0Lc28Jn%AGT;m(dK#-^9jK;9j zCz(CrV_0HJ<8=z$z-LAXExh9I1pdkvrr@xosi8lkz^1y0_N6EM1XxH67K=|AL0gZ9 z(1W|FL!%+=d>d-y1h(#zsXyRz7ITo5vkS;sUnA&*eC?bTt5WoDPy|~}K9*yhP{S?2 zPXa_mXfsNb@UKaD2Q0>MVvt84vEbAyTG(7cbzbCqgC$2!r6A)L)fpg$3xOs$SWR%8 zJr7=Kn=@F|yW29&iPEUlPEe^D-EH9&KhN>|>ZJVGI1T5t=5-7Y`=-Q6t3hAmMZ=g3 z-366{tqG3g2$d2mww_XCBrH^-*E~+OWtXF6sBwY$G^%jsx<{mNw{<#k<`UiWD@x)5 zo#Nx&b{?Ai*4~g*7 zog5$Y;m_BJXJZ%QNl{L~s*CoeCwwd$XRIyO)O@~%sYy+LDbp0!GqrzKhqPOqWsmEe z2T!pX{EnYEe%rE=hRP#&llENMI$_GJq4p+q4~wO`dM>08Ib_hqeAiOu$lFtXFHl5$ zM7cgV_VK+>U$XYq(WuV^Hq;>teyE}|@_$bZ&)y)?i zxXCuyZ$IEWt>0*vlC$5LeEuEHZelc}n92dR#nA+Y;q6wZ!UN7ZNGi`Llc!@uO(w#p z$#Y*rR%qM0_fbX^&NKUWw}~LJwh$Zl0kho<${(6`$#LCi#*1&(t2}J14DB***)KRD zjnr8==EBCCB4iP^SbD-cmz@8~j=AucX^CqWzuz|Z(x+b^lB=;nWOs9{7KG$J{F4Y| zJUfFA-Mubd^wSAn*#2FlilVh<#&Od}d`1(K02^)XBf#0n@ED@^d|;KO#7!pT!Z!h& zu+VB`Z15Ux(7NWH`fiXZbk%@b4Y-50lk2MY$k%_-)Woc*)0Z4W9p?0gN*F#PVoK{7 zLeLru^xbmpe+y$pZ#YW44E)E#cjui@zgG+?0rL)-;y#yY2A9fmiOD3o8OL zUVBh`#J+@&WMhfhW#x>XVEAjo#|@Ij6uBX&_*{)MD1vnHdnWmm!$i(%{9bUrKsNc% zV6VH1(GIIW05tp~;rkfSD)8ia4mXp{v38RfQZNNT!MaP_vnD#W@%v?H08rO}QPgY5 z?lIgj?<@6RSOlhY@3~#~1h?#n0+-P#H5x``CH$+f?(pkKwg-1DxJh`8zO!|B1Wi8-qFqon&77iRJOa%I`NlQaVc-o&m4 z$YDyv3|NN9wNz(1>jp(%t>%EM^lxU?dI4vs6Cdf=vwfw!nWHpUu`QXr2|VL6f6Eqm zEXg<=o$hsMVk}`!$(pliFA`;3lcY&F;PgzNt-h3djwMI~`1y&Zcrc3d^OW_sXzSuQ zMIkC&K=$F?k;gj#F9csv-zIg1*&9q>f&Fn82`WE+Uqe7 z#&R9xX|_%kMVZNWK3pQ7t^ch!YrrT@vn_rE_=hBh6DJ>~oA=x2jeB?BJy)9XrAP0% z{|?+B3l+x?QRegr(XtxwivLXF93wnCImjKdxV?GJoZpuEoeYhz1J7wte8hS1#J_<2 zn8IJz2>#1Ri;pNj4ks)n(MD1x4E(tqq9f>3@sbFZn(aoFy{?r|@bSdQhwj8gFGvKR zk3T0sL=WJ%*nPlo2?ztkZQxo*N+$tjnoNYRib`itwd+sqNGCY8B7M!K@W5Pn6#FhN zD-BwSQQOQIR@3(Q$z3$GQIXy*;QDNC|VtoU?>(9+AG*!m(oS0v)N&m264)Iv%m zTD3cgNK=sJW;62<>!;M>&9&xKXXI*R(O1^@(2g*<($T8jkyX24-Rlgp(h=C&u)eiV z)<(Ge{-mC+F*Ve@yn)G;!2EsATH9T$2zk>x4i zZbNvT7$4ULYja9xPZ&FK`&*`D8F>#zak5`R2LBTMsQoT=V|ba7H*t6{ij$HL*JR_w z#Z z57RD8rHPAA)LKPn^mFsxZQX8mmNYyV#n~UcRR#yV0uL+o?k0L_aqQaPw!s;)_F#a0 z?V`HF*oE${ve^-q(}N9#l1cx@{wo)A3k<`I5WuBaQpWaLQvY?(H0Lh5W#*4}HX)36 zAmorYIaLO8?e4yePjudw6q0@!z9%X(o}Ivl?$PSPjON6^8$aUD;wjdESNsk>K9bdB z%7l;W>3lU;(SK#o#g{LBd8?dV2e=Wav!vYNBJq*;X(gWrhjvpU4WHPj_U0Ki`1XZO zF*u8mOl1q6(dh1IO_sd&CcK{Tzcd$d^WwoMViU}XPPI8^b;5HCgMYD~;y67R;=F=G zm5ZyyP>qF&Y|FHpd&0{(^!n6-QOsP1r82bLyd|9$i&ZM>8Ze5QX?)?8Vg($$jj6lZ zLE$_uV&jnn4@PmC+}ZhrkI%u@XDS`P?c?vCK19wh{RA|_E)D{0;3x5W@Wj6mSc4Jd z`5?l_@vDutH+5#ziSu9-=U&8lobc=v!g`?5tr24|Xy|n(9kEFcY-;JCaHZ&v z6Kx9o`NptYau*zka%ryxB9|va2ZGt;a-6i>#Z9fYs&9@`($ZTsU=-(K;H(S?3oGD_ zHc4UmjL$5Mp710t_2Usd`%~m2yk*vHMRID=^}Z^TxKT7dG~!iy{iJV>8Xii1>QJY0BB~QIbMt<8r0pe>>o=f85En^#xU}fpt>C-37#W$b6LK5=can zrBKmcxo~G`?Sqh~xxHA=7erZBJDN3s;p|jk2u~Bv@Kv;i6zL$vyzRIV&-6P%g5Wy? zD(nDI++O&x+xKir<7g_C3H?5b=2r%W-R)8mzm;jrPIE-L;Evm}&*6t6S+^#U#`nvW zZku!6pZ_&)f5-nGV5-s=C3TCOw4Dz0jgn-eV+121+bE5t28{OCMd+T&*5aC?+CVo7 zMB!jsp$qZUB_Nfcq?3zt4?Id9j8a-cp+@p|A`$r`SMl197H@aRZ^ncBsM~A6C{A-7 z(kHMtl`u7Ga&2fuCE{>;Fc#&yz_B!JO=H*xfB-S&M)=U} z4Si8QZiH`88H(kQX`s7@W+n$SAG$Ep7rH-&pAR8Z$t&w)K&!wjEi<)W0hZBTmuIE3 zi6%z^|5`u=r*da=q|->WyuAy%o2_VDjqjD-dmNDiSeuboHko@1T=Yg18E1pVf8>JY zNm_{bSl4BsoX_7W7j7zp*6ncVZ7aKNd*(%-dq+x!rKHn0CLNudwRgk3kaZIGV6;Wl zfUut^y>U2=0@&qIwAA0zxWa~q*V(%Rjh0FOqIt#u@(~3Oko7DZ`{QZK%xAnCt1`fi zgTz;Ttz6rN zhEh^}AyYKb_-%aX$(qvNr|ms> zj(>^pFg4$7R;Gtjjm-B`A&>AgLe5<2$%oTA>BQ}F;fZWyJil62Xanr+mwTC+@C1Ck z`hGmZ|DjtJcQD29X;N6ZMk|_rx$u3U z%^s&VzF}DmpkcVCo_A)*9z+YS9Slp%KK+Qpfe(%Jl)irJuCI1<6_vqH@$13q51sIA zRhH8KPW#W~FZFHG56|a^kc?>=${=-gb>^3?hzJ|I%tsVR|Enzw_hDmrVS4PT@U1bm zGt2|f@sJO#%TiiBnKe+y>0k(`dX;{Rzok7hv0{hTKFnH zO5pe46~BXz8wrsE;vG^91lv1N8-wTIB2~6%!TuHN#wCb&A1M)j6z%{;$go6M^2hg( z%<^D2sA6R#t4~E$%QNYzZ)(3My*Zp$-{OEf25$*^%AXzRJ>gu!#5=-+QJk9*@jSt4 zI@;`kU%vXu)vFFVFjtzicdp}q#~lHMnJqZopw~&=&R^kt0WoO!DBXHQOX;b@FjHAH zN_GLsE{u^i@Z!bbEbhFuQ+~PXqr6)VIGi2~anfRkrpn1Pc!PM-V+VOoa*gs{!1?J{~D_fI`|#qZ$bk=Vk-*)ICw!6?pm$@9G&i*cMXR&ZcRmv6Vlw>DppvFq@A@QVKf;IH&G zVOvmewodc_TkiIyCtSqdhzDNnPYR=0w*>>KFq&gV*NOIphQSDY3_hz2Z7Othp)60c z%uuxbHNI7n{5BjX@4X0vbPXDi1~k>f?8j&3B{NK|R=33L89@)TB#_e`;Nr=}L|4_paBiDDoHf(dEE z4xVCW86)I_&|36ObI_!hTsrF7Ap3&LrldtRV6;Vt19iBk^g#{y^}@L;aYmDi!)b3C zi#+jdA*JKee0ng-X8}05Nal%}&Y;we{qvBMEVtcd@7->ysW-&Q#fa}bIMLx=Y-68V z@Jg#lTBBN83tf@N6({HvRn^44D8bK>;3tJ(4R|FulJJqh9K7f=>K(g@#nTxNhB%Ei zehaeoaatYG!z?}iM!pB5d>)d?r;HN#zsIJN=^xG zY_~fpYRlXjT|3+g!P=z3kbGr;gi}z_J$VI}wobmnnlv))4zYmXgs_;R4;uYFcD182 zX#Z-Ktwz~g-^Z=L1&i$NYItjVr}e{wQ5juNxgRextJ4qRZo6Em_}u6RKU#FVF_eaf z=KxQJn0Sa6%fV zxG_U>(NP5^RC)Ou9*p8VjW~I>Ai<}#+iJiw=5G&1an2{sQ#s$q+pc#l{dCL^KfUOh z+p_rdU=-(adVJl_B6(cGP1lNe@PGzE7N6(esN-Kn5WKr_voLm1&Lk zi3}Tz#(^nB^Ue88jfyAT3lD=*JR2Bf%@U`mu}-O;KL@SwB;&y-PI47aL{k#2P?tqA z#Ze#4$LYZ+&X0&Qvi{Bpsl#AjR5EV1_}G}Qu?CwC2L32uao0^{ zD+BPO#&CQZ#Bvt}Kv0-1a;KaIdm!5KVdTu4_tZk$m!5FvG+FzT&SQ#mByk==-zwhg z(vrkJa6=lW2Sc1@q!nR$EuStnA@(|6K*7R~1Zshg$D%xV#eX94pCtBuJm2|HC{}tVn9W6aR^D1I#7wDc>G0&L(DWOf7_4n1m;YUseM~Q5%KYpQ4N}^i!sV zDecv93b3TK8+J`q-IK_Jj}u{9aS3>P#qOA;ps;B^V=(0NWMB$SxX+ZVisRHSw{tH= zJOo(78z>)WQ4+rgZ}A8597nhFC-BeP@L)gie1NE34H(6FFgiQcZSJ#_h-u-skQGG@ z7)8AwP}`bo4qiqo;y)seO^r3C_+rhAZ*F_=ivMWhKP9^hv9Pkkmo_ccJ$`q_;EA6$ zE!>oA?0XQ;fr{WYI?ktIRmQUSxYCwAbK#iS;1AniEDX+_Ld@c;85+r z6daEmNB!2ucCkGq#wCLX3#0h&CeM)(cVZS+QTW8Z^n_coU2!?spA<%MK0}<51L6#> zs+;x86=w|?#c67jMXq9R@%c~k>EV>2x!ML|=i~{?Tfz$q^Fp6_9RK%C3-4xRb;u$zS7{zIhxp;k~vXU)t zR7Usv$b(Uw>`tqn&Ow&0iPa=hoHbw+=M$8tH;yD9?1)_HAN{&7n{vX1;L{tMvPrUU zt~7tQU;SnN(;H|>wwslW%s5N8x1bt6|DWc#8qMi)?xJEz_R=y#xCde$!FpKS~; zNl%e7L&x?l!(V?ah&F`vFKm~kO(l*UU0FJH$PfQ@Ji5&DU&Z9cU=%0sAFTAlU3A#Y z;` z*t%1WSzM%{+|k*9)PHkY4Mm@LbTl{-!mJF-N7`zCO}KX+E6V!QWc_bdS-=iVcM^QEddHL1|@{C}A|D}D>F_^%@Vm(%!lBA4|q_Dd}O zQ*?zF_v<`(#h)Jq{I3XpXKr+|yzdJSl?Uu~fWv=u41W!H#sA$cfq!+Hf9)g5dN|SH zA59%FMmf&E2e0_YBHo>H*mUgK%33scWh957zWU*s? zk4)A*c*TF_*1-Rds-ijm!%v-b_d;y7h0D@Bt2iu-#VIAq{+Y(9JY2lp!oME+a1l5s ziIsA)@?b2^paSPDd@YSrdANJgYhS+f4>(zG0}&3V2Sc2v(Catbn>UKLx;Mx<|GKjN%-}P4Vr}=)}rM zxI>)nnx<8Y4?hhT$E(w6!$hY4{Yabv8n!Z1>ryC4n z7RKhPsXqGF)|gf1cLz7&IOzw+n8;Y&4PG4QG4H`BPL^u(af*Inl`Q(gGg=-_4@Plb zLmm7Vd{6YP<5QeB^E^+ z?|q|$_)R!RTHjN-flQ$74E?h3V+gHF^%Sz74e7OMMLx&hwf zysfSb_P>vwYqMCh8Zb)W=OoatIV^V|O+JBF8@lPIcbfNPCN0GPCW06=L|UVQs-2Bj*HYeX*qcCil0t>*qByi9buoj{@`=Y zdhQZ~-?Y*36PJkxPy7XncnKd(G~DqaFSE{;xzZP}fA~l54PF2_GZ8i+=Tw60(K+A1 z=@yA~JC*ED(6eEIV#ti6IYZn`*5a$ylut3#th7y2ifl@YG{wz9%I4!g>p%Nz-G`Zw ztbykt;L0{FVYv;vylox03UCRoUX>a>c^;}x(Dm?WTQ&n8Ka?tMf3wk5$+Gh!TAjWY z$X3!)oaTV?0_k?w7M(Em7=QOBw+@ixS zk_-_irv|*@{{-<56Tk9*_|8X^wT8^<(e3ZeJnGDu27dwg1Ll~^t*jY(6IhGz^0vw>^2>Ejr>R&}84Z2WYIE#Fxh^47dRU2yC^8O0aW64}G4 z+N^EBrd3tPllVor5k|tfEQfuZjF1{Iijz$b7fJUzcvZ18KHPoO&d)Px(X&K4!-y_y zn)3&yf><8?fhnzp$0H=y@*urgg1^K*8I(8T)ShsAo*E>SyW9`c!58cilhV7?9zLRj z_lVNS(YJwU(PEKO4H%UYyEi;uq~ze$sGNQM87~|$lZWvp*`%jN){~kgu_fe5Q~{GeH+2UwGs?g#GiY#*u1q!aBvhbzr-5tou|!8OigOm> zOlh`QAk&PqXHF;F9ylo#24kh(MYai0_@*oOo&VsQxccm_V5g`Ij1GlP zx7?NNysb_mK=vp-)5f)`_3mi6Fju4`k3;as(sZ)g^v{cc8JXclC6$t4CwBVwx1lh? z5p#4TxA7~KC0q)gXYBvP6vZo`dI&xTy!-Z^fAh$X>OrC5&XrWOVR~b2b>;ZRNjTGM zzNKOsaQ-6oVMv($ugPb@MGgHm?l9sepTDSKvaR$99(Nvk2Hy-sMSx+KGy1$&G$AYg z>U=Uv6chtR-kQA)Q0L)8{1H6OwI6{m7C093+f2RWKvE03Ay=MYLYxtd;4FCWXZI{l z8NH=mxZSciBN)O-POQ&0D0au*cSZr`7WY9WOsy<5RN@&KUqHiHsah|p*W@VqtQqMM z40Yq#cPI6gS_X5nlzsBiGpwXb=x&1#}ckwS5B62A5*Cq%s+tM#u zu9te_BNyd0i_@C<_=H21OF^P6eFfneThrSi&;R9#u#`Er9CEK8$~}P(jX7mtlsPuo zFyTalh*#Ws#S4elQi0p;vJ7WBkqPnU+h@+1w93%D+m=C0Nt7X4jFxWImCz$d4_~8M zJ@4{a5XHh*bOMHSpb%91>Y8e*^iYa)@2vXAAq7(cy7F6W?|0$sk2W`d(GRtwzRI^Sgdn5R;WfEwS>8r4 zigN?P8Tq%mU`r8U(qMn=Ve#aWEEaghZ%0ZYbC{XfOLm>pt?~0dT)W#UV3EP|@NW?b;HM6}-nXOfR<5Lt5es=Zs_19h&6U(m6a!N(GksJC z)$vSZNkP0+U=-&=%LKcR?~%`a|*Zs8*5{UU=-)4fruD9E7stA@@K!{`)|I}^}rdW z7ktv|(r5mfjrTQ|e|Nou*K2=PiM5Vx1YVZ9|Ga@)KDXZ4`S|2g_tRUv(zD0xap%}t zcMV2cTm=Dle=*9JmE((17MI~+eJ|%j-}?t0y~14B39L$Ox2@nR0`dWK;lbZ33C>)= z$%lLh`DW@_+ylQv8GAoXR(gfkmSO2Qy@2x`1R!Os;Z&<8AL2w9NYw5BhB8`(L^QJ+ z4=2Jr_UJgAyA5D;wC&5_gcT76mN;S;gP|~r^H1VnWW7|Bw-|ZoQwx@IjS4@B=g=l{ z#ls5q&D@I!Uh(@J{nVMt@&Po<0k3+RWndJi&(ZgC7plFEQ9kq*MHXjZ{AKyif&{`Q z%YPOmz@n5{K1t|T%aaSqSCwZ!+pS@DTj44Yc_^-gHxrOz2ZoyLML*FlEF~ti(^}|Y z@Qn`*T<&jVw~N-*(kb|vWOqY59lE3^JAs#7ar#=<2xd0c)>C-E7p@4EL9AM;@RZ&k zos~o|+K1hg4rPGNY6m=Fc-p}ocJGBX5Wavn3RFO|xZ<>`=0+icSNyhcgu5ZJiRuoO za9-84i{Xf8Oozo@)oT^;j46l+iBfh6akm@Qcc^VZ66G`;i`CUi%DY(rMIax#!jys0 zPV8oR$X*^k?!j`A6DqQPL{`Nj7{zIeYsgY24|^Hbo*y9w(SG26!&2c_%u@af%%*Te zDSHU=-)QoiV^J{*h05FVtT9%mt5c zGFvtXZ^%E;)-M?LKxYI)T?_~N+!Cx$qANq`n;N~Ft~EDtHpw_m0NFC2-j0;BbC2<= zTmnSmeA+FQ7A)p)Z>T`YJl3&TWxc(DJ8iuBm4N})|HCswoZm2%|3j6jN6}8d8$DEs zjE6e@Z*AkW@xC~x4K7x7@bowcm3s{=L}Y@q?1TJ?W@-@qoOq)@cPEsV&JRPUI7;1S z{S9CUr$61e3S(+1^p1F!Sox+hCo%$~IH!>B7{%DthR9UP8BuT=68f4EJmH^%s~jwP*^bKcXB#r+RA^y0U*d96nXhHw2|vNR-Dw6>|I{LBQYAV0>GS#<|9uNCTL^wTI}7V7a#A>ge`h~8XUB(* z*tJB?01t6VMIy;3ac&3;D^_t%iR)b=4P8NaZw-)Wb1w<%$-OnxU)=4`^)!U&Zoya) zM4nOQ3RO5B;$sZxwAlbgk1Ch)QR0Az2NLWDgf$ilkA;T<|yWLkF6~Uf+>S zWr+ec5oJ8dxHZZ0wq9P=U4JM{tV;9lP#reR{Pc=O2>~u4MGJjwHOWV;c+Tph-WY>! zqZ7smMu%ca$01H;AQro2>&@%``Eg>JxgZUZX;ITJX$lgp&#Ki+6^WEhj5SZOh}D#6yg~urkDPYy`Z$M$z|s}v!Wa_RuIvr_yi54RC?=RW zh>d=){aFY)R6vCvw@XdMO&#bO-wEig*%m5-QJi!R;bv#Yz)A`^S^M|S4~_4D6*lHe z1fw`9q~VS(jT44~^CA!eiRo11ATWxPBK+EKa2v_@R6%xDEhIY==ecT5(i&LaD2h#8fcX@Mb} z+tDN+OeSw6zM3sFs)a0eI5-w~M3ZPg$T9b-@pb-X)e1RD^1tWI$APJDHzyB1$WSvb39aWZ)UjN%+cICsc0MAc8Mc#Al(+zX82 zJdkjX&C<#15K@mh$(1T+w8^fu5sc!bsv*Um$UWuqj1wzS$97^vB_c7Elk4iHHsc73 zNZ|jJpgw}po`#U(o|Bz9)j0Vmfm8mZ8zRL?8K}iU0rix%H?43O3V(a|88>kLe~)D{ z3E(2EH{okIxpHrhkl6>9)7|u2Y?5tD${brFuJjVWQS-e#Nk1wQe0T&QMT!!>G_kYy zvpoE5#Np|@6hZ>hTR{;#anHjaLh|I{!;>zRQTb?c6Ih;Xk1T)ED460j8bY3^{LNi( z>X66o0nLKb_z=M;PG2i*BGC}yR5ukZ0&_jjpP@*G@SBa@hm(evr>q6Rqhe`4g zjEvu3r6_ij_ircSz`S-Ms!iM8ammEMnz5H%d+%1IG@ZP`_1rcmmQCh3}eJ)U=%UE0fHxO8svRipuPkI z`X=eU60PK0Wl5_YpNzxPp+$(h`@VDoG!$2 zw?eshBX!-*zoV7PWX{nmSiaGvPd}R6sXQ~bFO!{a8e99m!qu|MP|Vj+lRou^2%zjK zT|I+X_(@l1`EZUwOsnKCdeGr&d7g|GwjsE!C?Ef)(l}HVM>J$oP2J4O#_5F`I_~fy z*K0JMP?@}&+uc8;)UZ?S{xwx2@t$<`tmc^~(dreKW{N@9VvB3yRdQYR*1_*!Q!8#l z9WjesFAx76&3Ic~9w8@Xex-6kzj9PZ0P1D%ybi$sks|@M!DV0+XBV=DU8TZMT|IU| zs7_E@Rf9f!Xq0YkrwP_O=aa35JyL&D6}C$+c) zK;CrXJ_Kfwo&iDq@M-l_T);wzO!!*)q<-cbwfDVrcnRf~XM&Okh83?HaNDJ3nD-$? zX&_>9{20a0M6kZ{u<)5v?>_I~Ccp3Vtfv9HYgmDf2dJYaB_8F%_}Q)Td6}s{Xnj79 z!EKF_JC0p?<>~aS@E-yoPY*n6tL@s@uWR{V8_-TTs7YWHN$L zobM6NR8eePtXSBM%14tsi;qvsz$i}g&fM7qAM#TI6T#RA@EnyG3oHLu6|twGR$HuD zT(?XWkU7H7S$bLfI2i5fBx}^zDrFozheulmMo0Sr;XIc|Yw)Z>sK_b#=_RUW=j6an z423vNW+zmx1)p~NMKBaw4kDw8lUN=~85qU+DdC*QLos;aN-7LXAYlDj|H{B9&Tj)c zZCs4ASp~-Pna(mWit{VNIiKk?c=eS<(n{Ff2u5+99r*o*Q?{GI*r%{^m4Q*57Zc7I ztgvPT7$@HE+;64VR~mE$YQ4DWS$h`2C{B9nT@==6P_2-U5@VM?=}DmCG@F{^poL#Q zL@MnWGb|vmR_UIN;&m`Gw%$bWioZ9}JeBwsu9K`8+vJlayLbDt>+IXWhiG5%)Z%+i zTWQJK`D9u0!Wj?UF+VR4A(-erd~5R0>M0xNlaIa|(dOy%uQj{gi?)7#smQq_h;C~1 zZc@5DuZ3>Y3h`xNbSP9D-VtI9#o(pPp1z((+gopO$isIFczVmgEB-!&|5)$~_-)I^ zkHPlH2j|tP7?y!ioIW}jIYVlF#7Zj78xsw?(sYGxPz0~|4L+P+-Su$Tz9abDg6d!= zB`}I}B+-zT)oMd?M~xoCs?XfywqZKUz$i|77_ZnzT&sr0@u>GS9WU5chC76Cxn|Db zsP?TPk9=JJ`AiHUTCO7)?XE!Gt?p|X@e2X|VHkv7s2WO|OKTdeQPWWujlJ#l&!(4R zR|KPd)e*a9v86GbI%RAT+XH86Y0AJT&T9#0BsRLZO7b}J|FWVI!6?q_O5)VrlE_R6 zoDq!TY$Tl1h=x#QUxtpouP)~G{?%ef%iN2!UAU}*4%VY=O67`JQU zi8oWc^(FJxyyw|r@1=O$^ zLwOq6q?Uo95qk9gQRp@CfLSCHJW^kBbc#f zWc8)irq0uzdhH+-^m2^^6FTsElk9-m&;0%A_ebX4(O^12tXp1+kwd&-(JE&={#=7~ z>nq*u!@_K%RKWoMm7XXGdC9>$6q>t;Gd{wz1hOEm%r@fCOWsk9w_g#IVZF{WW|Kw-Z#ZaPT+z3W-zEu*Zq}`-yh=(&{ zoDq!T^aV#AZ0pBvbo!Tk*VqLs=O}_voJY~*A5C)_dIvWqZJt`M39;$@UVOjjw0-u(*l)=i z)(D1hzK5NQqX*U9`@Dy-n5sUc#C1jo182)`lJHn?^&~3L%k~8cxdH<7|2_IDtPJiH zJL2BSMED>LlTn+chGMB%?X`ZHgx*)wumqD>b(FUk5a{a@94B z|5LZ_F7UhXI@U3+Lm5)Zv#IBDFH!j|4}J^@$c{WwW@~hFB!&s#KM}_sfG!nRL3R3x zdU^&zn^|*-L}@r@KlJ9F;uX?Dkm^P8SUOPf(u03T6`(KU>9es2 zC3@vmz#Dpjq#l8T&+l}>A+Ktzp_sJcD8&!oqM{B%c=4+9bokr>Qxc>M_PJU3L z5AMuMrSIdwweW+Lxb#4Is>p)uONS0j0;cbwG8DShxA;1@!MRsvNh~B!?z&| z9*#PK2mb$~I;ZA^OXj<-m@Bu0q&qxNJ>u-GYWfcwgoC|fC%yT=w7%PxsNyQ?X>xU8 z_f0w-QbjKL8gMoX18MjkWjGmpWCaiP{)&b-7S{CLmN@kiS%@uYJ-mZtCgc>$3|x@o zW(_#_k{9=}R3tiK!?_WP0yn~+&A#;<{f_!k>l*jNQ;oHUW8mCRhOTf%B6!7rGnuZ4 zm0CAy%)qTLe>0!kmYjXUAFw{GAP~V5{*y7}g5MlO7v7j21cPN4LZ>Dko{k*&yf~&m zfLHuyVBU)D8&TF*9t6aM*v?7JmfEipUTy@V$Zsd)t5DiQrUUcdRiL=6Ar*|CLQZ6O zIv&Aj2R7Q0QY5<#;=bq3Y2Z$=2c2unxl(1rggPqwF})7Y!iFqUkrE`@DIsz6a6}1C zA=i1)t{1)g{q6(&=GJ$5A$8g%?iAf+86EswyQ7~wDp1v|JE1C(+WE>i!x_OS&cDd` zoC5g*PAt`FR>6pSR#^F$pK&_XbMwotUUwD@0Ho?T>V69#ZC^;ZKQZEC8ZvNQu|K#1!<+Rkd;kO1-4YQ?Qr(c2`T6($+ z42}OC4Bhz*sK1%N2Cs@Td;XJoo!Jxr@#lxh z!0Y?JJ8|!E`DEb@*L-u*gJi&u!R$fKY>wG6!eh~0mrNbd_R)cPcPOjdX`q%%*NUH; z-Ub?xcl2BpupCCDQj;8W!@_+l4;q+P5GWXxMt6r}U5(9;4t&a+bMJWaGGsI^4`5)l z-IyE@B#-o5x@`PieYbTl(eOeLHIm16@3P_I`%fbO!`EYPB)a?zh6*fBO;U6lJh4yp zoV;6!J2eKYS--pe<~u**_RPEYvum;XNSYV?12e-M2=z6;eQ(Y0_gNRCi;^ys0T&PX zU|5&S!O~|K%MwyPY7z~rSI(O;WrNLjEU^n#23$34!yj*3d+ogIl;x4?69;2fBOd)h z5_2^MhSK_VKbdOQ{k!Jz~h_b?Q-g!rXHgp8QG~G*$`lFq%+Vk7V zNL5*ZSz-rvdiCV20d4l%_VU`idp65^ot`76fBkGS+h;I+h@rY6q*0Mf=v2+wpx4HI zuHG*1jx0UlNQ2$??!(3&b~n2$Ug>T}=f%FYb-olP6$qGsRGu^+X*N|3m@pN|FfeZ> z)hvSurgM;ru1(D4O+^_P)$(mguCF1+<@g;+N-#?ntg-09pH9Ky%H~~PJ#PPHBY-nu z=KB1z(lCni4B+%NCLsf;>uV$6j4HDNJcW2gTt0BMfVn@I=8zgaiq;RY6w?WWn)G-8 zH^6wN__a>Y&P!-?rxMDXw>xZnxHr7g!HZW9j4m}$cCCnWt?~3Tv)2px7wD|85ZD-H zXb8Ax?4KF;OFnVYlW+r(a4ea~a2b5P&9Yq{sKqk)(4Ace_M1&+^mkx~^xDWH7-jI! zg!2p@mci@IcuC?aVVWZt#o3dDxq)a1S@mw`vF#52dDDC{@4zQM-t0Kaqu-Ow{Q`jU zby;QSAq^cbX}c>dTyzzPlf2;e_c-YN-Z#BIHXknCIo?Fv)K>2>z=(!9kSFRS#yyM#wwPy!R3PT6;T}e%&4wC#F`@ zAqA3!Fcj-n1yVkm+-Kz9g*zwtQ>r)5c%k(}Ii7}c0!?&STZS_z5ssEnbCVm&R>6c& zO)mR1ooI2`5I}`JNN-YOY2YHNY0cqoKIocp2SdH#8A8=_DqOZ9xsUE);&?bL<=KsI zRnFM0&!oJJ-0UHb92VpG`NwSf;tV#xzT3~qwfcfM6?Us(?mIV{@#xOfErvz#h2?2j z-U|y=H-eXG3*b}y6#Gc{p+{Jx_YTBQ{jg3Vfe&$N1@%jB=y6j=z_<}v=Cca`2Vc-o zw?6ts!8Jz(beqXLvjOYSIh#HF{LCeIivsXIId?;pj$rsu_9J-3Z^5`wpVU1%VB8hA zKi@xic!pl9aXYg>nr=n#ir=Dlp^)c%Ce5FySpD)1HEaF|-EyNDrzt`NLpV>Ose6~K zs1J{YQ_W+<8I4pbrKx^~%*9ot6tqA$o0bS(@xM-X>r_@E6FoiCx%2lqub;KnX`uNv zrnwA^;-n{l-RX?e#>n%I)WbO^T(Rw?Pd5PPDFhR0MFgWbzoc>bB3|L#u@<3n=2@dI zoYi`k1J0MT^A*7mPOt4RgH9@fQC*8&KP;S;frJ3sK$&^a!+%WN>Vw@*!~2bbpf6k) z_g$7T5xjP|84bdlmvA!WusXqw=+##Cdi6EqG!qoTD9(!r=SHMtA_$sF@gckKMwk@x2uy})wJrF zb#CfVs=!VeqSIYn4f$;iKF^ol!1Oza;t&ck_^5bYEvjluJcsL8$jv=Xo;J_v^wW;KLv%}5s`g`;v5QU-`ZDgy3)n;fXELlNo8OZ=N`aW?4!J!>F|VThIw??zz)wg|K;dU zFCMv5K3RP4Z|_{vmtF^r4gw<>>h1^9o4;xJKhh>36kLt1KfCJtpY?xmg69-ztEp|>5 zL@b5sc#8lIEg2yz<%O5%5)sqbg8UP(~9w7Lji%(npS#1L;&ZznN%8Fxt%y zL^>sEHN*MEChhLn>M9iL<>Tk=fRmD*n&FIK6sI}pp|nWoHyM-v=-W3o-U&F>34`&TcDS@5*4*LG>fcm77sj z(@4)c*e`SoaM-CoKL1=5xdv3+rCG-jjAEtKrea&fkP5Y=fWZ74jg%U_ucRL(F3`n8 zO2c+02@Ha>HTlra$r!s|S%Gapsz?(CG|j!RAB=Y_u@Q{ooJlyRWVJvQ{?7MW;SM=& z(1~mjTitRHkY)^v;0eD!KktBZeD_<{Sep-e;Q{?_c(SCij$jlg!f}C z%+f3CUYQF%?8FmW21aq-K(8 z07wTY+~Zj9te2PH>8-mtXU(&R5)#})JQVZvG=}ov@gLrE>H@r1?apAFWngqD z`warlm3SydgBrpMj_7mc)<4luqWw(-qc}Gv6G8ENNPG#LV_tstfroFX2hMaKRt83K zT53hOXAzJ=Dpr!vwUh62J!Y^$_FpAT% zNy6#V;OIF$>(+f>_*5hNS_)T0*(VXa;-~Zz_jGnDRYZS%_Wpg|ynKq^?GoDyuM>U` zABI~Zc*RetDel=U+gS{760lKzvrVcO)+zmb3E0DjsvW^Ae#&EU&u8gpON=nXA)i+7 zy516GG`NqmsvW^7PD*ugFSdwN%nR+NJx|`HJyXsXhJmD*uxk(U3+e%L&i7OP*m-L% z5=NI)v2C3`xvq8t-QR97sj5M$i17{95%qMsvRz$W%?RhJX763MH$oP5Zg*EVy{-ZV z!}^saP1m+2w}>UK3`}#{cKL5;+o^W?@J&h+XXn>?xm`Y#xH2&Re~25lacSagC1N>7 zm`*;FxH2%!i5vUhaBhd1x~l0dk!Mz3Sf-U|mAo=Ao;-5@d=h6Gc>0K>kQ#mOEkUec zl`X#{S3D|B#ZCgEue`*f7UbKsNQDQO8k=Go`{iLrm#1DYhQEo|z8hctE6LP6Z&pNT|m+e29jVgc^ zR);Czha6i4r$uV&6YX-qMjPfqS6nd zcrsCZ9dXHDxCy78%^aptKK=uWm4t58?NYY2zWNR|>2W9ahrC^Hce(FTfl zQF^maKK=s|+;Y4kd_-8N(PymmL#e!)BfYT z&2&T^s$6;+0KOXduPoR$529@8AyyY$)5u~}cSn?fC>}<$bUg-FTy=Ey^c`@SYi#4R z`q|s>jiA`}g$Y!L+B)GZ3Y_QYA%bA2LTANQfWiNoSDzhfF`zO(*6!)Yt}AU|!p`&z zZPjI$UohxRWQEDXWH=)j#d!j8_8gkD)V-ql*ZcH#3Dlj<6WfE}snG-ZRAF#yJ6#XW z1Jf*1-r#icR)jcNXiY;om%FuKt_HdI=~xV95p%w<67enPOQwKXWqu(8^S^NPWEuu$ z@X81_t)r}m4XKCDl_MG&s@?RF`f7Qz-lAV?Z{33NlwpyMrz`h{XNK{@Ms1xqCFrfp zueCS&wd=`aGkD$2vB^NyX4joyLYxtd;-vAr)flJEpjL}K*H@;YOI*<`M=*+$Uh8z5 zu!bAX&F2o;wCOTrcJWo<^-JN5U=*i+Aed~X2rMB^b{n_rn_UkYolmy?X^WZn9Sq;v zEhPIF*0+dY6z6VaiBf4aZ%!<)nqZwh|9D)xU2*n4c48Izjp(P$@3bFZ9C3B#vB9g< z^HS0qw4cYyRt83KUQD#lV@)!Lg;8OFlnNY8?tI2s21apGy0x3nI8DCv@SQE~R-iM< z(y8GBeP{TBEQp#{w)7k2T`QJSqfD`g!YO4h5@pd$X}m=~nq1Va{Ha-%kheuttItsi zEF{WkP$!9vIm#%CE&2=a#qJj4rQ454)RMISGAMKyuTYmYEX0e+qEeGaD^VhVhuswn zPOrwBAn-7dX(S(UIDmB=u}2o*?KZ7OxqlzczeJ39xAAO4W2L42Q))u+_=Z?ON37twrR-@WY}x2*MlKxG9SI(3SjILo+%5JK+qFLE(aQl7 z6VEb+;zzPa*KTR@+#0Nc<`J`h4Uc%s748{#|Ni@f^DTG#*I~!yeqFI5?^8H^_5=7 zj%~Azmq_np`>d52Z12jzYgc0-d2TZLYsP!wRl<4lYpZ|g{7cl{=MP=f2ww5i_QZ8$ zCYvmXAf?4>aW5T1-xb!oV2^{Aw4-9o2tSoFa^Analx4>sc2biZ@?^Ig4jukNK54u0 z3PXB#UJbI7t^s9W6z3j5S8OBIk-(UvNiV5MH-^a88*|Xcr=QiE5O2rKw!yKW;}(|e z*-}~y6&PS~fR$b1@C5z;8F(O_UVxg^SUVxTZeog{3-!p4`|P~tknYDkc5+Xkb59V?-Z=30*P~jX?IYp`HR+nND)x_58Fuv- z^X^lci_p)|u4GL^S3Z&29|Vlaa!*axp>;<-w*9etdq+{LR=H~lKAe%S*IIJIrGKM1 z;j5Rw`co~2d6emrQ zTZ?fT_r=uOOXV@!A{fGX84bx_9KC|r9GV1;wzFCJ9YuQRUIfN&tB~?mR#my%Spq~< z)IFBPB7f1Vi%OZ#i@KUAGnBe~hSg|CF@O-yPt6ym(}v~ zD)CK)dzeA8s+w7oJd@9Y87a2)Ih8)yA$(JvtsP#X4)=8b0ExbSq}wFhi?Us0N!V_L zD2OZyZgF-vI>EAMZ)rHNJUZehv$W{sY53#0EQs1KhW*YbgL4ab0{&dO%$h)VqSTbW< z=EpXi)v+)H@H4j6^tIw{Wd?cOAgr)f- z7{a+a(dm zKJBpKPjE9p#a#TkaB^4tq9blK^YzHenz{*7F@Q2Z)y^mb%H{bW%HTJjM0Y{9&Lgfq*<~ag($PG% zWncv7S5%AvF;`>|6#P46ev&H-6EV-H(L_*_^jo3hWuD_)x7(w1*GqI3A zDU9MYF$yJ|Sdo9d|!lp1IzDZrSP4fCdqU_-((~Nrf&;ir6MLtsSuZ|+w zXj?fG69L@Z`U?T?PZ=iaq# ztfgM+t4Z!UlWC+zOTXxcj-&&@>7&ysEAg{c=x5ZVYXP>N>~3`O z9<-;*bB4q0fjo7c)2;FIKHDLU`JGi!Gl0sFpMyyTPRl}7PI3@wAc+IWO(XGV!uA2Y zoa;4=CB|PprA8lkN>;pmyR-iF#U^>T34>S`tyFeOH^}|-Zc_%avFbX>3;S})(cOu- zNEOsW$fcDxg`?gUb^u)%#@_T(Y7+4y(kKicePuK)bb-Xb)0tCw-jtSzRBMb)5xmOV z^#q?{{*CLsJZ=)*s!WGw zTY`+Mir8J6WM@?JIt(^_q!1Pd@3D8ZCdI^Y8Wrl@Jkg&t(^5Z+OC42^Eurn6XHgTaH4Os@F?N1wcgAD!bB#=j&{7;>@sEYinlCJN}0;rWbnvQW8FS*)kgmqz0a_` z$eT$DPdy%lm4t+Nq7+0dp-hyCB>x3#Ki7Y2mzRmM15BN6aVvCu)5)f#d5*S*b7`@4 zA?42vUVfvtqf3pCTlz5u)+!K{!9 zk=(`A;(TRbbXewt=lmhPZ<1do{`K-^g)Q^RqM=lM5tA_Cd$xW?PLXK7TaOqm_Xr7mVr#c&EuPnVa#+w4JR-7 zzg1{fM!7mXlFqI;|0Ji+Z>qwi5mhJoD1npNk8?!%0Q)mT4~NC-hm$S&AQG;G((&;^ z*HQl3!AQPbh}^uH`)&-mwN;82r(0x|x&^W9!K^N2V5kp0!f*pgCc|VR9v#m0`fS(< zoOuy)`#9UJk7)X+Nj-!BMsaSBIdG$B`f|={lkl>=T3##EG`Z_|X)gn#IQJrb@t$Xn ztN0#9G_3i{o9nK3&zku}?19orfA)Ye4B?zXf2Jq6+;udraEjGrOM+ul{C+RpHFJMnuazMz6e0#l5`ZShOo4LokDUE-)o-tRN_sKeW$*AW-q zI(f^jwjgKDW2^*C#Xi&OkV#C5jD<-!`==9IBmvO9Hc108+LX!iA();ZOdT3)r`FcZ ztkoZy6D#nk-}bPNTl*RJRyutO-bTgnxH>GyNKI}e&y}6a=yZyQEW}(v{7!=-1}^+j zKU(K%88O!1rpXoh$ib75D?bw-@%zCkVUO(hlmry6Wa3^%W>#eZv zO!UJoV%_kk7S!mSIsI6)qNDr#g)=AswJP0WPBUKkz19Z2G1yGGq6t%(p&4^yh$a|m zT9>F%cP$+P=N))eYXf?UJ*#|59@QC-C)$Lcd&S)|SuR1gN79cN;X7>=SaEhz5tZYj z=^W+JPTg)yWC`?UO;bTC10w?utWfBH==RbI6EnA*OJ28mO3I7GTCK4GCwJ|I?adm`5k-QaYW>d*CE6$bV zaf80@)k~M{*m8RdN~o_C{o zMAnhgBH1fYp1Bxmp+6~)PrnZ{Da9X8`fPMq?d9tth3_iZ2Jrnh;v+Sw%Z?{p_Wfap z&(F&5SRzM6z-ff_iMtYEOT0K3}fd~2UUtJzc44v5o9*^L0l~}%HfN{|r zlk`PDyt3=~zJ7b<4M7g$K#8Rp8&{66o?JDfP)8+}ZX->j#vYoN*i4h#g?z;@45glt ze8j7Sg!whzuL9bim^qc*PoMQljk$qR&96!HZd0N+hu5>G(d7z~R2EkaBJrW3FRJVZ z_#FLUmDId8pauw4m!=b4%hgpUk{S4Sh1;EZZ0m)l|B6W!AvN*J2Ks+=27qoPLh_w_@QC2jf{DcoE6h!r(j{KbTr{>Xx&D@)9S7$kDMgao5y zbXHnpkaTgj!!QPcizgq-CYpV>51|YuJ@vLpZVTAXGAJoKQWeZ{4K6gKFvO z#|Dj*xQBljKLw`?4HeKE<0&%?;w*zvO1C9(Zc|w~9(N9Lh8JdYDcj@&x5mIVARbHv zQKQbGekfw{R@tizFUrcY5CkL+*^ zG_;!*Uk>(t<&CB9rTA)~Dfrn(Bh^4L2HPQmZt4ltQgkLfA*4oFf{Z2NlZY;5@ zWZzEnPuS)64NbvKvqY~Gd1nHIc$8m|EViXRlURNugf$(}R4#-vz}7`tHr#$}X>7uE zw4$4MBL@Nt;VlEBLP3Oi2%ImoP z%uQ1p)c|JGiusmqnd5S#pcDNx{xKGS$kMFD({`G-CpmW|i5)fS%ByWrx9SwY7hB^| z?V3UZJsnZzH`)jN##QsY8@%u(5l6hl-a@@8d7LX|5L*W9bVZKbxGWE40Wkx%CxaM+ zcv9gT2|BkfFSq7wsPn)efeEnSLe3Y4dpZsA@Vj>Y6a%YDlonXHOsPnJLNzQb1SxrP}@3#WybirA>apt0HRV%EBZK@%8d z$OFWXJBWsyvqMoy=1-`F798Yet4#tmyl)D4S6cr_>hxbdzE(j#vRb z$B|8n@rB2Fi}2;)3Ku!(YE;yEMG_$!&2-Rx@~&mgW4gKSu2&NZ2)i9v;%#^tlpRfI zjCkdec-*dvxny$1ATS$}??7^uxjqznqa5W}Dh7fSZ6*r}>_nO%p=|xv*ZWMIzAcKK z(@Un%MWc+_3U@0!mG#|HWdOGdkJh55lGBsixYOMO-6LC|+e6qeS-0x2$`XN*RDr01 zk09R)yda@Hp{FKw@sZXzb;oxnd^K`2ucf*laR)F7#wnF2QCCgqEdwLmEIfx**Yzbf zgr{4IlND!muce;mN`g;mHnv6ZivLG~zl*Ja3HT?yE~T2t`Q(W{#iv$2k`A-U1c47_ zUwNoNb2nxVW0#uf$PdwL{84W!3cQu9sH-|67VqiAdk6hjyVmYn)3 zlpLz|;o}01Xr>ns>cC~GhUpEcSz0-(rm7iMT|8y9I+7=eO&xY>*r|5E$w0JA9i9XW z%XY4K-Vfgo+W-Mc&Y#^3lMua&7r`j@!I)TI8(=pd@v@0Hp7f0x%S8D{yW3xSv4BuH zMocqUSebs@Yb4z#->DYyGOAFpFXev=8l053j7DiV3lmW6iTVWeGgOKL+x<`lOE_9) z4|dM-A;dW1ldE9^WT_|urtsS9F)N|h)tdFX|1Kk}Pm?mWCi0b;u zAP?A=a&D~8JRqoxi=^)=W!6(J=!xI;LGH~SlaAeGV>bh%oxr30ji{kUJ$Fq~l>|Kl z;ck&IbG_4}AU*7!->;%X^i56;X z3AtzMH%@KulDzwzv6!f-*QIUx++|t7$$O*+yEgl7^ow?X}Y}^*Jh- z8&MX)DCY|$aq{hDT>H4<+*NM4?)-n2!WqFR&UFdr_r#rCz@4sDkG;BXHOdti&zv)9 zl{H`TTC*#l3im_7TzWIv=0v6<-FtvlQ<$3&fz-2*454KGn-8p8u;{1~Zxb^&yWof+ z4Shht_NPwwLj!50lghe!7<(L0wyCq#=kinrYNv$)do+5 zOqlg;a_jWG`;mEM`!k8mBJh*Pn=klz&|8gj)iRmc2slcs#;05%RnW{gIkv zX?oa^3_fqk+=J)rNk9Dqe<*xKX0A(G!FB?bOXK&;@oN)xrA<=kTz3-P5JWt)G%dAX z-YsDLHg_cF!-%WrllIR%gB)|5OGR%DX-Gld0h1Sej26+hWZ0Ak$5hN1WIFXA$!dQ{ z9c1+~8u@9MT|2zv=X@$4|NRIiGzjW(^OXNufvNIp+J@}lDzgjKH50j_6|z1i(`k#x zPfdCNiyQ6uH-b@|^@Q^tG+?YLwXDeL&gU;UiRl7X21aqx3gc!nod&NPu2c4S zv-s!UDA(hf2qxTML@TL8?j`b8tHhN!yqz$ydPZRaitMMwAIONDODL#G0~shF#!9t{7bwp(n&@KN z`s(Vb^w1g=2r)u`J>$49SJ@oNSMt&1da`62aT2J~!MKxHp{#SgnkXO2CAv;o?CLv5 z9q9?xzL0xLhTV15fRucw550^s!Plv5sI97Rn2e%V86Hqx5z1lV#I4E~ObcFnRxt8HfXP*jdltEXu@je?4S%mZ-`Oq;Q0!fgQ#K&ktsL==66G>xf zatl~;jc>Z9%Ecfdalon|aWMNM$Y^kW6M;Rpuh8#nnh7U7#gK$YhzM#ltt-9Y#}USi z`h>?D?9t$bBvm(;-h8pxg<4GUN8L>Ypx?30|yYL2KXky{LWX6YqGsTt1 zSK;lKwDx@fljDRdX}R52gwA%#QW26+D*Gb ztY3CcB2I-aE&%s@+Z0TaOtmfOr_`kPSrJs2ywx@TdUgZ2NofkRO(^?7nyLM1g;W+X zma6wDL49=k2uASLjc9Nmn2MQEEkkf{CXqpn%AATnW|)3d_zQo{q$D0s?4Tw+bHM7x z{wXebMyJ?p)|OZ-()ES8Mc*b~BAXhfR`H^|$6`kuCir7`5wYMPN@hxTkXp3P_@N>u zAFfx{u~3IbK9mBRsGOf8o~YczvsUV`4DVWg`s?szZ=XSu9hK)7QA|&+aKk5^OkZV` znsmIlTE*L%ZOJvCOHBU)EDG1CFrvebGP0F*wPSE`)9 z3tllA`pBYA1HPv;!6R9Jj75^^sD*K*CYC9ON|k)XS}1NpDoo!|j80$m;fHp@X;;G| z9|AssOrn36rT8hwE16JTdEgeW6O;FTmnoD||GTfwqD07#Rgd^#KIO&u0xi-F`Q>XU z^aFyx4)YUrEfkku6u59O6wNIrfzSDzB;O?4zBA>Go*(R&PnJ#Iyt?*5sruuja9iPCc2D*hfT`cXa3x7KC^yXod5oGP5)J& z+@VAYt4ag&77On@9;vONoIs6(W-0$5a%xMR%&O*SNiLDeQ7i!s&|*lt9eVRZhTF$% zN&M5BzL2Mn@1}25W-;PiPR0O!5qN>}KoHGJ(?SvFu7acP%VCvFaFd@eN~2kCE>Gr) z9+LCXBZ`-kt2?c#mL7p>w#2LVrB3GWtirayFm}yCLZq-y)<7|9f`Pa>JO{>LS>sdv z@dcp%*rUaY0S$U#o1@xXDN!y(eN`Wkv8&%g85dKyjx z@Jm=)t7@U)O!=TaC4Vf?fjax*c4EvDPD2OW5ve+BW2u5%g z=&psN!yCK=D3S$#@3+Plui!KmGypQK0dfzgeV2hJ{BHxw`BT8?%_?C>Qa66bv1`|E z3X{8#X|~ymU=*j9>>|++o}^43NBdv+pZo;P5a1lf%aom@L@c)miRC zG`SPN28iDIY&(#KswQNyWRNZg-)@$S0)kiZ2YsRxloV?|kz1TCcTV zbj9Ns$~0zA85qUslg1AqI}zfP4O24U;7eXy^=iVo6WbD7TxGRNM!f#z>&spJSl+$Z z){oXeSTo@lFl*M-Lh1&hnt99(3-_%&XdoEEHns!^GX!DcF~ox~z9$AoGlg2Z777z^ z>6sVbee2tJQGNiMfD#??Wh``bC(~jUQ5Li`5{qXMmdEd#(V~3O5nayY8WtrNRdrTh z{PxhiyD|%+QgC%)_f0w-a;SIH4ktPCahfCXg&DI15bhPHT*JTIxo*w|WBl<7!520m zrESaj%fORFc>I3-dzawD$gR?|QPbNqioG_1M6_X?#=i(gaXtn@ioLAZ7@iGv)rh}Q z1g6R*6rwU+RtCxk*%LpME*MI2W%)5`Mumx^DoOI;&SmX00{JhuNczq#{_;?UNg-1; zWIlJt<9i?QPh1{aw4!?hion%nKgmvgPNE4fRess|$}ivz%w^-3oq8-*V$6g`1sa^Y zfep6JlSCm?75W|#^BY@~Tal}{Ta{axBkh)+HgNZOH#Z2>E4;{6Q_Rf>MsZ#OoY?wL z#G40nL#O#8IXnq)%H^ZU^~g$+{7GRH^(jPd&#ZJZl|h`TgmgwQit}y4xjN(gjZ}HsY|8ccHC1-C+9a zS<;D)6Onm>>SuO>X2_WdOORak+rMubQF$``@?22hb|89p&F-qyQ{u~W<20mV!Y7lT zNPtIKa2=uaMj}*treHZ71YH08xo>=#*$!W*>z`xKDP77PJXc;NqIxBuGntjA&Cq&hck==4coN|GSyg#1o2!R`uLp>w{H zJeQLfA48oDvT%36z_?mWT(Q**U;wd;>BrP4oPH29^at)PwqFJ>394zrH$CK#KBi(0 zd?q&93sphbfj7TNk|`5DapW6#M-f#=Q*#WOW@||M>u0U?)6*yscG;h9(tX?QhHl>@ z1rNhW5eyB*<2-=o^?ia5*Ch@a;PZqR4WjXrpB06fX_AcN3(Js;X@X%}%5wOhKh>jse8<@~vWZ<0qZ+vUn< zoYd-HxN_rzW*~NFQMl?W^5d!oE_^nfvSe>GY$002I2*c{*)ZA%)g}ZvdQRukL-Es9(i{+kJh9}O(fg4-~Dv@ZvBBm6~X6R6ODwL zM1v_GE*`(btR}j%JPVA4@Ros54*KFq4>QgKiJR1<0*v5Cr`yk;w(v%Zg{5f{lWA6{ zNv~>QcXIa;XZ-Qas>3M`njteJ;j=-nSFCZy7Ww4X<4<|_)z=SFZWdM~l)Dnixx&#H zk#CZb!_VAc_>Ft#lf#?dylm(Lr}@3-mVhcyl`FS5+mRAWf@qc7hPhyzRYg&dwRuG_ zbz>03+T;|$?Z@+DAmWOtoph_Q^I#x?KehWvvY*1Vm`pPkq{p*4>6v%Ofbp2Lj{wQx zIF1OS<~Rp1y5`*tOfH6==1HmNBi6o52AnKLk^yH}bkK?=&0DooD~V2~^)c7UyKZc$ zRwr7h(a>XZ$DVCI?J^oKWb|MV^Z(UL5z%&QGKj4hObcwNJVW6+F^J`eb45yIp2`Vx zDtHA;qYIPwm`4MUO7&ykZANv3${oWPy+cTikuKM<-3$gX*Lp_rLSr9>FAOji1I#nJmM8LHBF30hOT+*O$y1=FusqbP*bYl# zRZR`v-l}b0y(I3uEsxTKSV6=sckQwuD(4bMrU36@+ z-Gsb5f~E2{^yjG2yEVzyM~ymi*(zJ*-7m}t3lQl>M$Cyk(^FzZ+#}3YZ?>)`+e|LD zVSH4MsqK&*wDLvgZ7I`YCPMy_PT=nqk``EwmMASyI}$yJtE5U)Dy`D&@{&0S24;`# zCBW!9^N1|_LJ!DJT6_1$`yU@HS^JO{tpRDnN^D_5oiMRbzzusoazc@&o0SF8cmxl7dFz%EO5F^C54~3%t!$@h zgW<{aca=SSXp&SZWGzX-3zRC%=tF;77T}l*^+ZmZX_g(KdchCFnO>YV;!JV?LvD~K zd3|zyR^bWF;>3PW7W5{!Ifs49z$i{*L%8#ndDqj35;!9m!O5YdXzU8l3ic<){yKXT zMrQ=0I0q8{Q=BDy7#sV2%+E3~igSI!*^T*W@T&hJ0!{9H+P8*cqI62X?g!v%2BPZ> zf4p7c+GK@Uv52uy%C-weYuu5}>;iSn!3aiingoY2M)szVj3vpOB}S`C!5=v4jz+5> zvIC7r%Y%f|B;`?wvg>KN^j=dsw+qO^P?P#QE98^qH=h3OWnE^%y^o#_5rPy(arzPt zDNgV`bw!m-`Dk*hu~$+CMsfQ3CHIkT<@gaNddVs2z0yznJ$)ed4D6TD`;+Ne&Uz3< zM~&T_nKiHPT9-XKg0>-;;`E5>WL8%@qLR5t_;r_cy_NH<@~TM9J9 z$nxB+#61Z93mgf_mX8CKEVy_bK+1!;3cnnc#hL>~QgUuKCDiBjT} zvmJ@OooFU=;kVbacQ|4w|S_W5{z)j89dc{{5!IoVZ+p8~1! z$>mSd*yplI9!1~OBu!WDJO9Bqkw3P>$S1Gd`%#n>eumjO7L0e)Bwy@#;6DF82=m6} ziucaPvdpnPWL)q`*QTUYbAi4vvOO5so+3(;x(t!XiV;!~Wnk_(o^JzTTZJ`M+!psD z6KFva2I*^=%$STn#{Sn~)5htA8VZ@F$p|%t@f0OI~X z_40~!{)lKYF~J{ZU&MRyGVmlq{=nYre`+wz5=l%fkq-*K5qxkM-zrEV5HN)&{C-kq z^~BU}N#1KoM?(as$sN_AQy?&klP(0f^@(7p2c+ur`YMj0!Rt~X{tYpa;5m0_RWS^e6KID`SMZRo2)35NpgmIh zeX&FJtm+BKAaa%IGko?-whXt(0Sr}%p3I_A21c>_D<{uoWtwpYn*te+_b+(-A@vcw z;$M+O<599^;T))_icM^C&u3|lU=-(1v@uC-hmA4g0Y3viL>rR`MsfQ06SU+T6qL%yQaS}(5OV7iLA%4u+IAfA^|2h`{`TJPKA2VRRi;U7$5*QyU(>` zaHW=OhVqLnUVmO7lm)CakO#kJP-C@*xa>E*oALt71NF`%uuj53SHIw&3}W5Nok)%U z3)>1?)5_MlP&>g@`}5PP+KF`gq?u5vk(Q4V6C;1p96{~D$Au#S;)!ggoRLsQFp6^= zsE!3l4;iHjk0@v1EOJacRbr4_+=1&>gcBeAR{MIH`Z|RsGE`J~m?v5P>vP_%>N*<} z^E!Jkrg#yI;{29yzCrN0pa3;H5&iPX{N46#zn_26`eJr>>{VhdMl>avU)+;kw0?!V zHCIyYgM7F%i287}5oblrZvz@8EKFBpL~!`!LkIFIsYUC`$q*Y{^UnogS26hM$;a|w zP&63bYsg1Dn3m;=G}66DlAd!u%0rF%pa~}^(IYMCF0c%Y4rdt+r!y}pR7Rwl5>abr z!<(mERo^7Lefi6vAMV9MEFVp7{p`t5M72CGeKct;HEA@qqm(@An2(KeG)wLRP=Sdo z;UF`tFJ|xU)sr-luX{UC7)`WOlUDWH3bgw-lMn(yMJDZcejC?e?I(A9Rz$ogD zv=qL@((w_ILQP6RaC4%`ox*cxIAz>=&=Ysd(~Asx`Ot2=(umrnmle|?*(C~1B$JOO zw=y53vVXF$y}>0nr<9Iv{$)0CQBv)U-ID zz|#K*-HSP*(6qe0`kzosS?PHAko;Vp15gXGWgk2}V1oxUu6Dg>vtD zB`T+L%B+4QbX)V@DvNT71csMY=(1D;)6MX)P6zOc|75~H4|0HEoloLGjUEc=vtB5D zNieO-!6?pi3FmyqX{=HM#@g^PSe4=|1EV-=2lbcnTDQwEAC&Vp?=|Z7G)yE6QF8Hgw`kOAvNlLQaxZnQx$N7i({iYw+UFg_( z`0x>(2ww3Y1_>;-$Ih8HRpW6vff8%&VFxZ|u&PuwSd}{ZW6%FOVq#Yu!X7DVLZt+X zy>NZI5j8LhwKM9bqC|U0RjYDT>2S`Z-9lHMR+|Do6+qQ^78xp_)fRtk=H^5&iu1gZ zI9b@?Ny}kg37ip(;-n&!EvCDQ=cF!F3DgmcqNZ0E-O4n)(4NUEl58_#{C!PFkwQfd zf(BK6**(CBX}uIEi>vxFP4$y!HxwW%sm-MQx2cBih~z_wx;5aT!Lvi~g+%qffaUls zYe6Fz#Yt4T&sp6Jrx>C1KzS)}o=Wl%(iy=h&f96E9qHyxotsX9y-9@xd^AyFO+~tf ze3Y0Q`IEvZ`Vl1MtH|Z022_%mFu9sEZZSEi)M&I?KXeyo8B*c`>rtOecX4GNQFdoB zXDs}EaeJsXLD=tVy5*1UqSTNMC_td;0CGh7@7M%^+9q85TryFH#p+nSZogol+D zQEW-%X>u>~tdxOKoU}1@uduvl;j!Q1JUG0=IyenOTm$%G>r~j9**Y~I@*C}ge&fFK zeH5<8%IBNpnMD^@P1$)HJTcpcUaz8hIW25s0>zMtG~tNcRaeZ8os-7+y#yvE@*phH zzQkAkWC{FONJWC+`i!UiYP|oI-#N;!Oh1`K9P`MBd!I*9j-wwi2nNwClMgZ{!WVDj z1iy83CVt+`VrG=#nTpur=#5nQ_zx_Ody~nvJ)-7u$;W@NM|2-C7IRzVxLVJx{}<8R z!i^*W?bS$EO~PKYa1bd2HEB$JGYPJfGx3J+zpy94P?Lt@hRYG`a9v5ChT@_Q+VpAl zaB?b95zfTL(tTQ9@+{;MmY&j#7M|i(peQ;amkdlTqZJF2=#yTcSF1oe;l8FphKC-k z3)p+qZU5^HQ}0+0-KI5R%`51__I540QMFKa6<9bYYzmdnn-K`nfll#^1<%QlmvmQkJ5+!7qF5SMtTki^^M-{O=F;)^Kmf-IoP(23XH|{P5hbE^@L>hX4`1|(qOBsN??HB zI<}(4s9O}!temkC2(?E@Wxtb_1M5%x_P^hee%}rcRpLz>4$*}xr*cVl8Ey$vW_FL` z>ey28aVo-`Tk-fx=#029?k)x~<R>KZ= zQdMIOPQa_H9968Yte=e9V-SfFd=q2h&SzP(G$eh+h0RlMQWxLL!L(*|q=I)-am(2i zsOlY%n+26zMtxjBS1!_2_Y3hhd_he{;4VB?7&Z6dh&E)mZ zhRs+I0+!|7Rb)Gozvz*U+a`-eg-&((a0W4QILXa--!e{H$nkYlz}0fbHUt#zl*n&R5j-d zVfiMh9JuCv?asx8o^#*2^MUuyCWqE#* z_KUHLYY6gRut8KTY4Hi8h1rS-8>8&n+h$s^ZI?iRL-Mw!s8r7~kKiO;7gwQd57cUJp@!zii0VI9dzlWG)JrOGT8 zbs2bV>*0?Hyr8U%`FBr{5a4Yb^40t;#xJfOKjGlfyPz88ABR0SY4B>>>LZb6BQw?w zg~ZYgfKkTWL}?|e{Lp2l7k07w|m|lf@y^!m`AT` zH!I813O9>cU>zyB(h*dJvd;@=A~Dm|WoJ)BTdl6!E=#&cWndKNM8f$3(`+mi6;6#lon5EtpJiYar3 zaEIM{P3bm1pX}4^=G*QXd^+ennpwqUzx_-%r;w%HBIH_)_yNLPO8@|1x zCcT%paX$IGv9@^XtEj)@CNe+Ez$i{1Vowh~Y+MIzc+cmj|CQvEamT%|+-egSl$xUm zMsd;$FRn2=?~gus@%D4);H8ptlY=|n_dVS}iTD}8D9(L>v)I#@ex^93!<(=EZg`)9l}^wJylFY%ha zI?56_Q{G=kl-=biD=dXu@0(=H748{#|Ni^ulS^*T&3)ss1eE2uTU*_AxfD;p-PhJL z#Drb0m8K5iVA_#;$K>A5==E53DeP&3ek*jme9Q|-!yVZBuv1R>b!!5z1Ct#U_jk84 z_+ru%YSJ|r;0JtHHMH9;xMrWq+?6yYYLXov+3V@^4#Vldugkr6{;+pO<)xUd(a7_P z8W?QL2cy<0Nq6|vDr*dGaf_t$J zSJ9ENlvil%A=o=^V@u}EG^7E3=9H2%+{rb`5*5)|?_;NmalCDg*$iC#33gMD< ztFEtn#0HZ861->N7Ld%Kf;ACCjfQRY1GDFa{}uxAw723XCU(ZSQ|p6ruWW|!a^(Mf z4ZuxdAuKaG73{Z&V-K*(*6nJ4Z22JTX9?t^KH02ar213_0+|#8=fIXFc zd^`|Ybu|}Esr-njd?HJwXleT7B9(QH?&U==inA?5z|~=ihOGQdqoO8RGUn_r{;0rC z>f6Ez53N0QI&enU-SqsQpIhH3@1nbG@_K{j@QX&-d#0>(c!cb{drm*}i6;(Kjk0OW zd7BMiPm2C3@w15d)GruJ#8abtpHxYIho42Gg#4MMR2#>N{8fef7HZcFgtsagDKCgJUwmxwn1@Q;@kb8S3`^q8tP!635Qe|RHpXkxH&hJ;` zlaKEBa8%_83OGgQQ|x%-71Q38^v_F82Ga;dH}DZ&Jq535W!|H zlmzDW!mg&Xn>a==igPgGyn*%8uHv$XgIk~W^kjOBXQ@UkkLzROnC>9mXS^tyrl_eA z@6Me^)Q6)LE1zn?+rB!LpO6s&W!9@ciy|LQa==&yMn^jeIB{bT*;k0Z*W%5XNrhTG zYk3&bJ#x;!)k2NFGNvC*j*j3#xa|(u(SiKRM@v^z!zfj>$BUP;^O-{?ZwwNwnL3)W z#G%`Xu<#ZjM|`0Lj}E0$b z!W0o*hUB9uKKB>yq|Vt(@X1PZG8E+WoY0sZ@jxZ&43g4z6fg-_Gv?-dma|a8R!1<3 z^CrT%JmVZdIT6&PD;hEed1<5DsSN;x+WxQHOJ2M>H7kAZEz(fhDD1~=;Z_* zcOO_tZJHw(9qc&5`8+$11}~_RiTP8m-tocyn34!*1fw_)2hL*W@r{L=iF6PvH-SO* z&k#}Hq-7ra=k-Gci4KJMR5(&|Fs=@NEu2NWf!lc5GNM@{LUB*`awbNRxJ$x?t+_Es zdIn=yV%3Yz4_g$**T`S#RH$v1YS2v7BLYNRKIx91cK3k>PDS(Xp1#a!KAHx^X84!invotcMn&RB60*!XG{PoSdcZ|y?PtSPp(q2QahUhM0NwHNZ zf>E4n(P_s;lrf?z>H*#r)ek7p>glqNN51@0&$gS+b|phh1d%ZYvaP8n&wvp;F?vr0CWf zUg(fvgQsMvqB{aye!oFtK>3zIh>NdfuLV~5``K$@Shuq^vdx|RNny0NJ&09zur)Gx zdDJzf4)1ZhR+xK;UKtq0N%1+CBlwWcA$xDO+MIWAW$EmHtaHId#hrjNjYb&G2u5)p zNjMiVopytF!5WJm{OJ@#v)5em@N$oyuqSY)E?gNH#d#*-^yNgt@*}*t#bp@Ywo&By zF!eGiz$^aG3ID~cE+3KIrbZ)A zx<^#oUIs>S{!KV9VVt(9)Zmyr6QeY@BctZw!#h+Fyy73y3GnGDjM;Qv$PrBte@;)1 z1_8df8ijYMkOf|rtf`*hwP0%XY`RcW28Z@ZINa(ig9a}{(L;UZQyCb=xd-9A5pF)_ z_5|WRH5&Fz#=LmaHg}vs+lUw0FQgaq95rc>X$Y?3t+>r=cU9qSzDtSSt^9F0pPB39 z-~9ZiRsLy(`^3eL#0_r&B=61$R-45f{l<2N`v(O3ip|s{TEIX)l;AQj%7lXmr@y+K za|SPU!ji{EKYs3xS5W9Vy((S?MsdDGI4@%X^jCAKN%(Ri_GeP&O4k5WZVqfAB}F2G z!uum=eyB;dzGU8-_dE-`^HXEbzh3RIy@er&v#xhmMkI!!m556u7DX^RlvN4m-7F*K zSE=X53qF->PuWujMsfBhoF}jsYVexFFQpDj;EZ4tCq0VmXfp&^wq;sM-==3_R2ppo z3u+k{#o3;`>of|ufcQ_19$4vz+L03p{sMLt%D^jr+g5~IH+?Xht7-+yx^1P^!;^`V zd>h-HsQ)NyrFLtZhOk@7{!8~zIz+`ccIEnd~DOJbNQct*D(&)*K>d5bD`asB;tP>M8cn-Y=0I z8(xs8uANw2I{}X+!m^dYsWgls8eSq}8E&t5j&LJKh}UrqC844wT{8k0#km{dybv(J zX*96A^=pOJ!_!@385qU6C*iDN`(*HPXq3kLarf7CWGAT%jN;sv__;>qghuD+t&Ngg zj9mysJB=%AaBn5VTwt`f!)XNRZosG$umIS&HBl z{~tv28mvindaWxH8v)!q6q8>DMsYsg1vo3%JeQbXUh;p0#-GowMj05z`8?sI_f?=7 zCE#T%SWBkCp0D4Uk#>)&^?iulSp3}4RRQYdg&IT zp{-ZUS+2vyHI6L~KS6RbS%+=Fd?GV}`Q#q_|G z{Y1ZO>KKUq6yrONQ0NoAzH(`w&7VPkx87sf(r5P^ z;RG_)&RAHngEIY*lZnLJ}Rfzb&&_Av5EHB+&LNgoF! z5=5`n{y^2n%XPQ)#fOkjfRWw{jBlnm zd&4@8YpBwj;l(RmZpnFwa<)5hHrpP>=V+J5exe7qzIJHadE+wC9-m&d|IRD0?;!i5 zKP!EkQuK;Hp6KVn_+a93Mx)<8Yh8WB6Mw(}pL0st%GQ)dCD!*6G>YkXDD!yueC`f# zKqqISG4u9${MSuSa7uC`9UFC_9Xq)qX-|)k_uREO__fwX@inF$k60IV`o#hES@cXa z_pQsC{=5af9I!Q8ff6)|=@O{=Qt7z=u&ZjGK?_C={bJ=~yEdmuC7@Bvt%%R_*$P}m zCZ9fW2B~c_Q9inJ^1fB%kOtTCL@DA}(B3K6^4Lj&!!1*YR!diq&5vBI_XJ#z&-VKA zSX|EFt4D{rrj(#jOuC?KRxzfV1Leu$2;sgP4E&Ium5ZMpc9w;Q3t~>#_}~^({Os`M zBoX>(@LTKrm^wQSC-DyhbV$lXZ)T94^n-)= zFDr(}liW6xPs3gt-U&ktDQWGs;o{rE&Wy0EVeL4jKGF>Q2fqToBhk|;zI=pSms`>h z4!J}`U2sQ?4q(j&tk4dUTjhmF=S_cB=n^#AV>HCfHWt$xRP*23bNlCRff;#!=jBJ< zH{>Mr*mk@=cBjqiwv@%ujAjHoKdw{RuEO*($+F?0f5a%b=_BjQ&_Ch^LLf5fgb^j` zEyz_U(}_;F{Im!1mpGUbx8Q?FJ)KSD&Xu?jZn>ta%o5riAv4JnTJ>*t?)vg62dy5V zFJAmenoJ+{$m0OI?in+irR&0X^nitBh}ZRk!%_8Bl-)XjqtWhgRJuk3O`*;VR_(CQLb5#9Pr1}4`0%q{`Q6N}8-!wYoF2#d zBAkg+4V1mKFtw3xZ19_&n`XjkL3z;u(@DMFz*&zDJhS(Y z@RoZrNy_+^A!;^e66XP`YsIZcj9GUkaj%qe+F!&6zxw%2H?4gaDvGo9IusQh>pXJ2 z1mR$z?zWsZLg37GEqa1d5eAG6DA|GR|9U}&2G_eHf_QdXJzO$0xY4M+%eJT+O)eJv zK+Ix=o`IiJ-khsK2ulNov=fEK^~uXeYRW0d#&jd;mU*{wSO*ZT#7F$JRqzO#3D?yD zjbge}b$_dq<%dH93E{|_*LmbQ-3VwD)17+zuDqHAw~~r6UE2d1#dLX~%WRH)PPESt zj$nbl?$b}>l^%*fh^H~fr}mp@s$8FTW>Nm49*EqHb|~6fo#~EXAv+YcR&s_ZH-_Dm zstTf}=q#QWoSNxr8KWR|O?;`?@uDpfamj^uw_(CcgivC6vN|*(YJcVEGfWAiLrw$~ zUg*~_a5Y7N^6`CFBxh=M{koehd17ZmWY%q||KM1OJqj%YUqu^`iSF3q$g5x2C+9ln3hJB&QgUCA zta7k6*FO61J#4yXd)M6dW|RdtXOgb?m;sIGmYG7E2p!^U&^fxD<1|ujsTAmldskrR?7PsxOYS@> zut<`E*@YSDf&bXu06h!Rf<(5KAOWwdGQT?|dQ)jsm3+|>Q)cgJ6W=wLKqb5}fkd@s+dDwcnGX|Ge(W_YHmDy<(GjhnWA-uFP z%Sk-+(N&ZdsGqK;KU06>)f|2Sky3Qn7qH9BvlxT+qe+#tK)q#u`y~JnhK}FHk73{u zMTI~}E~y@UBE7jS1+zBkzE85&=?~25-V;_w3d-ReYNzFpo;6brZ8(Je#GCMlq^lS_ zd53N14}qi^zZ8w)Q!C6)to@E<$*`ur`a}1ycQjILZ97FJ$DS8LTCyk=N1?`O{|=I%BP)o-XdcH<`rf4E@$oa`qk1$aM9TmI1D zD!g(STJnHKFmr97f`!&c)>i^^E5fHw+!x-{Mk$!=ba^lFgu15-8Y#r8o$Eoh_J}j3 z)K=!|pxh0HE*>;-9G^4nIweX$S#M7u?M$|z22&!~CsSfSx@V+?e8`(9txMaQVbye1 zi%r&PN2^|!s&3O%(Kx-R-T*VDURkYbuz>k5>g`PR%Hu@wbBXS$Ip!BGs}xesE%{IW zhzzFvki)>J1P%Rf8qtj!v2Hx6500uOQY4^;pAxjr$&a*T4J?rkg|I-GTBYR}d z$$z&$FaskT&=BVJ#NrL~iPukdYP8w-*l6#ci6(UVdAHjRfP;y_zZ>}-SCM>R9!U;v zEx?M2^)U*xq=ZLiL$OMr>k46l$S-DXXK4x*E-#DSx8%1`p5P_B zLy?(Ww?z!S*@a>1&Sk`ax{bY8nZ&7@I(@2*Z!n1y9SapVu30V}NRPf)1H}-x#Jrho zor;m6p*!y~iR)4xw73_+4EClBo6!}ttuJT(Kax}pLMVTc8OdA8ZpH=%GWKJU2(`@=o5Dp;!sXJ_rSZZy@u{p z#@Y8u&?sgv!fZtg*D~+on4H(frjYLJTn9oZf|~*Y8pYh1Fn=X_A5$`7L>J9FcHBn& ztAM!|Pi-l~MaFPAO|ZgX?^$%U#5AG9WpA#?O$|Jv;7m!_lG%qzT=%P$RxJ*u1KmfG zgySWq15xyFItWpM=|H!=*ednOm{Cb~MI0K|7ES?5qWj>%6d>cvorc8E%)6nh6C4k; zbuwn(WH)Lbi44ttNfI@TO472a*AX3qdd8*G7w&OUpw9 z5RY6{FoCI;wmQ)Trk=UPNf7gb{n-7J%AoU`*uK!bD~Yx!Fz+lnyimZMg&BKva*(L# zbW8^(U4vsd`_eYfq*EdV^E{>#n6(%1ANz^AJpN&)9lpamU3##JS;bBP&J4CGMXzS_ zIN%pLPM=v}qVoDWdeu0{vx~Xw<}({E7@x&HK>s-U{8QRYe>^!eaZ5dVTMap8_rchk z``O~>$q}YcwC+u_w*2m?v6)ErI`AwY&rHrc;N)BDU?K7N&c}7_2>-A*CZHlGp5=&*k z)H@J!gHBom98Tr%C?mSqCb>%xz5Dhv8_dfdNS=c2Q|cpg0IQXA12jJ5!1XUZZS+q& zph>GH|L)!C=}6GfIH%cEBJMT3LEVvHT;Hd3qA0(BL?Er3A zsX?H8*r?X24z~bCTg}Rm%$aiubv+{vnHfqA^z~IyRHeso|2tP`*ztQEySg(r={S4v zc)Webyf_t+L&4aiYD*e5fkxkRB!LH+FLwYT4^TuSZ}{ZxgT^5cXd(0Ed?n@>ktbaE zkQT#)n4?of^KOy>?N@Q&g`9HWnWaqb#<~s|VO+>hCGvNe+^q_99*{I8)Wf!Yl!yaQ zJ|m(Bb}fFO+KENY)cSW^QhUQK6b~DSE}&8Ev0J9`-h?w{d%2-H!$yI z;gz5f%v@Wvuh6-Qt|ViKRFxCk*(Ru`3mH+3kOH1NWwYfPrt`QIWK=j3R$O?pQXn3N z(av%knq7873Dw%Y3i@bFShVl)mmPBVYg5p!I+~6A)fsby@H`GO!Pdh0tXdb6n$brO zY@-{`oxI-*D<)(_I&KNV67MP}-+WWe_9mskWe zi|Pp(HuDPUl|SuiB4xhDGZAebm$beCjbgg7c}%u{ zM=?K{w+B_a3cZ?ZhNSVxA-yP#EKm5zX!8SqC4MP-t?q#(hu%%aco zJO=#~KR4s*4QLc|7-2p{@=FcZ6ItiY+dPI+46D#6=6-~EAHyDd8u93d=ipZiF&tOC;a{86AVWDj`W6wdWrEQ@=zC)DI`qDE*A=WE}Y8nAd)j zLOu+p<7yWOXUP8@2h+K=J(Kw7M?utO$b1~6$`9VdzAlY9odJZz533F9nHHaw1?U;k-i%p$OzJjwBF4AZyhE`xfb| zJX<=8ppc?9c{4p}H3AyNv?o3PVXvb#X&HUuTf0a+TaD$%%{_6-1x=g+;pZ-)VYCy^ z@Qw$yN;3XLIL`6m4*FUgAM9}i!8Ni$;$9nkjrGI%F`5j6lj{u0m}l7PI(1eplSWHh zbEkMFz(MMp+n3o@u(!yjL{bQsnTkvlv~J2ZO)GjFq6CCiNf736atr+INOD^0nq<{MbVY8!V4K z{7CkPKDzYMf@$Kdgj0!xKKlH)7Ggu5bICOEc7l@gsKF&LDTHrB1RbO5K(EIeeq!%4 z8ya;4)GZov`ubn|I0ykVQqbj!GqwSynapmH8%#{%%?)VCuHZY@iE51l|4M0C5ZW9d zhM~<&;_GRkGGc^v*&(k3mS8>1dd(I22K#hY-%=`bHFtKYM$cmsC+Sc(uV9jaBu4tg z`?{+HXBVzZCo5hCX^{MGNNgr26VfsT9!bi>TKhCU*bI z4g&YcNSXJ@h-7Yww`g{R zosHM*4pU~$SnfDN+=_+j+NZrITGvvFb&L~{g*<5KE>%l9IQPh6XI?J4ic|z*s6=Ix zQi94d37)wK5;rl5$BRm7F?HX+X!1rf6|~ZRBArut*+J)ifs}NDmx9S~7blrVwNk&_ zd~hTxNC_mDsbd55v`if-ZRIObeQDimosjh~=bOICZ3dBuMx=(%m#7mW+X0eC&+#Z7 z=z21K*xOA;B7=x;?%@T+_`N~Ly0%mb<`5n=rOZY!nzcoID!nBMfYt*!TK5z>(1kg? z0&ZW=cDL+UNwEZyD;M%Obg4G(%fc*0vIYb8QXa6*Bi05;0S&M_7r74T!Q0q%b5aXr z5Asi_y|h0UlIw7hf}yAXV0YZ6mMrVyH`H-s;R+^k{h*by``OX1RIQZh{)42Ob*}~} zq!jYu%L4KzjYQNT_yb{H$Nsj%oOUn;q++&SFMH#e^H*TUOM2Q3**2yGz2cK*m%=*<0*1bX;!fzbR;se-PqVGi4v0hHGPi4%`BD@&b8lfu_EC`+c%lo z_Ybl?W!^n+y}-#w!h_MifO$3adKE7o*iPx?FK#0>Vr{`Uo?vkY9P^&X*Be zDX zv9Th`4xE5gq;w!K5nHSkW+YyKtUzG1!EY;%L@MdV;{DK|JM^ltn>zxUVlH=7F6od@ zxZE?_IMw7Dn(EEO>`~3gjG8Ut!1XZOGVgRDZw|unE!yo1Xo@+Qq8yBbv`}iPg63H6rkv<_FTimQ|_}t(r1IJR66Ls+?C|$ zZiaqGqf1m++~_}RqxVMi@)+*@Nw}r6$Ls^}I@kZ&@Po-{(MJQIwUD3R+El|_=QgJy z8}nU)zL;;P)WrARxol0<{|(1?()>)HXx7Djw!G?5WK)!aa@>Pj6?0sIzL?{KiX^>P zU32aKotT{?A*M-flrnv;$8-t$Vy275Jhm=dU;Te0W;gW&wpls;-s89geKE(QRLpzK zZpu!?ds}Oy!y{wWFCMnUfK4(tX7omt-|sUDnX2M~?pwJ;bn+*qX^F9s^$oN4#;bR~ zpE36&%_kEMo&3XLBapQzF66@`CLC#td>&FT3wh>tnY(TpcEW;*HzTY;3g!_^9WW$L zQ*;F(g_fE(s-n@3$KT()6KOuBPaI{jvxHmh4EN?L=6x~EZ}f@N(jsdf=ch&5*qC>h z!I0zkv8yYEV&3;vof$)S)Bd-i<_!ETo+P`b6*jt;=-_7a;^k3i)sYLN6&u!7NJm-r)eV zvxYp#uhN3Y56zOWcXwqfemLC?#$Ptx_r!)WBxROD_z0>VuS=m=TKjv_YTCCBx`PDi zzh7nCB7ob4O&BStqgR5an793uw<*(dQ>WQE<@dAZ)UZtMJV^oZB1u)1Kk1fNF>8Z{ z(feOb+vn=~#4dCBNP50;B*F}`Rb&?SK_rL`lp(K>d|=vgwx3ucSjnKk)S+4Mz(bK!%ngM649ETCzL(ciJu`3**$%*C9?E8+JM z`4i7LC1}9ts!6}o&&O@@tij`IRS$ z4E$a)SB@mmSZc?ECP5y$Y$z#cVf_RO+EL=Sq&W7s^C7;%hVonT%ZsW{N)+3RvZs_HQI)#r zNGMSy5m#L~p=dM530asTPTZPKz;!EJIdMBy(xtIELSP1rVewQHlEmF#0!5i^g2gJd zi(f8AH#zPCl}X_)G+P08OycjV2iyfH#GSRLZn0CA=EBOXshl~5XQ5ytL5y$MgDQTm zf4L3SNSN_3X)v~Mc%!BrW`=lg!cCgGha;PHQmwO(tS`9eCoY7x)-9C$M_{-m2pML6 z>8Xpj+GJ}z8tq>iQsx+%DEuLaTxvZ&E#N*gZy9gvn&M*W^!AR7*Akj1(w5dXQ?2w- zXEM$NznR^fwnBFM9KP1cswBGglIz_#nG&2y+^}$#M1c>) zcPH#?SZf^rp4`Gw(_Yb#E6>$7)MRVns@WF{$vo}}4JovtjRJIoizCNxwNdhF{LHaT?krufBo|vr7RnsQP9k;fd+DE!H^<8=x?z)U&UWW8S-n7v`FfLVe@G3^Q4 z`b6*ZDObq!lC@&Gwgfba`2#Gk*(`}EfqD_=!Q)UP5?hWU_V|#C1V-ucT`j?{w3fe; zMQ(p&eXg##5w|3K_Em(M|7(8SL-Sx%3+qz6I{k=d<)YexA8Kv%LvtG22ge+A>C-|5FjbgTe9Lzre3}M*$jAks&eIF+=FL#5`379ecwF^b` zpjtOA$`Do(=NQ?reO%4gujrrr!{%*~=%>v~#|1U#t)JTwRsjVV9kD+@ZK;aUj)` zBwf52b?4}$^%u9%yFx_^QP@9JCxE{9C1-(>Y<>+_UG0 z10RJjq(FP^mZdwOONB0Rz-J7MSCfkvlOO4?iy7NC5K|R8W$Lt~oGKD>3e14Y$;NxO zWt1XLf}a#~vb~C^8jGm9`u{>g0dn44Ndq>tg`V9iu6?l94yA0!(#l;9($=Vx)O<@J zg$4OjyaIf~EdU89)Kwt{XnLB%;)T(N-N&-?NLu0?vRhNmomCCwk~bIAgj-U9 zgh?lWKo47D65{>69Sk>(tKrAAw@%Q&vO(NS!i2bQJcfXa$oeiw2>K7esMQ7;_l>beL`2>7%r2?G$N;uOIFIBc|W^Q zwQXok!ihKLT+cGuk8)AxM=Z)xV29mMxK08eASyp9-<~sA2WOH@1kbj7Q8frp7qnI`}cwQ1yhX7+PPs0po6w{59 zuv=hLWp6BnPaR7_qnJayf#Nh=je``#Suv#H=89xj2${afsQ4L+k5xebvK(bxu%F00 zdh&}m*UZgC|9YX*m^VIq+ztlx0{fb#)~af5Dbdp>`t+o1&8O9%A93e_ckc51T^V$> zBd<$pwiuXb%Cg5knpJkU+J#)T9ia5eE4_&ao<5*D=o8ic`pkY8OlZhxfz;1cawzlU z+A!|zLy^z~^inWaCB>p*mx7ArY!XX60-V_Sh^4bTt1V`sI@!*EUh&DNG;@>cC#EF2 z<=^Eq#`c~8hy1c6&jF2MzDt<5pp%NMg7RQa8&mc1+7U1Vn;Os%=Fw=4v3Fo%JMH8k z;?j`MyxD^4^@npnqnLZq@ZT@FmBE^iEHqDDGH%h*GTi?c9!^Ck7uq+_AwM$M7F81b zNGqWqnX8immiaag385qd0pkG;Vctr*KZ!npFvR>6ISiKGO>(|q0(!--B>X_>)P~7I zm&Urqw`CZVNj2 zkyc`;cTY!6MOr9RQk)$K`~1>~OQ@1I)!A%SQO^cH&G}Crg=1uTNS+?W=?8U+MVr%Kb9t+9U~&HF$#JS!(re?D%5RiHQ>&QQqc2 zOaYBzE+B2Zjk@2*WHzy0let~HGigvg8pW(5J64ZR5hk}wRcZ;QF@xshdCf?n`*uM^MTVZ0RaEGN^Ny(C2r0S#fk1f-mO zYH)*Ta164hygJ)7bs9EzF(aUglr@tifydiANaX*GQM5XN7B&`)B``aUS}JKpnfsNa^OctZ}$96&u5~s2akEZ z>tXv?PYzFRwK84M2$N@f=@ebYfDXvKxsw#ppCAJoYOTe*i-7MYdVhe)*wzagd4o40 zi!lQl!OYPe3z^j2F&{h3>t{mD5;ThW6k#6CZFlrK@rwmfK1(8=gr?ha*I}fxb)?sD0`UYi zig^}c#^Sk=4kqan?fv;~7rlH%WhQ#^*bVpp%h5}Kc?yeXUurvjqAAb6)3bImXs+M; z)9CZ&B^h%XlQ?zN!}Etueed40FW=RP=Vnxk={-&|H`Pov zm?tDLDNA~t^P+UNvqFkDvIpo@zY$JE&ncE;rQ;;xvL3T1GpfgKFy#P-dg!sE<0-)K^s5xHF6m z*RyN@@#=$Qiqck51~pMqXV}@QAdu&fb?#X5e2(r5KbLNKJQ~I9OQ&KRLFv$=4pUzF zia25LWMBr?ETB=$W?&|=R5|Ft2$Hiyb4;!gKP$ASa|z0CwT70Heq?SVSHS0AomiRC z^5#iuqd(aNG|K<^gh|^}z;wog=VWXW-CsyF8l557FL)5P=7n~3>L=~$sr z%&vqPk4ERk$~sSPOR6F`9{m1d9=rptmRP`|Lp*q ze?FAWfJh2L?wT30t&4$25{F_F^jywb0vkq4hYNf<=qw7X23eGW)du5ZX1$p{u)d*j z;9j^DX(nas8taQMLE8Ni`lu1n0!jH~8gBNvNt>vo4ezg{(~}4>6yme)agJSk)xEr_ z8PQ??8?$cC*F4cuKn6Sf6~op=)N3H06k5t!g_svs1AaHoI_>UOpC3Oc(?Q&$l$RoQ z+FkshZVhk_4!m;=8(VBpMDJDm#A$b%2YR)Zm2OKrYu^R&=1!+EQow-5xWil+UvCx_ z)3plq-rvcWu)TLRN)S#7dg{H~f!{*(gNzepIVJibpb^a6X!0tyWSiz*HsXH44aJ?S z-?-qBj}aLK9){4=bradE4Y)&pq&Ct=PjTWMa=gzSUj)05Naz#s+tS;+x1Yhns?A!Q zl@YOMCs|Zi=vbGXSzcRT)toqVN_5uK@Yg3MXU%{{TV6>%%N;yJI19)#3Sz>zfCX!q zemtO2%p=Gr=!AF?Tz5uYQtB3F__Bim?T+8%Yi=+1gLs^cyz?EOR;1A2M*$;Ag2w>1$ zwInS7lSZEQ&(P5S69z5 zu0=KQx&qHjNFk*L`IFL64R%;>Ok*!zSM(y>G$%`24dGL<)Q&`7dlnfo7xI(xnGm~ zK!2$L(^9I=qFE^>l= zB#D_2A%JBF&=Jz1?T-b+-=D;J!~iNb}|iWrqdt94!Gcd@Q*Z;9GJ zIUrAIS?C3>B7p}qib;KBti*jxHGM4cJ)PQYfiY89mrfZ**?xuP_Ioz?p;q)cRkWC7 z>{p}@A4pb~6y^gO#r&Qy8@V4`zlr#x3s1TA*5>&mAdsN@0~*DonDCZdHdrebNH-@C z=i-Gfn@6wsqiHTEtIAe3)61bL3n%F-+$qJ7gLIuM!Wya00xnuz(;-2k-i@e{R!m7F z3vDQBwWtSl9(`)|ubXW%nx4U=8%?O!@uuMptYUj)Ot&O!GLobFYMP!Od(?mohV&HZ zig}!@|D+_FN-inB%`u=WYAj0{S@`*SOh4r6OQNtm-W#sIjy9H3PMWvyL0j*q8Ja$t z@-$9VMH!mzV#l=vjbhp(uE&!atlLlWl#7}_O1EEVQ(jX~@76l?41S;$kegs0NKOlK z1uIT6xjUGCGHJ~M8pR|g%u`8B))G9xzIJxvKbH^04JNU`q$TF@Bz{1z_|9Vc!W-9*{v?{rS__> zMj&%;Ms{>|Mz#j((Sy{3K6;?21(nJH*6QUN_o|~XnyS6AiSKXj$$pgnP^+aMntyRq zyaWVI!-wsEUKjUJyw1vG9n53gEI0mS-ir_8ao7LpWXpB3;u#+q7T*LExOAF*SdY*4 z`tsPajK(q&m$V?T515#@iQd2Frejp9&zpOAkh&wSfJQNGo@4u(MHxQ@a}ItR$EU-; z;H7d`bm9sg)hU=FR&y9n&7;Va&__3EbdE$zE+@nN9nA>eFrpiB%q5m^Vq47=mfz*n zdiunRJC8&gV%Na-(GUN%c0tnQbJT@f2 z3EneOwa_j+kX!3|S^lEiIpOM2;v98n-|WVU;8u!~a9`8h3G+F6E!U8vU8K`2Krb*L zZ$=P}pT;VnQOqo1+JuV!titr(##G+4=hnKh7tko?Fv7f#I@;fwRsV!7+Gf_@m;g?5 z?xh6ZGYIGv-==$L;S~h)~nr`4d@o<>|O6F1@l`n-9J`>q6y z@;riu@`EfB*VpgXG)|rK=!8sExbc8?w_ia=kds)>C1@0LFSt?W;Uwm6U!1b`-6xI5 zYlXuGymHycwZIH?ETB=$iwSc$wJYAeI0N|cojkk>{MUOv-*wVN1I%Fe;>4S8II#S% zp^iqXr8PeXdyhQh;$c@@JSYgRaB2yC zqI-^+yJ}hYz5VmCc=xFUjbgq^n74CNY!C!}+=is=Op;Bf3DWMfnR{7O4qq^9aVDgml(_Y0}t_H#=sk zwOF>=UC0gog~yvSHEa!FI?6Aeapb_AXzBSOTdWc^Bub0?sQRGX3Y+7>^;&FU|KnWnODf9`+LQDY^;8lp;bB5++hYZirEBBFh_8MoC=HY ztpPyPpf`!szH|61%pmZSqw#$S70G1GHM~jTNXlgMpF@QY)sdDvns7fNmi?72@7rvV zi8>ZOT)yu%mDb&}GIcEb3)+BQn|LEwF;|hs`x?gG50;0Hc;%Q@6Q)DMt|S^?YXcg^ ze3dYtN@A);<|a1Jt@-DXz+BC8a3eFIQOt>5fLTWLzM)>P? z&=BVKMYlF{>%T@m%S}Hx^Q>zO(u$k-O%!`)f+b7W_sA!wjkjI8h|KjI{JBt8KB5ViHOh>AeS|)7C?z_T zH=b!X>k<4rgkFM%8e>t90coM5MLn>hrkMhNT?#8b7ng!Um!MH-3OO);XW1{NGJWD_ zc<_|xF-@O86U{yV>|V_MX)krsCk}Q(lnXzdj3C-a@YA9e@iT8bWTO9E`r7rkE!^A5 z=?;=p=kk3UMNxsnI+^%+@F8|= zSrOX=Iq^dtL(n`!pZLbcHW`;KKW?Mjj9}w~`>bL*lm!P_X4Qsuhadh$&P(<(anw6M z?q#YG5i)z%R5TcG!57yL*o*$EbmA0Y87;V!}Zg;L0yf3rvIFy&85qnNh+Z?L&_ z^z!CJG^^dLmmbaT<@%!pjbc_3pPz7l^rt?iPrL_*rK(IMs=Fm<6!SB{na>&1*#!1f zu+c1wEX6EAqnLju%w>$}rV+J=Ww9|=%n~$;X|Ir6%45uppNm`X_R8HAFe1Bs^SAB$ z+=($35BD71nU+UolAAD)74v}iaAIAwiT>yly}$GFBkvnB1X)>U?$>VFucO^~v-{y+ za@d%}VvVgCzzb65{=cJDJBUqg`QE0XNj%t8vi z%+xB`5;Q8=GH4Y7!^c$2u0$Tmn)<28O+N+ClEIL?Ky}h5(n&%J`XXxy8mYp2;m4nx zSjiCQCjMp$+X0PY+IKXpvGENaFRQnbL5r9K5wA0N0gYlhrSbQLb<*WU-FgnwDONzE zm`)4)wky3vZ7H9w%K{q3T&kMo8!XmDFBi39I^z=1DCTysoaRyVwM>lkqiFQlgWg{& z+lWg&yH0*6+Uj<+{a{i*-_en$%A>opj~u+;#Zxop@Fa;m^B&E4;-RV!79E%|cO=~< zdCoM>fHEm#zM}P$UlH%!VWVJ!tB)SBPKSds=B^})%yaSNG!ah{CrMbWSTSvWYr6~H z{{STSF^QAB1P?^xt4|s|*i>iC{Y>J9{a3pkYToLPr=joPcHCKu7N49k4<=F6IK6qw z$Ge_$CrEzc7P*Sv`tFz9HM=4ttl`;xCcLz`A!8`^sHoBUem9%qLtwI!Bs>dYL*nR{ zSG=*->ytBPQIbT|EDl7+d%cU3?D7u(A2ZRu>yNzYf+0;A^Ei`SK$=0HsPEH_$85G9 zb|m+BX6!$X+4bCvc_LX6zpjj_nU12ntjcA~lSvW|AMBHfmfrc^UgaZ>%$TQ>yztXR zFv7bWd*dG}tLro7*(8c=Fhwi=aKw2la%l8(Ns{lHd;aI;PA~?i{q*hHUtH5@P5Jwn zLrg8E9PE?x4VIWrJya6=;>MWv7{B4ZqpkL7 zn!BH&&$IYoj5*t(6Qj66I6P!Wu>=i`TZ?%uIFum1rDU(^qj9p)Uk=OloWJryJ2CEx zzvQ+-OIvzl4=3%MMI3|TXm0tB1WBLRN@F8v!@vK#<7F40o-uQoq(v^*&w$BxBoQnXZD8doM!<(|(-0d9%rao1UI{;7#yP;zG&=Dy>WksLNhXpj$QOX7*y;O__z) z?s&*PlL0A(V&1k&=gqs~Re2=w*o#_2AI+kw1#-(Q19U~q+{6r&pf6^?WqopW@&=+6 z(w=(PtY{0-weRKEzZ?2&_Q^~ng_K)3@+YM!X2DB)Rny!!?YC=p3dHkG(Q#RPSE&RI z)qWq;Be#t7{4z)l0;XZ;M;jwuGkbhkgPHYM<4VvF=B5N;6H~4iUj;<&pZFS}V~5oS zo|zQ#<|GVbjH41X3copRn|Guf9ObnYQI2kIC4Yu6%kQQH99=DjPyXMh>d zDCT6s{3gk#`t@>{o`M`_0)?p(WgJ0|HQOSUjb%t!z`dt%v!QBr!wZgq;B+yy1umWNe8q)40k0r zJ1#f=O5Cm!Z6tLu@3Bd6dct!d?vHK!(}VBN-)Gm1h~HTa9#{zKk;5I)Bzs;+pD6#s zq1SJ{=+Bwx*9*5D|HH*gXewT{ON$}*luCbl+G@pxl*U20ztS*!5X|tP=B8XtBZBdB zkD_gb4mRg*c1A^UG>uLa$87%LgTKxlnWixT4N2{fB&0m`ZX9Cm_WN}D6GNVI11z9X zOiB@KUSM~_VKNj(*V&zW*KTqf-L!j>v?`pLJsQP)ikJ=f)ZXfmAN0_)xs)M#Xc9A^ zQOvCfb28EUd#udo0x?1haPV{sYe}*#9*tsdOKopRVzSEMFp1X*?ZIb&8PF)Ey^ z08`5%t|4i*?Ni4?s%#}pTF74@Psv_d3(4WHusrzYm4$coq1cq3Qag;r2utZ+k4BsL zGpXTiNyDX4CzHExe0p{W8H8ADZomdKf|=_@Q`k<)uHd17$f!=Icg9WOgB_;p-hhTM zX<9T>$jAh0$U^}M2c%E(W&<>+sOf{IGS$ban7PHTAOEkN3Nc)6y~=!2SetZ_<2(%2j6jE zgzOaQm3cEbx!Pt1mWbjZ`1TJ>;{4taCH6wxX&AHPT|80IG(!^DzDAya+8hqNv$gfG zsB|yJEw-c^z5b-cTIkObG+Mz~B=1<4-Q%rt@722i`yG!D*Df{zI6#zwDJLuHpR~(b zO*5iF-YG& z3ZfPE2?Bd^hb?j%OZ15|OT(?a>5!MkuUFkQBVMKJPacv#%MR+#!-2RJD$K0|Q^3;p zt(O{8DVTvt8!mr|h5fX>T55e?n`19Nf3jJd=o39~$yFtqh zeWH){+WNiMN8xELzTUkH9#rKq>qMg+$OC-EaN#V4gjxK_adroFS|^AMowk$)6@4@} zl@^eSC0(_=)j-&zz)L(yGTVoze{`V%Jv+a+7mcr5c?5$0tzGG0vImY#y3%i#aw!Q%wQ zm`D}$UZ>oo_p0#LbWcXbijNF{!I=aR6CN&^od6^_P}J%97!(N=#`)$ zO#C=rgZBy6x#d&oCUfJ!su~K9DUOSg?E~FMz;#S7a&`HwoYcDIp>9Bu4ol%==RQS2c(7W5>1^R?Ie z`+m<+_XRBQ82|#SB~6m~oVCxrC9PV(w0&K*OR43tv*)U%jPjdR;J}!poHkFZR!>@Z z+G(8$tL0xK(46m3^x&;Gs&O!VwnS$`YhIu-eIl+FvAwU{c~qzfH9`yIiOLVpCe4mgTLY%{pQ2BzjQ5Z>h0e?d&JqJaR13H<1TgW3TOn=YnP8H zy%fDv)BChVztCCdW6Y<+3}_V7#vUBY7Ir-TW9So|zUSYwZ)Xq%*>w9Wa~^*23Wr&O zMlr9#SSqX^#qZ3OH`Px^%o}9u_z9-z6rKYb!nY&JTAJ9|bCxVH0<=D`p4@736)b%l zF9}&dulUyU`;hv@H@G}kx0Xp8iHA+>B)Q=j&?u&}8fE;B13l&thZrI*f#Wym7_BW^ zQx|K~go6==}+lNBFv# z`1GbLs_n$pI{!^pS+PsgIfK!4COIH>=@LtG5NSDmboMKnz79qkbJbvzLIn^;HO@ww zIDbBHN%Cee{#s<)0~!*k1tyP0%!0#|`)SdvjJcn~Et%ZZBB1}hI?hDMB~{|NMj?wmv<@uX8x=L85^BO#ctc?`K4wKHXK z>w>HV4PlbznQ3Tnp=`ovJO+SvxD6`pPATNg6}*IU)*-O|VTvMUP6}GF4F`w~&Q+xH z^of059B@i*BW%WSwf%{PKJlsr+iS8bV|Ge%#wkKNWg;n$Y{HB}MzH4V#kDz4%jbIvz_BZ=+%bX;*_pr`z(h6D@&?shK;&XHoQwN^> zFhyDcjbP^NU7T-ua5`BrJ+|U27Og(MU0b>kdIpAFkvsz$#UwLt&La9UPyRe-;o!Y% z#p0K@%Tju&QQWJp-hf6izaz|e+&OvbP?jx*y!kR+`w-eAG>Z8%Fbi$t1(@%Xh$MA0 zB5wvI6_9DLuPKGtH07p~!E9Sqf!!5~hI7mcq5eV{aYNBpKwh#TF(EJ6=-ejYBA`)B z%G-#!7Fld4F&cWpF~U8EgHb7@NKO8vGz#n{eIHW;B6w4B8yg>ZAf`IgTYJu5r}5N? zcO!VRL|9mLa$b~KJCf)HWnm9kf3rh!AgC92^<7s#cT^pn224p}av#8aO6Ztb%zEu6 z9cBnwPg-5S?j}oK8frZ#Yg+ozL>6D^A(PH_T_L5Aasg8Q6n8x*ma>?K66OjXf5{&5 zHUeAhA&*AILmC)jvc<$stu!!ie&Dg<+RnWWOlBAlgao}Jf6*;4t=7gy4Ch{PGqN}} zI{Z?^qZ>m)9FAhGRuq$}-Ip3Vjr1$dP4Ao}g?#j?(bqivct0q^_q50E@3RCnifMNR zzw`Fw?S_Yj4xRtmlFw%EM8PWGGA6y>V*Ua~A`YWLFwX05oOfJx=RujU+UJ)Z@@2)K zZpgD2!)Ud3asGo>Ew#txxvKr}{{A1`wTC^&TZ*o;XJh7l?nKvgUEv8m@K+80uTJ2Q zK`1*2H`!})*eJHgaNp`E_T9!nuTyUI1T@s#mBfBWGEuE-D`vquIUQtr&#<CsRV zdk-rwuP~-!w%*eyFYhBSk~R*v2C%RWQ)G{7wr7bcPCM66Z+FgzdO=}I&``Cz{r8eB zuV}PvP_i*x@4+R#_#Yq$x7yRChB))&x~+tEk% z7Y@|YK z)m z2b)4Ph4sr(L3vfK5syhC`!d~WaCLG2dV9Sj)}J6uo*{o_v6i5hw%fKCOYh_Bo*8GE zMba|hqXXhC&wYX|PkZP+kcQmA@><-~%(03#BRAEwf%=GULbZn%$uFo=c^pOp>|9O~ zEEb3pCR&;30kb{o6gX*emzjy&pzD5S9B$R|!`C7u0 zFb;J%UV>gr*aSL)UUi0CojbXZVkSVqAaTZ$Ws3=0A|Q&Z4f=we0NjE8>hsA}O}o>p zt{bx#qsKRYfHa;6J3F7Ph6wO&2oBA6d}mUAsp0lKBp&OEu6prO`U z%nmf{H;TuIHSVRg9TF4K6>EEJRg<;mL)&0j@(#Vu&NM?;uS z1-A%HL^)p3X~tD$B*mKAn46O&3-0tpN52$ObV>fC@qxuvrX0k?RW{dAbPiSMHU#B0 zam{v=H*=D1G|R!WD&eaE2^D!|xI6;jn#Mx_2}`69=26#`gcQPo6qtN!{6i49*#|0y zi*I%H^$loY(gF9lCoQ302F1A{>khM70(SHw!^rnnXF7N^6>*cvLt( zlyep*y_mK>+fFwsM3HsxuBB?x9LD1VUu}-*Opv}0CQF3qv0F~;@o8ZrC~-I5VszC8 zG}4sZi6q&T>=QV8X@R%?XhA&hxiyMn+Ti2a#DmYLYLLhY`3#dH4`>u~F-VKA9VZ3v zd6;?g8Ei~SXL?E!R?dy~R6;Aa4+1_+%=!(IJ&HJKN*4$|=W?s z(ULzYe2-Pq=eIH85Fb`jJ3{ae19YK%bu%0?44M@A zc|X!hUZhGoaBsUlkv-jr*L0Nf>%yej;rg+_@7=Zka~~fyDr34P$BQZi7e2J7e|9ju zL)ES(NWX15Gu?38!VOovL)8YD0gX0g6I!DV=BBv*ka6f{A=53%XFx-kFH)avO?}oU ziK*>!Gw93gQn-2p8pS*rx?^5S_NxwZR*24M`#c)hzS#?C6!TKTyn^U`d8#4cwK9&J zpewdrKa>17OV>ph1)*2`TM3`u&?~~%HmWX)+#VtN`AL5#pjUiH@6YfeUR?~418MoE zL`b;sTwYBJk7@S(HRnhKKhtV#{-llA20MBclgw*HeyUlT)O@DLXp&(9t{_TYc9^fy z$uYu3&Cm&sd?IgmI>ljhk6gKTo#uBk4+!kK^5G%PAajx(8_*~wU63><(>bJ%DV!?H zd9z`QgGPuIvkzf@ogCcS`P@KyKD0*?GoVq-iwN`mB&O(He1)?NZQm*>&wxfT|3R1= zCkLYT?j~F9`O}FTZUD>~Nz8yoF>NA~nMCi)QwJ_P`aRS7D4=k(2cJVzmo2w}f zL1Naii7Exe>688O?VN?Ar1Ph> zJ7uRePN7QZ}OGR@#*Ehb~Vh8)Tx&E{oTj`qKLtxmX&1S(XXBD<- zFs{K+jd&Ywcf2RCu*WlJ%s_bEd~e<|lqwwfZbP0AV!o5f0`bm6cnRr75o;$d+e$$O zg;jX^#5tL97B83EO*|HVq_IUGjVMuvL)z*Lzu5bD^^N4cui=+qmm+EGCyoHa(UoSy z-j!SA%#?7c%RH6XzloVQ2(rx3gCzD7E#L3_PgV^dmWiI->*#H&7t+${ERHU7n5-XI zv5BIH2o5Jm`b5t!?lAg>X5{sm|Il}5ZLor$tbCUGeJ83aM;^Uz9d(3!vBTD@7j~mq zx~-Fo_kf0)VlkURYNjVKbvBlrv=q#MMlnY~p9?+eo0}S%o9y9)J(sG@R^^cTBkj2s zbyZrWF|QSIp_LsCmU%nv@4}w5Q#0W?dg#GcKY#klDGsv)jWXYjI^ti+5vFHC0-iV9 z@u+o}0gYmQNtmmN-tRbFBT4Vjn#Z<_1ZessJ@bH`@ZW)xWF`{*yGhSn=_SA_Z(d01 zV?aZg_Da2dR5y-6ar(<+0xsdG!8gskchf^4+hdc;7SJfBecH&T_w(hz-Rf<>Td+rts;}5axxnaN3(3g)vD?<^fv>&$U^6 z->E&RkK()suHJw~F)g3>Br#Q19Kby(4Ktt-%-qSu=U8xDG;1lnJmHA8C&yGkqnPsv zGrqjz#-Qq+Y@Z;oIJt}nXbAIH)SSDKgxZS7k#n(mc=}O=X5A#F_B-2k#FDJfBu?b~ z7kI`(OVw^cHLeX(@%J3PI!Eh1`CZR(_w?BvuL+U}eSa^YQOq`kSw=jh9-id8{d1qC zbI30}Iyrn*HLmLUSi2d&Y?CpYC#zMGS+ia^;iLBbGiF}0wJc=J^)eTmGm|JHDt*>b znQtKGeRq|qH%K5^pc zk@a;=*;(|jl__~`Wl(9V7Nn4*S5y}NFZI#EVmCKRP0nw&--Mnd9BOe); zuNg-?j<8E6g}mvKR2%t|(iC&_2j!@$zMLLH!t;vLvk5LHim=keveg%F?z`uYGc%DC zkTsEmAY6ybpOmJU374y5ron~&|E6{xILpNbU;}P`Ytskap8Jq|yMHH#NI+lA$p94x zWyAPQMsn}NtlLjiSbTWDBmVRta=*s~W#m#~#J z8=Z)875E7Ojbhr+uwbr~O}6OiJ~O*q-)|-`gSj%GQB2y&HxrY7f@~j0-+y=g7Tuqq zM^J)H%>j*K+IYqwPL~&uxaC@X<&*thc<^Y~b~jH2G>Vyr4ww(vPC0tnSdLyi@0)%X z%_5sZuQy{fm!MHh`;-UgrssaW>D)tsxmMC{1T>0iqwc>2$3=Eu9}W6> z*xs{0{^&f{N3JaajbP>u0bHTGeTcphms@MH<@F8NuSKi^MPE4tohK!9K5r;LaFGoT zXtb6oB<9&kjn&ymR0xs!)H>g;2Q-Q~l{oK)oiMWK6dsoY^HVDjHwMOCx3k*?m$Qp` z=hr7YP;_zZdz7HjN@|GlgNZ3$7`iLU{lA+HlOt8i^@8tJ=)^q58;nKG34Wez|0W#W|R5= zL}`&WQ6wUm~^k!Y@WoF*pg`VexEG9qYdT72wEP{DCYiz z*?}bGcP(!_;kL=2FWU2m5f@|BV_HjC312@0dc`MIFx%3~*T?5J!gW6Ggg(2UdM;Ea zG%nhh7&PnoICim93n`!4EkV8PDnEeQ|BV)QD!BD!JK>;jr97PEl#q3L%E0V*8 zdBApItD$$*+%gCO(V;R2G>Ul`Fbi8EPZpA|;^j1?%*xf}5Oi5no^7a~I<1^en#!k6 zZ=e@5QuI`YK#FYSPw{ew^Tt@+C-1$mF~WWuvv_GjjLgi6Mi`nJy!Q2fSk(d&+2cI^ z9s4@|T}r*}lA9?3jf%_-U0)=sULtEG%%HCF)jFUd%+qPaEF{l9nAlWZE6!Culjf~FHcafl9eG5Lz`=>0ft<_8|E*iC(* zz-U1Dy4$kT;1<|2J4R=Xf4%VYBy2T@a`WZP69^XMvS)AZ3O;hu0vbIY)>4z#@Za?sU2`Gt%xf+MjhL`)q}m8XU3N_pfg^;9k5cB?qLg zag2#n{z4wX%w-<+3$`BeJ!=+OwKmtNC%J9KA;vx+tWGxnF9{S0Un z^9+*boTR^`18Daj=B#`&-<;O=fJQOxUGp=O@)X^Vh8@4xv8y{@4$RT? zwt?QDb=t-q2f)Ygo}E z)U~nQNHT_vP7Q811Cw^1>-A`q`He`eJF)$A>cwh}O(sckA+{(-nP}Jq_=~Q9Uhy5h zZ!**{Ng@<=5uC{gXcW_ChzsPPCWeol`$8Q9IRrF{nWJ_EnCc&kT}Z(UXcY4(!dx#Y zTkQ*;Lzn5?lyf}zvJL1Je|{>@Ob_?Vxru=s0vg5qI2Duo0P09J2w9w*3LZ zoXa}k#;oe$6F(pI^G6p z3G?XWw88DdW@YxBe=Hj}hA;yYAJ8Z!Jq1+Qw9>%go?6ieV~jX1EVO54>DjgN+WM;0 zV^Ghar_{#1=h0|wJEv}&uoQ7^tlP3lKV9(`nxF%X2xt_uf-uLn*n8vNK9nv&w~Psg zM`#qYhNRLB?>}JS(TF#v;+cZpme#9QpU5GkE(-C3Mt}{KWWUR zvVJb1u4iXRgk;5&qz@l3lmZ{nC_K%;W^i)+DLZ0lQ^ra_qnKl9S7k%G&r&|Ce0H%4 z{+7H0RZnHznj;FY3cW)e!gBnRB94HDShqP2PlPEhY)bq47&N$gVi&AMV<;_(ph#oZ ztOd3$D(||@XYYFp}Bqp~D4+-+aotpx431hkusD8Kv{+Kbm z2y^>zyi7q(2RX3QYUfh52e8d`^g5Tw!wTekeS((e40JiMYAf+Gs%A$LkWI=qgr`d9 z;>nUQl6ixwffmQX;}mCK#iPLdGc>ioe-akZC?*|-n6s0?tK4-+bHwk1;Q|QSbu-E8Dw4lV_w28> z0(!-#>C;@2#Ahptt@?j_^U|2g&a=RCaLyah5ayP|!%rmJ?NBO`EsuXJnl2h~*P=5% zJB&?SIwCT88V($h*}td@gP&WnA|{i4|Rr^+LNr>$Wa-K%6YoFo5q=BI@?Vw?L4zvi(baH6TX->7!F0x6 zy$qqzrYuEM3hi3%R!pFChi-Y&TYH@j-0jo5)uU0|Pf0`_p`8%B6A=%NqEMW;{UvA= z^JBu?E~zRyOTE~8(6x(4f=?M7)HbtwvReas#i!t9yRn0#tcLnJT$uG$hIz+o>dBYC zZ9Qf7-D&a&XcW7gc6$RSMb+VjEPDuA%F6K0o7>YdJsN3B?pVT%bB3o3F9~00@vbK} zB7{aUZzRlhSmX{<9Pd~>Yoi<2B^tl&0gYnXQ}Z|hwZr7LV_uLXnJIh*G>Z8=@fk<# zxx<2O|M+q9W1DX$&zK^7IaVc4RzANj6qOt2j;NLQ5C1@1$ z4><&hphIV_xBEU2FN**vycp!?fDbRPC!=t)thW_~iZf zp3`>u)~NQ{93n?QidV9%*^Nt-i1=V&X z7tQbffJQOTC(O;cSDebKTdB@|=pF?$ipg8`UI<>1DPUd@4Bh#uzUL0>Jlh_7Z%OTQ zmNn$vqt|*qBHrnox(Hw8=d5NRkAOxo>G;2}P8_XATTZ}seIrIwA81eq$;~uA@7mpJ zNds0?j9lQnZ??N>PBMEY9S=cL+(A&Gy^<4n zK%=)w59?9`^zQ~d?9c|wO+L6jx4>CtEfE(Ed6 zV~C#vKPeOa)doaAjIIsZ70?i-&HQjVb=9ZbwQdxs%F1xSX$;Z$rw{>+VqOW%LPtDo zP|m9%XpOU>YN(`;0?i_TpCE^Fr7L)Rc zn0MHQ{)qqZjjJU~GK(HRbH7Zq6_U(5RG_6Xqg(itG=!3$8>>rsoHX zzqrp1D$4_%%OOh*y@*iM55X_A21;!jULx0!z-O*Mbm?IO@3+>OYA~&-hIZH%iR*B) zMerM~!=4uIOM}Fpu|&wxt*t*fyWf{+g?W@(nksFfQOpeZFz>On9S{2Yfqh%7mjUK} zNh=W0C}uTbcH{$V=VYk^Aceeni|X|+@X4QwLa%L))Dq@v>nHt#vS;zJu z8Xs??En$bD1ikW1;l71V)0&zZC@?WxImrm}&fo$m2Q-SlDa=(kPSnM@f^lNz8zUltpGhqnLCTH^kI=U1Xes8PF)Eeb|8BD=ca|s{o!N8G73zQ<|1} zCS@FWfT4zhHHuwvJrev(t8rZtj(IG^MCDR?neBo{qnN~s8I!alEIwGnkM2HZ?kZy4 zP{>1(9SLX@^D@NUm@ZKBB5|xNlmE8z z*M*P?C)O%OqnPgy<^^`yk;@Y%eWDlFzVnvqc_(C|N3Y)EfNkD@uVH8+#=$`e8pT{j zKlv8^QXgnVNBvpKFUI0XrtLDsghE&H=zYP^8t-w z+IQaq2USng*z&K$(jZu&1vH9z6!E!5l22|I{Akey64jPanVjRnSIK~0@lOGCVLiU? zR@sc39W%0&P}NzhK*}i-S3sliHxT>5j;qYoQJ)J}9`%QPW6ks zVbA3k3lBY*PpGZvni;^=y3J#4itvbPR}E<~<&Mofj5NVhvyF6tXHZDFZKVCYw z*ReajePSm1<%VAUmLKpUFuy@x78&G#Mlf@fg2QwtdY?~~?E|y_F|zJddQEhYHEtfC z0gYnrWep~GyHih=6?P;aTDjA-TZbP2%)q4&XcW`ljOxzvbSrM=!COtIpqz#1>qxYOtWX$N`(R~P4alq&r>_eDFnz+}jxpmasiAnwET0NPjG&i8p z=GvSBO-bj0>7fJM>^ETcf&;ODMloG_P+$L5;o#U#Zt9>TSlN{u&G&O3?w2M)B{0 z2`H>nRYT*%R)MWg=ah}?z37|^w>O#St})--d*gB@2CMQl6FgW{G%@( zJm?~XvDX*s&xZJ=IEsGy|?13r*a~sf5 z730u(_F>V&`csI4D~e(ZIa**M7mPx{a%p+xTyOR8=knyg~y{;egt8Wq6RdCNj!Kg`X{|b}GCP%kZ2gUFrWeEs_1{zY&?{K6o!?} zP1!-NSXJI+DsVFr4;725=k5W^h4t}Z7SdQzN_t#viDsh>rB-O>UA_WUf`;VsA#9pSUMPJ^%&U4L_C#mIVT%l13K1Y}z z;ghb`vO|v}y!2jy>I;ov=GG#W>l)wAue5RUlvbcFD(WTKGS$1DDRqb;RzJv`kklX5 zF)1h;ZHbLeG9lsGa743GNtZG}4{XMKR{LB%z)ek(nSk7!^qVBg@A|}VM_RNaR=lD% zEjIi;mt{XiT3owFqtM+5x&uMe33~BXfcC?*gICTy<jZ%L)3}_T{JYhbRTocM*j&y@H)boHwFz`DCP&iENtqY-o@-o6LUEm!)f*P(~C#9 z>H`-LSswR>onCo{^T$p=uQk(AT49r-nh~h{8x^b4V4*1$yQwv-*rV5qDd?-PnO|`O zt{CC5GPrZQ>RvQW_QU634{6`^u*kykiKRs z`P*N8G<=ti4H?N?hJK3gT**Wl&2F>Pgu6%0u_5-h<+KS`?9|%;2+C`&7geH#XII{L z-Kz~YV8Z^&W@YNwI1`!vrq&>D3PfTvq0?`}pQ%XbXXZ#Y@V$sR%2p6nzIN8%|27R7 z?mk;O6O9;f`7T@C)!;<3KPgy0KHYNmfTfICl-C?z!o~!XeLzDk{{*$= z=x(EFiw_(KcL<=yoqtGn>kW3kOVB7L-7zz}vNRpND&!&S+_B{O9EHLLm;sGqjwZ|k z@!<2x?BPW;mvnM%2o=5V>J4ZV^8>>Ck}++sTlq_q_10P5G%{uh8pT{jn6W&6Bu%1^ zrdRPo$YabBG>Z8qVKxyDe%sG~_P9|GAA_ibrw_bj;@XvT1u|%RK%o_gE(2sD96tH&4HFKqfkQ-4EaR=+EU~ImWc2CB7rgBieV@X9w(Y+vJQ{#?`u3 zsg?+gR{JqkdmD}Wuqg~6;IJOP7{Y!~^`otSCxaiEDA;mP*vTBuRq44t- zMiuN&F?t(of!PeJyD0@p{psugtE;!6uB_WA;X0nx95ceFm&GfPcR-_<6eV2fi1_u! z@q=(UCOP6ckE9Pg!dZ(zS zz7i?$8tQ9uQ_0rT=|j=;U96E~;?jUw9sg!V~) zNoS--ws`p1(>qb(-~WIGaNC3GrH?WhkCG7?(}`ew5_bJ?36b{z`&hTvHfS_y?_0@g zSpe$#jkIe36@Zz^qa*3EdGy+>`Gnuh_-EHf@xq{-_Bd&ab<^aiVN!WT5qqbc-ASCyWNA1^ zCR;H)nz3rcVb_kMP9DPZLiM}z5Zb7$p)Gx*0di<+36Ge&YNVS zy!e>cOQH%+HMC_PrTJ&n@vnM^EZAzopz~IMk3rtTs8<9sbTZQ+ZC8 zd6$eu8}IFKxwxq|O?&tj>W=scsiow=(kHIC8!khi|EHQQhHQf#j#skoL|Nn-ugbq2 zZm#2MojBAYGmC99A~A=e%uB4TNGxUec6{)Hv2$yC(hd-fq8x?Wz+gf8rUm0=u`q%SJ3mF;r z5|2N?1A4-z{Ngz)k*7$X{LmbY9O75QsjDvFesDt8p~LseI{v@XzC7N?>UmtbSN2_` z#Tp?y5p^G0Y)Mj-lG}A%k9W8@_oBKjk|l|xMGLJGsZXUcp26yD1i?`1BqCgrCz$z6<^Jn1qq~d*>yS$>n4jc9gN^S zfH+SC@c>q-ILa7ruDoh??#_XhX1jGe1Ln(Eafq4_4_@#egPO7oomt=3g5I*a!Be@Z zm-z^c;KYmn#g)eAj7&i8Bt(BE70aWGjJ(b7m+> znBo`3A>wyrf!_QCY)jyYn#qd`B-?qnH(Mp(cn)TU{UUXT@<9}h=dH+KxB;eA>GWU(=Vnx#xu7yhycPGDmsCjBM3^W}4@PkQjW`#O zX;T_xT)MaNaC$I;vpVXD+ifULS^D{-$eaY7C(+4&avsT-_KLWb0Zef*yU@r)`6)8c zze;|ar|lehbb2sA=ixs8$8gPRL)meHsJ+7^jjZ^$yB!uRUz|SrFkm%8>|crrC5!w;izCrsR_7oC>c#oEQFj z!5O#!|AnMQN9(}|&Z&s=2AU)Il?wopHG-@-F56UO${|b?C&7S&NB+ofS&K=H5;#R> z68RN6auH8nWSKcS6444KSE#gcu%kqABKyH2gO7OumAWfQX;)F(6AO^cn`q=pCY$}R z(_L3T{lLrBftxOZl?F!Ww9%%Cah$y5i6BAROq@L)jNtqR>0C&{ro58y2njoeCOmET zU({Q6C~sZ|ok7;Cu^lwkIvJv3G1-Xy*7JTuR|Mzcg9t1^ANeos)ABNt88w$t`&O?5Bww% zUJAnd=4@?v&zoKB&c}|{rggc|YFdMgmaTb_xsfI=4UCL-9E$HvWc6%`OKjZywhqE@ zaK$(Ns5KZyyMs8WIBDnc+H1r_r)!E3mw{JSPE+$|=B$h8kpHYb$2Ah|eO^z{r?R zM+1rvBLGe{JK}SaCEEw$tB2pcU3G6jL@msA$25eCnKwZ$XQ}fh@nUKKq((F`!!Sc0RvC z_P8;Pz|E(**aso7d& za%jkDUddvpXxRk-{6Nz)g zKhDA**T^58Pxm>Fihw3Da(ZYgl7767gTO{&6!Ia1J_b55C*zV7HFF+KDZDFRq5fOE zq+9!zfa_LulqMZ=;#vnC+T}eJ=*pG`MsOmt&2)6(qeGGR+Q0d3_RYg815W1eCCK07 zQca2X8|ee*#LS5IA&w^Hh@&%l2W=&Enmri7iFBBmah#kl5DwVgZQETz55RdtnV917 zA{5Ser^W539PAjF+W}XSZi)+6uWt4^nbSJx!eI;7ZAJL##*Ak5P*zAo!Dmc7N5F%T zp_G6aXK`jCA9DVZux9eZv7F_RB7@n35v(^L)(VOIA*09T54HKmc%exhXx@0ZP2F}_ zls%{1gAweL5PQYZ;|i0}t10sIOVFo*5%g~XdQ*oKO#L(R=fUsEyr|w~zYN5Cklyi_ z2O~H!{n3mii8+W$j*Sbj!qjnayU6s6pHq;;z?#7J)!#LVXU!90ZYG~!M=R!m-65gu zAW~M3Myv4V;^y^lUsWz}S!o#QaF30pff1ZL5a&d)%L>mfN$1kQ2u@s$%~ZmvbTXqOVRVm94@Pk6VQDgVBYtW zi}G$tB|RRD;Cuk+gv)d8si>UX!o>JQ;dO8U%bg136w>QrDaOErrrH zdC~+A)r{CeN&H;<70siGfqU~_GNDWve@M4)K(|TnA4NR{!VD2w{*b)^{Hla?DdbP` zA*kQR5zV2Q*1d%27JmrNG%$kmX~bDEZjSLu-D+j4a7J3|R8sE22+m@}=}uUbt1?q; z6H{oT2O~JwA?lm;P{J0lgH9*p4Jh&XEzPF*qCj5)^$ zqCfQk2I+P`cKbKXchXQOd|V?4+x8x{_}1n5sk-%G1m_80FMO7>JrN%-13lD9>_&wu9e1pS1`oPDxyhAe& zMsOxl@>bSW{$UEx$|1&a@#GPenW!!8r?YRwA{t zCd%#=vQMq^M!9<$2I0gT%G5Y%Ft{e*FkcRPzJ$Fy*@Sqj3nIU*5>hM&l3=&eNgZgH zv!o<4a27@04uH5F%0UFd#_2Bv-|TbdHJYlQ`Xo}hk*GWg7+|momyIl3ebLyng7CPk zQqB6jJPf32Dsm^oT#7icah~&OL0JBBQ)9-Dmj9$IL{2i zGtV9S;X_AoC?1_2jNm*SIKu+0>|_~>cpWaL)CAdsjo1zO;BIxPAT*H#Q0cZ+q+8vb zdGHd-(j|~q+(`ZvcdH`Bi?YA+{(&5%6wm~|6q6_k+ziiIIm>p2hw!$BloRo5O!`e;Gusn zyDH#tm4fhV+s=c6j@%Z)CRSkMuCAQ{^{j_|clXL|X9Z?6X@E)r@fuDHd7WfSig6Y* zjkH}=nV~`&X2Zaoga+SFMf`(Ld-i3SpQuxnE6$m46-@ul;>3APKau?}JF@0xqD4uk zzd(*5pN^OlXucGS#MVhN%LHJ{@E&{OG@J|@T92kD(b zIa-2pL<|@ZgqI)l=J(6Tnt%`UR07C(f-nZ1?3^JP0tzCFJ+ogF>)eQ9!JfyR=8I6B z_3(XlUi-KKNI1Us27FHgBRD%D&K(G!8G+wY`;k@uTvs~?A3AyQn|o)p0i3rHi&e&G zgI6gCc~vmC$3f({olckNI**O#cEfJ}g)u<4DnT?8uuaXhL4nTD?Pb7N7)lx#8Oo)I z^GbwwI8hwjUClB7)p1jYwZ%6Tw<0IMcug&Ut_n zDu3nWgn0()eianhcgo|xj0DhSc~1i)*dIlTvBp`F>e6u?YztQ`XnjTZ{%wJ?p6bfe zn3N6C3W9@i)fF?$u+Z({3;&PVh48u&R{l%m7uw{)T0lF$V0RET2OGC!B|SvwrSkfY zf_LARq&#xOfqb)dN)Ev!^O2;QUGlFXEq3UfyIy?$sZqeUdx$A%U>Ij)&4}7FS%uyz z>OQkfYp5OKXh^KVp}RFkZJP&=__rfxK1Q0|aj(MjE-PtmH^2&(WZOI# z!I^_J>?Da-L)Y!TPhVKR1TMPu`uyV+E1!X~D(uFlfe}9FxT1Uz-|ygC4Vqrs0c)rX zB;M*+JQ$(#R~-9qM5iu4qL~^b@{UG57{-|>g=6=^ACdyZzCnt8mhDYaO8-=~I~=!O-G9ZAK6m2{5VtKz1H(A!k~b2Ygj2i-5^@ZG9E&hfchrLsoTHG= zPY7q6|FEz9X6o%@|7moo(wPQEaNdGAXV4reycm&+d5dmeybrGDnxD{WN7r5tMsV6k zGw+DgNvaP=T8}OodHE#+@k;qv(2yjY9*p2zj&wR%x(5{t8@F{kHHef_a(nu>(bL7o zZDAoSPkw$|_W$f(2}Dc?e@M5sLZ924psQ$|#xQ5fm05G<3=YD(PkMgIs!OIS%U?$_ zv2n9;UJ(BH)^%eyF9B!YyPn2=qO4Ko81lPSz(~1Za`~X155e^uP(pCDCe(%P3z-u2 zS9lE!`7KYbg{$i;fxwCUeS_ZL++oVNz}y{QS;WcXlp|9&E)$u2CN!$U+Z`@^x8=!Msz;cYBWpO zI0v{Lc*6IMzotcp9&*?BAW9fEQW!!KPkGCtMpg1`*!cKc7IFZ=6XPp{d@IUU2*bun zXM-TDwrfPgGVipPRIlpjEnpMw=-Rc(70Y0YN$E4a1LGar3+vc}=gdVF262dKCdybZ zvLOs7vAZVpbSjEq$~A!Z7U1h3Ow<+-Du^jC3@VVt`48f3OgQb_z$bGRYU|M4)CUDR z3QkF%+?ZvylOL{AEXxF=k#5TY5Wg?1@ZxAv{0zgE^GSr$zyQv}egBXC(-5p859gfV zM-5Aa_`F6d$bc*^CG{`{quszpf;;k2WGcnkO`;t+>F|?cWv{>=$eOvJX;zdVReu!t zw%PbfLiVuHW>ik9r|2VDGjNj-Y-!S*6A=*@?7{kJP0{vxJ`WQTK9GCeWoKWFWlf!* zrt6LeBNN&R8PuG1I67Gpx#%|j$-D&{3IL}UEA?OmXCK7*a~vnTB~tb)1x^n}aP~u- zHRCu%@YqR%H^}0`?7;}mJjD4h3To6p5u0L*ySKuzdpJE9!8rnP{zC__%75`WZ&_2R z!Q}2E0H;&BG%$j*8s>N1PjqTvj7(BBjsoLr9$qS}jGb~Z&k6i|93hQhA-r@mdM#1U zNtPA@sT8m10Y6)ivyZR$VMm&NK9diLOM`FP3Qn*K}(u`v~>3NuSin3 z@_&t>j38|%ta%&;s#08aTz+EgxV-GqL#}}*WceC-nx7-bW0f1D7MsX}kjJ+@s!ne9 zDDDJ2uib-@S;SYc&1qEbX>A<(yWvOIz8=`Ipt7~ z`5T2cy7&lbq?Ma(0=(t!VC?o^5^{BQ#4*ss6>k&j!N^c>#>|80vP6^WSrEd9Y7fua zo{t+P7uwX}dN6{s1bQSG6-ZimGufHC@T;?zF1rQ~Ej`8YUt@om$$^*V z{;ThUI0@Le#1sh2^R8Kjodzbk@8;TfUcvC3|7N<5oMt?7l$K)T9!zrYEu{Ct(fA-o ze)wO5*Ql1P?))KxPXm+O{mEwy8J8GY*gTP6*GV1;n-w*bFkQ!KX|5lH{83~&qD7CY zj97I3U?J3iGg&OS*WC;hh)mX>il~fQCPkxN0)UG^kV0 zyRq=CJ$27MzamEB%OOluJE>c^xcTAhugxsAs}XDf2?RQKeuOZ?RseXZ7aT*zv55Bq zMP;kD_~~Nnnh3wlRkRpBiYT!O$DRCilU1Dt!M1Z@cjJqpO7eYMceXk5)RlM#?>L~%OvRo`$EoBkxup0Y|4IXy=1KHE79#qn zM?*>^d%?AkyHS@&h$~|RMsSuv#ATsLffLL%Cp+MBpqzR(HU0=Z;xCV=D_~QR@T;34 z5tS-i+CTTpGmU_=_;LZTEe#CgOcWzdyr~T|s|EkH&xD7sT@BZtpuF!%YuBA`^Ei-( z#IZnbaXE<5=NF8mWWT6vdB2l>@OS13oS$ev;sU%`V|tlu4q{5gr$oEb;=u^co{00a zIPJVJg?qP+dG^oeyW8d0(mn=hKMvj;EX=#MFi9u8-$)<$8?zuzH^D=U5f4AB?t?xS zmo(LoKLgPOcxS^NJmSYtcw!L;5micRZKbXItiB4S^*y)z5wdP4u(|ETf_`Z2x zYfS7=s^JCBZN#4K@E?PXCj4-iOW3buX4oqjbPt2<8D>JZEB8|81+^eXnwkU%b+8GL zk=H~LJ|`0NnxWkGXOjzW$Pzbo7RAyMTX$s{yeXVCiL^Mu)m7dwn)3;Z@{fKDE-J6^ zLK(((%y|eOEm%QkL~t}u_I&`%i8oq81vci{p0A#2-tL*A<~|Vw+v$zXUf=)VDd*{& zWMjA0i6Kg+43^@Gr)sxpU}Pu>#EDlHlZHY#Ast3NU3{@FiGv=D;9Q0{@dz;qr`*rr z5Mqjn^I!z$dc=7Q&SI41VxlN`a2$-?69x}PaBf1JchEGdYVjsB@=)(W652E{g7X)^ zSzIYQcX(dz2pbKA%DWhPQ{fc+`~k%cI zb=^D|!I@JAaCRd;sLZg`JI@;*E*L~~rhyThRmB`fgABaKch>?N!hBVY&V{O<60v|Qs9=zb!B_zsQ4*G|;Kb^hfhHEj?&#PJL!3a*JAu0nB4x>7# zcwxtrTMtHXegWh9k0kl%YQwA3I`~;}&1`)7cg$#b9TP4LKy7HeN&tudDMTfLL%K#B zW-+bG@nr-orHY>ul{7Gdvl!_r3on`#wmw_dOUvy$Q+vgCtvV z%3=P}TMA!DJP9{jm_ zr=0L<){^H$crb$V3D9sTWNBZ0wub?QdH7t98YP-Ih$vF2g5IeF(t|KzX0nRiqg415X-a~A*`v+{onJ-iuJn0gsn0M^ydv;Fy_EEt32I17HV<8V)HUH)A zA960itCH7u zdnP>$ix1j{l(=|q5Png)|G~D)9|LwAi_;LDHc7kSt%S)W5S;~5pt3jPC|y!Nw6+$)viI#YKc*K7q zPKb@2?uxIvG6LS`gHQu>YML2FaLz@#2DhT$DffOL@uQEoPNj?BbG zi1JF3SapC19AX*7kDWFoZ--?!3k!$@wJ%}|xE~=Q4U7!yMcfM><-9SHM^!@lqNlv<*?FR?35sk#`52vEoGfGa?1- z41W|Ijob)~pk9E68s9DgrYcVI=qd)TZf0#D*JFVPBREe%0jxt4r8p({GIsP$@1A#S zyHqUjU>Iit=bh_s6e5eU2^arxUh~hM?Gl6+_O3Z(e|NmSJsAcBQ>7M^Iqr4`mC%$u zgW|9lhdz&@o3rpAuyKh$kaa^ln>vBHfmCTQf?yMFIAtf*^TGRwIz^rzPRc}!WUI3|c02K7 zKqhC|&bV6L)y=}@s9VVH{it5QF*;DOehe z215nVsSZN~SFkv9k_HRVbr?w+jt$PxJj ziUcHng$5p_$4zqXaEwMkoW$e}OW=kf4Vz^E_2Kul{iElv5_q~3ElOm~}L(X;6#pLSh4HV7x|y>!|S<);OvCHxe+J&0>MHlaKf$!Mh8Q?c7d z2%l-HBRn=?*9Tu1-l;R(PyL}vpK^ou!9fx;3t!Z-&uWG9%g>r~+4hb>*z(82&u>_C zeqc_gZdJ#HS*%nd0=N#FHe}tIcfskHYZg{1H(f}U~JpqCBLMh{C@Tz^Fxyh zq10l=yUV7Ya7C)2&@+?0gYebgw=J%?42lnWLlIlVbrGf8`Osr5o~9?eoC_T($@?UY zC=HCDeh5djkhrAqIc@s4YPuLg{B$!JL-vhv)X{eqJQ%_G6yjV&IH%);W8>~t!$X^= zuY0fa%DeGI&uK>*7{U2G;#^EP)eDdnn|TKrh(u=^7{Q62MR9qnPYHNFuOM$IOzS2j z7#qEZN4?-%f=*X8^S-@_ZFB+# zue_mnFo?4aP-n{+a?C+-iqpj(u*h2Y#9e3)dr|x$4ZPqlLiRsR?7tX)i;djab+39@ z$B%fcQ^Z_%zC1j((;C^CVfNuHVk0G+TxHP4b_ZyO)fUIr4&gH+=d^2opFL#ad2ly@ z?xuDCR-H*WwWgj2BSXQ+U2)l5xJH>5n$hFNB%zku**P=TXD#`3Xe!hmjG!KboT121 z0!^fCT#X1=xTwq;7JY4poXoJMknACi1a5`%hru%L0HW`M@j@Wa>F}U!x*#_0wL>U1 zLC+piIa=E`AhtclwlLTd7#7a=d$_);nr7fyz}r*xc3M01b=Yh!M>JC`*wqE z!m`Cb?dsVW3LLySb>nMyJk%?2;$&>KTN|E=y4>WEMGj_oP)l*^fQ=?=auyd4Y6!0T z*o5!AIsd%LH@AjFuTpz2`)JUaDjip0p4J)U*VRI*u@V1XK8m>8G%zxS=wG^<8j0=j zZJTi3!Zj6N``_>&95SC0+x0w@vt1_~I`GFZYKU<}0_pFl^}6fq9B% zc?RkZHtu3Sn#bD!>B9Ijh8#|sQK0;cs3A+~z-8%Q} z%Y$&lO-G(Q|HdK8Go7hS^5oH-cWJFqk3zwvn8H zaO?SvHh$9slRlhG&qR1^LQZ*>R1F@=ZHNY&aB|c3MUU3#0Wr?67w0zBQt6_>x?Lg<0g4{*f?Q-?qi&VT zOdGt1l6LH-nk?uKY=TOQ>@V5nkvSOwp?!ExIXfS7m@*SPV3utYmcC2J%s4 z+LN3rb+Y{P2lEHL_u@>ro|8MgNk#&PZ=109_irs-w*YRg@dy84I?BF%_a!6o35q8+ z;epLF{=Bvx_{jX>z!R9gR9qVNG~@CM6C+`$X`Cw=DiURuTNftL9{a#19RAGT^MCvN z0;t~e+5d`~RlP*TPwge;p+W5uD&Gx7gGlNVqj6-|gnKgH>GQ%h5M}MPdivQf-Fd!h zk`661GPH)je&6%FgO@?w6}w1bK>~;KT_g>gFu(f15vRQjFspA{w_#(s+<+%m6Bub? zcV)S{@<;4;8s*8d$gHrVjR*p5c=m9zHwNCE0;{liFwDWkDagSkR+ADXu@h`UI&X%d z)5?PZW~acqDWlcBUJw=z&YinxLsRBpB$`CYRIvPQc=q(q2M+5h9JH&W`3ky%C0Hj~ zk}=6?DXT+C10zGZ42SX!;ZzM4Z-8#vovt|3zzEL!5$9Dfb3i9XX>H>|qOjP~jv(TN zq=6BfcB|Eq=(H-2JPo_Pc5K=G`@oFM$X~JWxXgQ%PI_zs;KL1LZ=dzko{NHT`i^s+ z`KR>Vfp0nL?Ej|O>95sMG}0|{+jOFC?GDfS4FjLt*Slrw@nB=SAM^OtAKareZFL0! zLk~u9;(?PngY2+cFLtHfGMoW8Jqzc-2u}4+qmOfo-QL{kgBmrvBYQl*+k+9DuOppj z)7Wc?yTJQ7w~aa$a5^)a21ancfH(ufshTF|a$K(l{VIEKjYq=f8M5?g;1Pc{1l@t| zh+93HAkEl>vfG~7W%>WIpTEHRZ3v&h@S<5l_|w2M{={iWzlUG;L;QjCTm1b5-amSP zfHXNR_N_q+UJAW9i$3ZX(y9{M1jo=vZ!<}rIm?UWAA!)tbI}<_hNv|7XEku5lIgBB z2~G_su?r?Ji1SVuQer&ty106FuF1oYoUEhjg;bu%F$YUw2>cE>6I3e$e$Xbg}JX&I=A~DB}89gRD zXCyAVN(OOY6;JFcy9~*1%-ggKDLi9#S$OLqN^*2Em}VIo`i>?Wa}uI6#j!jmTv#a4 z!|A~YPGq!M8K={m*&;KTav^jq9*p2zjW~~w(j)A`?;bgW;+$aT34+e2w4lAG;h|LwW7UJ6p^nk*Xx77LS z`5!NVwQx}Fw|4a!0!DNzSR#-m4Mj_Ja&~7y7xIw8)6hs96F(*tH-HqnBJMRUxXoK~ z;Bn4^!^HY;w$SOenh5c^DKv9RV4lXkW0XdkI>;pEik7A{4mGM09NTk7;V%rxqehr~ z9Lqa|Q9S_iANvL&m7uuFdOwzvxZoFOqS?SwoL?oO)e1`B}obti*+#W+nrXxnqyhpSs6L=YPQE95*W(|IC6roKdPP$%^ z!9PapQ;2a+1A`)JaiXVY79kB$RU$GY{3tRT<8$l52+p$rw76u)^jC656lHGyaD211UP`MVwWT$~aoZo>gB_>0YoKr9od%-3YCy_sj z%zdPUX z+9!EDWY;12k@<)wTj2=|91>a>(5brxuqWa4U}S7(;n?ta9>!J(=F>JIJTSE5_fA*B z`BRM(_jcM0*VsYx&jJf%YyTJ+X0oB6h!$LWp(#8yF!zD;ltcoBrr{KX!Uyoq;{ybu zc1<23Bb{g+nCIdklvxVrGe7J-l6az1^T14ugGljXy3GLFv{&OGwA+H1NNc=0R~tH7 z76;)MF5TW1N*8(Ard-Fs9UMUI&JV~U>WFB|SS1J-UpViFKl%a)KFX71W2FQtMME10 z<~ssW3;5l(5B5zy6>=Iz^!Oon$GMP%`Xhm;5UhM5T$6WA8;CO8Ipycd9ifT?J`kML zk?@6tp-Eu=h=WM9my+L`TLW%IuMw(}^TbpI$e)CiOJ47a{Bl7ajEG(*zzNUoWRFS| z6u{-%>}(Lh$|!=^$Q=*)5Zq~C1osG}mtK~SsH~^~FKk8TSu{e?90d948J`B5)b?6#E+6Npx*h#`*DZsIAy}J+x{qQ(4`1RYyD+=q)6s7aS#N=HeuiOgTOt%8e;8R@7+17ErjnG z{%TOW;!?x&M&d3zBi&D=iv@tzZK@KO??PRyUwhB|oBBfd^^Obr7jLe8Copt9(j5(~ zi-7Whh^a~Qco63W8&QX_W3?})T>dvK-n|G!RY%+wc8AOo;`zgf1shjB8*1^MUg5~1 zsek7O<{J2CpasM`9@_Y)Op5_uUJcjmn~??5$(5PoyYa?*e4M2`A;Bez;|ooud+_Q6E2xh zN07?1h`dA_lC)rZvFr-((;oQ3Eu|yh<3gQ&Ci@I=F`Dvgw7`ND1n& z?e{efUwy&_fjK}R!%<4Gan=Rikyy~S)t;q|psMyE0#Ule5>b!9fEqLzPoR!q6Y};l z{CU#h`_8;-SYRHBPpObgN5a6Pl?1W|{zDgW+TK>_*Mlt~yiE6nu1it7P*j>tv|o@% zK=`_?8BeTBAv<)-1_u2P@B-!whVDv(^WMWPnDEL$r8pMp=;ju++nn+wI9pydH8E10 zRN@!A;(>&M#VdA2SYW%!F@4c?JJn;5kC~+H`*BWt1)F~Ty3N2KeEG=A{ttJ#)*6)i zfc!)i5N-;OMECbJHLz04y6S1Z=n4DpzUGVlpbF@$3gwx_?^lLr}fi({`~!gsxryi_v6|u*Aba49kvczQj2ex4fWp`?;M1Gm2D6 zjk%a%vD7P3Vxw2^=<)^J>U(PY`pf&)g5zcsWB`fJIw%33H27-GYq*#rU#(@O|b>Dnyy`spXKk~j$x@|uK z$Jkq+|7+H!7e=dE|AXF$J9AJgCnQLG&{J!fY*fcOIZ*!L=nFb&Ly5q=OACZ9=zR0n zo!r8E7~X14;tSWj$dK&L@^x0UjW|{Ya%|%|%RYtMXx2=jv$9$fy2^ps87G%76}uz@ zu>QXh$fF>ZRFj8T2Nlh^jE29U(*t?%4s+2H1MYwG8#qY!)S6u5l7XcBuqw?4FV|p& zdysHtqr6y7rVrK=&kE3>V3Wide&!Y8Q3^xB(PTCh+7ZnGoM6{Jw0!g8?w!OfmHqSH1gh2*XI^u_>|ky?3esS1BXXvACdMFQD{;5FX{{;+X=_sKzc$mNY zZp|0j@fLLA3+xPeM{k~_DOJ9RkH&M$#z8@ZyP_~hU-JnTG~ZkEWi{o|%WAFR^5~&D zR~rtGzu>MLPMr-02_Pnqp=)JNP7`1PlMJD#5>GII{|~_}Q>PvkoTbs!CC^_?t6&t6 zi^#!Kk)cy^$ad*B?}=HnmgI=7O4w8s7ZOsG#Dt{VTldiJH4O|iin1iUlObYxQaIbg z>A?t2%S>{Wyj2ARhwTQG#EsKQI3qBK({2FAp}t&*K&sZ%7cs-&%fq@PhwOglTN?CrZQg+a`Sf&eju} z*9+hwCYNDYWY!SP=K(tmHX*-k6P{~*v5x5f#n_NxIYw+FgA1M;U1g=0kS%-$h+c6i zNNmf?9}lC{{Ulo%xe+0E=T!28e!AxKe6l2X`jxQk|rg+RbhlVxFA-1bRFoc964 zJpe%F5>%Rl1g_FctH=R<%LoosyTUM&?6Y8GlOmJS0feWxayHgD$Q_weP%wP-=n*#D z9j6#<4QnT!xBw+CJ-H6W4AQ`3XI+u>^KnV&3^NM6x0zJfgtBP!2~Cm7qXDS;$pBdW zO-LZOARX99@ilit3H-)J1fqc;!U8J*kHj<~koa{9I6P!dnbQeGlLC3km!oIjS}>Io z+h97iWu0GB*yB{=`Rrw>DHz{C#YMU zahgRqS*meIuU|6Sbp>;GF?Fkx#rqgtQ3ldzwOjGRc*=p{mm#k|K-ro@>XrhW<-p92 zgRpq%&cZu4plO416lU4tl4ddL$OAZQQ^?oRPJN7T9d#H4yC=)6kI#W%;aCJ8R;Y@w z6Yo~{<6tuiRKZMYl!DIu@JXTPJI*`w)#(AxBy{IurDWTncnF~@Zx=3W0vf|48ccTs z0n%PGxj+hU$DxWW5uXM|q{ALo1PC8Jtf1+E>;>Ax91qg*5)DNK-Gf1#c0tAze^Rzg zvk2zbHm<-2oD@@~4@_WPM+%t+M(DIJAT*<~tHlusCGks1C+n&+Tvbz)M3fK#8Mz(y zY~HfGIkLP3b*qZOSG8PzF}~F9lt(uovbQB6g^jCo@Kp6#*re2eSwu`$*%hlRQs)V# z5YO5m^MKjkJP#TnmgSm1WFFGM$UNA~7duH_RAyxvB_|CNo}GnC10!_argSEH;Dr@z zq}Y>uFrDdOgwDUv6dZvDj za_ej^bo#}ir7MHgMbQ)ekRj)wEVzr5J>Xj^Hz6>`I0RK!hidHl5ap|N%(d-2)!J#)Md^udF;s#)x!6vMITVC5~3%Wr}(0%Y! z=x&xI5^lBpy1vdI{hQ^g{h5u!Litac3UP19FxH1|y{YhEa464!{3J3Er<)2D2bo^} zz^SnC?FHTsC5W3#)qG;`QSX{4Uhua7arlqIV7t`I71p4cE3P^!XGC6hVrU**Gb_9bLka-ZoWpt z7F9H{D!g%q0z_y7qBuPm#Q8Kp+4@Pv6?2B*o2I#iv^N9vxNh0T#f77ZrSQ{7(8OA= zV<^aMQdh)V*y{5KfIkS}i4E9Y0fDQ7%i`m}?Ztpk5WBsn`#FKDb4CM+gfjvA76g9= zblDvEPhw=;OOP5@tw`MrTKp?;?S6c`xeejeg@ipNxf5(&0A?leK&a%%uylJe$e1|= z1uZjy8RfPKA@*PUhp$2B=e<>CO6iVx2kLrSzidz!n{f3QD0p@U+=zR!_^B$+-cnmWXZ9AVz*CfjGMm}V`emH&RI zQ+4fx*sxooCipzz^+z@=4E`D$gt_IP*>do)Id&mXZl6mFnM$#+Pn@12n!o?SN^7uj z_t#0kus zw}{iqGsKrb1Z^V^UWCnRHvRXC!BEwOW$7Tow}S}-{Au6?zrsiHlXZq0?AILrb>hz3 z0YB*124G*pSXsy(mVkFY3&!N-CUxWe47;)OY|DcE5T9oW1JU_@Utmr9_yWUwt}wIO z&DFjS4yBwhdrZ|)>%B3}WM(KUw&NWQ4j2h_x16LNZvd=#0dX;YjAMld%-mTM_`( zQV&LO_Qru&fyp%O;b*o9?<#-hD-YKj3D0<(|MG38HF`vaHVus6ya92JL>i)+C)L}| z-aW5*-Iw|(oytHDMsOkn%{Edlh36xd@Vks!_XR(~b0e$>Xa&qh5|uRYh#&VVrVTcA z;u0#}5XX)XzkT^<7m8o!xU`DT%WeaJw+6@r77Ovg2`gM#=KT(mgL+0&@g?nIzd{EX zn{e#LuT0no3b4H1v;48mrol4|iyizS_z|ojBS*uF8{_irf82tfVZKCCu?e^L8`S^L zs~~;tlUKg@`nIdj3i#*t_M^=P$S@nRe{90nKDlZAj@5Aga?`rj+fVq5>b+R!J~GGaji zyqXNV7_2*GoA8v@*XKXh45AzSkz$dj^fvIx_`_MD#4aGyHes);+RVD^SFlw4VH(p) z_&SaRo3NujHix(F`NMQYC5Spg6flN5UHrjHL)Gd%XOXd)oAjZR6gn*DMJ6BLz8$bgOwT#ahnSl4Z^Rf#QAee?tBJf)fK7#igt_iWMYv^h@o&0P;=FefqS)d7}yd?QBx2G%$kpQRL7W zXe%Slf;5aqKE z;)qvOqeMZtlF8Rcg!o0slk&rJ3UJw@=K!OAk+)$I&M0yZ29cxuQO;;9%bCPW@E_ZR zS*zRS{IzO$5dMAPY3Hr1R|uThY&nw-MhM3HNaik_ohY4sr|+zF*`PsS-zSwgq0{(@ zfYaOIdoY5t3v_Sx#c{Um+wbaQdiM;%n`Yg4d|3Wg!0D;82O~J=BF;Yvr$!5A*s^9& ziEh}$a!d0L;acSuBFj&^R(UXj6XP80XR0pJ=>XVrEdsll*1t3`h||u|RCH{%ApbKX z{HJRrFh}{5&p&@&pGiO``NN5K0Ms_&%L}jjut}#OaA9D~?8;lqT_2dA5KYvqtm@cp zY4=6I!gpHk-@CAWo-CM&S|ASoP`EgvyDxn(Lh6Tc1u3Mp%9s(n(8K0*kTO#+8fg!He+4LT5S{!HJKcxYZ*9-*NvfKg~ZEcr||d zFVEicES^Ww*wewt*r(&zx1oK_v<1&?6Rud$`ik!T!TG%Xg};B?qSgXnw9`S=7Y{~o+KlHemeJ7)1YiGq=BIPv_>>R{`zL zf5tysKp#eMu0xm~fvwp9jv+|rL8No~%Ll8M z*aE^?dcaqo{Lv1a#7_1(i;rVk_JQ7!Vk+xXB>WQDXYNpMSKye_(pX4=M#@*2$Mb{W1V? zTN_=6c}ElGUjI(jN~b*(m|AfL2$r7T)lYOdJUlQf;^Yd*yuP@^%$so#-Xw?p zUNS9PHG>O%Cs9W!XxV0NeXL@$8v^rX9EmM|CL8F<<*v)<&~IvBej&b~8)UH2+QgsL z`J%YpD-$HJWIZu#T(AS~a5Zk(=9ss-O$tnT(!)oQTx=YdD6zraEEKL5oo&VSwDLy)l_QM4Wj z-THtc^ut!v`gr%Rxjz7}P$I)WN$gd`xa0A}nvi ze+)M60x5JvJJ>qtj15LnN5*p9Bju5-3M{Q=K4H|+(PjdoU|b`~^dRw4x&#Z)ig}o@ zs5Xd|>;Lwm1%0|)5|~E`M74pnO|HF|#|cEWfv*sUoJa_xn{f2eiI&Rf77)fJ$P8>G z_gX4Mn1uvl`Gt+t!gt4;N7?;^seER{;yP}<FC^vUqGqWeUPr(;&iDXOHhCpWiC#Y4#X9wvagb3Y!g0=op7Uy4e7Cq) z%2Kgvk7S&&ed7aTU_}QurWRzjI_!Zd7M`-O|M0r^eRk52z`Pjek$`->VCoqqPV5V% zz6m7Vhv3E<$=D!q(tEU~2@InfPtU|b-f5pg_Y9@H~bs4zuPSD2$0vCE* zX%5MRqMNfd0`n<>sD18l+QB5jGdZ z&4iETd!kFRh{mv{oAm^uW}W0{CtrCvl4bUhH-UH8=>j{&&npK`?@pFtc{;s9E_;Vu zEKjA8$OSB?W-R$->x63{6i2iuU&OPa54Z}njX+Wu8(!f}Z{mvz39DP!q6 z8oDi>(jp52eZU0~x5ZP|iQ(h9H9LqFm{ymMAcl`;!nCKv0^LgSU(_~!2+9YBPstVK zx$TRdxPz0uqz;-*Vz_tF!M7STT`7A>^IdO-cQq`1S(jz9m()Rv8^G3>_mV0M z%v8A}ZT6EeC@r#RGf}*MRMR9_*czKZ35&9h_yVU2%tJm^P+<@(yr}w@?Fm05>7aS6E|(TIdPlDXD71V;A?+083SX`UUVzl8U<$eP0Y>dT0#RB-c8M==QeGM) z1+ZvG^kbL+-tw9siM!g7U=ew#0{dvdqV|}gbX=C_zVVLkxo6XQk6bysZ9H8X`m6L?+HY(+_m zo;v=yHfzAH?;#G|py{O@YTjG~IK8YtGPCd;OS$^}zG$&95t#9$g&3~48H8bvzDH^m zeA5rY*=+{E^{-LFp2S$}b^^jjkIHI3FfqG9&n`ju^OVjtyInulPTvLqO;~U6cN&)B z_uyq%cOm}u#43fKxn}6UXH|gf7nh&3`p7jEyQCVH2O~INL7dN_E<|Zw-EZl<$?M=0 zoU+sK498=Y2O~J~7~K38r;|K5NC5cy!*hRJ|1qAocsM;6!C4Mw)0`8hb62l9+Ye{; zhGVn0Zdf|xwcAtS^k4)h9%Y+9<2do)ti}hRHRJ%|S{IK!QSP3GfjsOdMlB7DaP-fK0Iq5*Z>1piBZ{QQci#eMomzv#UA-jT6@a z<*BHMh=_=YA`X$CTt&nSUd4;kMY(vP-&GNPPfJ7*@Bu;Ke&1eauf5ONRh16h-}8_1 zkX~oaYwfky9?lC(CU4&6?afK}-{yUm50@K*wf=<%ltxC%)&2t()&>?HFqG6AcK;mZyGJ$xXLVWE zQQh^VQ9g8})VH=&Dc8Fj87erFrd-K|Jo%l$?E zYjJ(j7%V2${-m#52OE^%%=BzL;MjPe2W*@$K{l$R!y_B5mYmeZ-cr3x7XED(SlG5W zT<%Xw)g@D`$RhVyzn^nS{h<4MD(+$FlF0@PPcqa^V8x_bFB{sfg%vyDqZOBLl>~h? zQ|^W^Ws~%g`3t6k`R#^l>&m4K3DnhzT9*5T>VuVX$~sh@bwD`9Br~777Y#IfDr4`N z`t0Nd2NuA(;h&T1W`Kj8i^HXjz2#zmc_68li%!QdpH?yzQgM z4J}@FKmo!pY%6?UZDb<^Y_7no<5T!w5RQ24k9wj$0NEjJkqy-^1ya-GqdmzC!%KdnSnW{~?%1W|WB|)+tZYN3yM`7|fbb=U92YQ#T!pG`j3!z-MgN)WTTZqxkE4ET~L5*XR4>kA> zYH;`BdOCZtHquCj5#^@1u4Ix@?+}ZNPF!UD;n?N9qsdU?(4@LBE!~Q!R@N7b9P=9+ zN6Km4wy@M#xOBsjN0tsBIP8dlrTs@NWg&<3KgpbkpUS}Mdv~uq__rQ?7+W%Ve7ogH z${3$|Y{Ee{5!B$1@3eU_%bz!PePk#}$8%tX@>S;0?9#?+e*D8Fgt<30MYHSQ{f zCCAfsFmhTz?ttOc>|vi!2A;`ldY#tmCAZ`)0A{yu;q|tDcH`01KNBXsgIdxyo{Lq|_Ls zjy3|%`1GRnv+@Sa6gpKiu+V;O5s%a`E;+P=L_5`D1xnTeVL=eoG$?7##b29O@rCVJ z#aCMwVnH^}s>>&L!@x`Hs=UcO3-LH*&H?cZn-NTwy46W)H%s%?e8q)qnW6~piAFG4qTV$xMYIjImf43|q(;X0 z@ohaQTZq=vlE99moENhcVD;Jj2o<(~wLpKlQ9{)Bv-ku<$YMpz4vPH=%FrN6BR;ZH zc@G+>nc-6Fd>h0?tNFWG0;ZBZ81!>w;w{B73jpxqE5GRFEH4Y9p~((an3Hyudf7(^ zP(@-SYaI-e8zdUu*7iuS)a{4U_|J55Ucj64w&S$XzpFArxF9J(o#_ZewV z1kpZb_7^Yh0F$M;xIIlcO{vi^BvI%3y3kUSIW!tvg_lSJ&6f+V0wjox`0n4;I}MvBIp`3q#kcoK9jYSJ{XM zO!YD`6xqe>wMB-J;W*Py{>16x)PbSV`d|y3frRdbTcC{b865pq!b!^6XpT66*S~Ip zdXdhNO)2ORv1?~D%7?+__*DFh?HPHiyCGG_)~wa{ZtrNY`XGzNrNG=g8b4{XZ^uGq z*%dqd`MK_;1&-4b?RaI9VP1xq*yFMzqZF_8;ug1!GziD3{Em8(C{rCe)iIX zZk<B~7fY#2cn4ec*s83eP-Xg-`GyykK+0W2Owy~sVlAKwq4D2 z?4mei-|$EgEhF@r4#(fj)%dCGGUMUdC;fR|q0(Qt^7E&BcQHMg-LmHQ@iRWJxG^aY z^{1_!qU)Hna9=m6!|QdtncQlVmrWckS&#xU`&Y}(mZ!T^BUY;O5u(Z;reQ$XevlYQibzcjTO(jKcn~v1QB$`1jUKNH>@QZ z1HO!y$}(c_Vxxv^QmqV?i&hiGnkmXWuE^PUM)KILhJhpLSa{ZL_r}-FEwm@^9AV$7qOR=2+y;{6tC%2QU{jmj&a&?W zQ?mR!L3Okh38FsQ+c#9I+r_AzK*D?#Kj{*{b8ao*xt=kT`i37zqG1rOKGBg&T-}bi zc;wWRCbkvG0*3VVN!>A)9lSKJx(BG|Yv4E;ypv(sBOGF0F>j?ikH;U zqHDoo*5&Pr#&2lTxpw6cu?0jJ$cm*_aT&b4B+byr!<(fyj1{ZW=n%UYetZ@gvZ-@N z%Dg9w%$*pTvSPqBw_;;-#MU+5`J)M&nJa4bP8X@wY&l28O@}CrWnH&}D-D-6&~x({ zF-T!!?Klcxvg8)EuQg;0QIjEDl}=&d|yoDh>Dcmssd@5F^XLo{{2gXRHg=S%cKQ z+syvu%g6lW6WbOl?t#?L1mWf!QlD0aL)w+A4zFZs@70F(@LG&=B<{mv_^5l2TJxDh z3l;Y;W^NK|bb!fHi#<{#g|0`EevwqkCZ7}Ib)e3c zC3&SAi!Vzc1|BwMz9_26J3)~5T`w&U5;#{J^08dEZJ7{fW*O}WzX2~<(S5hFVrrvl ze-vfs8m``EmJ~Kn1wY46(jlx4}a5Kkzd?rHBRW&IWh=s=V{H^$E`~ z)+4dsL^`>h8Sa3bjmP#Y1C2pbF7>0{=S=5xj`Y*2wMH_~Y7q>*5dk z6EgMf8!Y#&MJzYj<}m+-pR`>!sh|(g?dHAPmM}z#CIe@j$Vpkmwz)#2Z7^qDVvmSl zELoPj_(Kl2C!XJuL)n9qDZu|3|Gou-|Fktom~Eo|T;ylYZG3?k7du0FLJ^j%+zVUF zZ8x^1!UtoiAj3{1_rPJn_6+B6SePLV`Dg{rLF;Q-9^x9n>K@(AftZD;cpEdn%gY?W zWa-6C?xOg77w3veKNkG6qhX}Ou?Y-lND7aDvE+W#rsK2wMvdV{0wL7dTe939>XTL9 zC8LQ+gWs|Irq#8+wV6!SX%q)j)?6pCOR{ZA5Q3Wy@3iB_l(9LZz7^a=R&mu^vc__g zBFu0Ty_4nUkj}b+7q?SQgqS(;w6=*RyVeyqP-Igl<}+?U2E|mnzSP3vLGY)( z(tH^XgXLXJY-;PPc1u$}iw7^YQ$h=rhVt_?`N76ULD+(w z;H4}(-KrnEXC~z(sdb%&b^cYSpKDmEWVVz9i_c=hS%tJCJ=yazYzjmQ$BK-O!)57IOTl-Vc2yiUs{o(;P{d^$Te_s)gZMqv9 z;mHl#U1(&khicQ7=c@Xe(FT`F)vz%YvwbUMC|TR8E!vjT6SsoX z#z492tdmYGp0TlbR%xVsX1TGde&X;*V zR$}!G`<}!uKGn5IN7E%8`IHsKles$&66J?1>&ssS>vZe?-~1l4)%>1>!z-wb$gH2E zy!oZeF4%O|E``cHAKSU>woSWnAzk*xPv+cmTb_8qOK{W*cEu$ypI^>vW&#Uy;vU|8F4%0JG@O5&x1vpoHLK_W7p04M(d5*FqSd; zdAfwXdMrS@ectuVcq3{H%2Q4a$xU(&&AdDIe3iL!4VJ3Kq0&YafT94d8ss$_pmA&^ zpC+0;g)Ps6oLCY}8Ml*o#Z*-odlCy}CxDxbn?*1r_B!(1Lt-^M|#U6icf;(I)4ApJ+ z#!trOF5IbAe7@*(k z6B~AY-`WOgLH4vX&w&-j=QBRpmL0+V=0ZwrD1d+U%MW?vK|%yG6F!^k24Yt#q$hOK-N|LclKLIg z%3N-{Byaks7D+E)f+jh0f>SNWgf$7~c67RKwmq>u8i^a*r6rWejky)v$PtOqE7fh> z1Bc$jVTa~5$Do)tVpM+aHi;x!k+wk(f3f+6m1Ks8qr_iT9gNNO~UG#T`o3nbeh*r2r) zcq=)ZxnL)hDNtEb-NFVA%;_BhN82m~1LiRD8tP56nq!)QLtYIx%b8;cAL=cFC;p?s zU~>RJn`bQNnRV_5WWDVZgkVSEbI8g)AAjt%n~_A00z6Q6fN`kJF#t8k5GVbGdk|a=J-(TEEO}vDN8K226YCUU$UPEyw2|yB!m|-SAX4$a;8L+H?tD$32vmOgd+{`&)Gn_(K_ zIvB;ts?RLS<5Z(~mv3)42ctMy5So4RIGu_r-@9@3=kEUj4&BGHM=*+$TfodWC5*g2 zb?tEV#elz21|ws)>QH$NMK{r)Hyp``@YrBA72>b z3-1J&-PJGq!>0xt`t_vB{XhK6YxbM^{}8tOioJa*P~UY%a+%e4Lo}hdkrZSne04>p zzHr&Fxf_@spMh>T&j;}QRT%cl((Qfw%=y~mM?rV})J5o05e!wqd1Lb-qMpSA(^!u$ zno_mGI^AbkH$3Rl15388J#s7sET^`8wdz1(w<^pB_3=6NLEDwK)86#db_d$z`Xsz( zE+W@Eur{}E3f2tYItEG~3uEA|@^15NylYMt!+Kf~ywb9>M++b7MV&9xlh z!8n`=g%=)L;uOz$0S>q-&!^&aFvNK(MS44RG8Ii5?1Y{g?D-K-#U>4Lf>l-%?3g4R zWKBURtRf2j=w1$mw_NkjHZ`7%r;PupzdKhMGcyIJPo)uz?#QrtRsLrg;_1EdeRY$W7Y5+lesF-sydF-GwVcU z!Y~`mj5!++tyw=JO#CM`u zzRdaJnVuz`|LlGJ>3iLFbyX@U%M~RU6 zgEJoyoCo9o!pjXX%a8=hZrfDCge_XpT>zsvi5W}&XwV=x!EE+0M#!?mTKVWUH-2D` zZw~tP&-!$*V098!R1v(A;{9-D+q_DorksYwh+q`w z=gH5!A`jq;If@7)dzn;d7KgfW;xDSt9JcpDcv5P0FLMN=G)jnz@g3|PRsMi&i*O0@ z5EHxA*X1;OpU9IBuK-?Y@s=#pl{dS{j}?)tAHL)6Uww2>@V~Qot>-_2QJg;@|MOWi zqX*JRkMpHhZiQNMljO?xzjFDme;j8qdT5?X1fx{EgNHb^WUIV&_FaG7l`;H89%lrj zIKB2-CMWk|i&9ZM_#Quy#0t|C>5`q2@4au{3D>`GAK<)PRBqEC z0b>}4)BF>0zMJr=8My-?owhyOQlWV4SLw6~y^m zYn<>r$4pzC0gU2Y%8~HSJkCEHvHdrWyBU3y9p1M7;8Pm|P=A!aA{gL&6^HwAUto#X zP;~RmC68s`e1<&{dL1~#MjoE(w5RSfiBOH|274w#s7WsHG~44bV=x!NdR-O984o@u z4=yTwz(w^Vc3EivP|459)lBS8mYdsf5kZD83GpD|azL{$QXEqeLN@i2M?i)cm_PGbL{(h)5;lc=3!({`2J8j?TO(i(p zE=qE@0hbbL)o9H#l7GzbNN>h=5e(7uNQ^lPKjV{mA|@URkn3b`*c>}iM=8t#~Hx@=c~Bx zhw*31_1dVxQCoM7o(sf(fcW?ndlgscFCrGISNG6j4~+Yh!cg@0QtESwbgG=$4tpLN zsP19RyoU-3TPf~O=|R)SY;;xK>w^9UA2s zws#V?JHRUm+faEI;lu4Ot^rmVJnz{Lzvqv*gvP9+XyHJoekOgUxdqb@5TBCN#-E<~ zp^u*XhXd{Ss(Hf<6F&4Cm$6gp9d5%@HDN~c_=N>b|JJSj$qy)7Iax;b&Ye`lz7ZfY)q-OHUOBFpAo)tA32|sdQf@ zPS;SqsT@irt9`AjSRi(NGi#<>EyHR~;Sa|d7(7Gzm&W}zY-t^KTmDS*9W z+iWi^c!Rk0fQlGoSFmt5VZw$+1VgbbPM*Fu-_7F`4ZqM+k!?u(S)*C_Z8@gQ;mwhL zhkEX?e)O^uTD*M{yez0>cmoW3odOjj3w5> z;Y%5C0R_1y+<*=?dZZW2%9#lCHySuL#Rl|J@Hak7PD5bkVsD@cOHrlo9$Hie?oX*6 zWeeLofw;+PW_%Y}E#y8UZn$-&JGaew2aa$koenVc0n$nSOeuCu*K2u?zRV6lw*gf2uQn(!^HSf-Ys|UFU<>*0J6U!8X*c*-;Shk?iHf(6t_8LuI zB;L7VFK!OD1|8Z63i;)OC!G-7+dWm{wS{9l#m8L=rzX``TobyN)$&eV_fcK%6LtCI zDBzDe#vZj<%^GkXr&-(gm)GIT8pX6w_slx&`uR$1xd=v0@gi~0ci(-GB>-Nzjr&)( zWSDPlQEf`)7Dzn4Z5mm?>uWogIy8s88gm4$Y2K4}$H*MY-B-xn%#=H<;SbO!^M=WU z#ucrN7r-cT9~+^cyGeMkESN5gc*908iqkt#$fqVx-RR{qz!cO~<2yxhQCxQz;@b-v zAy9%uU_MBpLxOIEuH1J1FWxwNU^0f0XzNr?>yP%tJGRt0-MH>HOZnuM^O9$5Y}Da@ z$@TQ#X;S`xIsb}7R{7M+=k9gPFOC6o*NVe-fT6$^EpGxepCo*UQ>Rq#{o?mN^V^>u z3!I{Dw>0#6IhllGpg<8apbBvhW9BLedQZmvNyh*sLnh5f zBwjqcx}+#=3fw0hdb~?2PJwx*Cmwp78B=>oFzgNsnLYTkXWssi?#JRUghVhD1^lO9 zOgd8!nCr5S9E_6<_r8=u)nt!G>z%G$Z1L!$Z)@Z|Rh@M}MnVvfw~KynPa45DAo^wtn2tR^EE;^eLiyBEeaYkjQhB8|IG zmpohe@58L)GkD|Vo8oiL(Q8VEt4O7m9F3Fp85BnrmBY1d}Y68d~5K z*#j8GNyEolb{W}k&T?8lpM#pUN~nhjUhzLlt4xz!3NWOPK4r;0s4re(hvT%c52=V| zHn{-Y#3Oaqd;g4BF`vpCC#OiM@hv=WpD|4_xaUZXDI7Inoani=zWo~*zltvDsNC96^RKu=6oqy=XVpOW<^3tds!l)KwzVj$VvRul)LIB6r% zc#IRG+%B6NY|p`Je6f_r*c^0n zCx6gV;0G%*?xqyn3rc_^h`BnSE(9JuIeFNL^=ID8BN)X=Cd`d_oT6$hr+xp&58ipx z)mSEQOj9l9hCKcVUh(@#4*g2`5iKt2$g_h{ob)}z)7u%#sLTE1*1!E?O)vN1T%YGN zf>E5OS$|I43I3C6*&XarrW!Lm<&iO51mlpKg>VY9DDN2>uudB-E&+>R6z6@!`NKTU zC~1tD^@IuYSp-9z*8$4jLT)}KIDHy*?D;b3*2$ZXnc!(ex&E0SG<8P>lBW=@_(+O; zq7$u)sy!bH7CVq6i`>F%UpA~HGGmV(=8%o42f_BUL~M^Uf&tFW3zip)CEm8KGcSAu z%w6zM@3Ab6PfNch8jf~4qZ>T#mcRU{3phw~Ce@YIrpPXIr@qf;@?n&OgEXqi6kUsv|r1B3h) z5D}P&oCDLIl9Kc6h4{s9e(ydJdUAFxb8~$0iuIK)p2X-X4&!QwL@YCkPx1_QCBwLw z*>1qd+AI4Jf{3Q_$PdFBF0f^E;9ANR*|z+8m~nC5aUvMvd<0PD8UFlTaQd-Kj(#wX zNBbBe7{z(u6!6IzW+t(VTtSuCB2Pvzij%h`nm^?ERJokeTHuUeh||Xh+A3LJ5;el# zW$MShai7bFYy<YT<31hY2o0`Pb>cz zKK-hP|1!G}Cr_s)Et4?>O!PxuG6dKIRs*IsjQ`|U0Kj>2HRlr3aVDdI^@kd7>}i{w z9_ewPCGf3DVG7N0*V>_hz6Pn2tBBa1_7KcGZr>^e3I;91(OJu+z)gAcMes_Iq8JMw zra)all!tk6i7(}GMlg!=f$GNG|v;5Sn z=DM>^2ino!YSGIT;*a2oUlkYoqE;-jBZ^x@FeH*^A9N8+7H8xgIwg~w&f|bl?>mCY z;*9Ji_I=WA<>MBmh)>1ozV+j-a`d8TQgcU+N%m_HI9UJ>F`GpF7#t?q^R;^3E4}=R z>lR-Z^>lzyob1w>+Xbh06X5~3Xu54oDoO_!;ry?J&e6XBG$~eKy{`U7`e(O}Tbd4P7#VSAy43 zh&v_RJK=XibbwKu7Z7Kgk$sBWyv6Bil*UEb0Ss}1xwKinOMJszt-qAJV$+Qh`Yo## z=cFT;EFFI!rU;nD0@*&U?5 zLbKRc-cB0u0u-;Ag6WJs9%>$c&69_|!<5mnDIQuGQ|Y`2dCtEdlCe8i?Q=K#G{n zN;<$czK&&e(A!MwuNHR z??L#B!D5CT9iHlU58s08nvtphy){nE(2<$S=12>% z+HV^nnKH*~&sEI9x0k;v5_N#dQulZ3Yx82_^kwpbqX6z<%tZ9vpA@DYeH^o|)^Xct zdduNNsIrLkUzR;qYqdX!@nceyncjj>?UvzFp*9{Wr`Jc9#j$T zA!DM=2kE{beq}d|i}b7VZQ@cV_3oilGr_^MBQ2l84T+C(Z(WA8H;OD9&2Sf+U~hfg z&xv42`U6m9ax#a}ZHp-D@eL9y z_?QO|vQ@0%y-h8Q8|-pwcaaF(*3ItR4Z3!=#UCAUwE+yG_VN^aIORYg>NC979Dzq2 z4Lru4zkZ#7`}v;som_Od4WV954v})=HiUW}1tgFZJf1|wTtv#J2vH^txkXC6_Zj!$ z?w9QT(KGqy`}y-D{J@yrNd!H#B#|rMMBxo(W(9v%3VyEx7aiJKia)UE+(P?^w7q5F zEV6L6u+RdfdvVO%MwoEmRI(0+I6p^pH}mK70ZwaK5$pM$%DgAn?mPW^N5c32cWY5l zGCLS2inR~UL7dMMW_X9{O@gxnjN-Ho_GQ88Wmk01vN~QPlh|!cVcFZQz$K{cod> ByvYCn literal 0 HcmV?d00001 diff --git a/tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/cProfile/1596585376012363.8_1596585376017246.5_151-algo-1_3/python_stats b/tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/cProfile/1596585376012363.8_1596585376017246.5_151-algo-1_3/python_stats new file mode 100644 index 0000000000000000000000000000000000000000..f75b03fadf6186197f526b58bcc5f7b40a159073 GIT binary patch literal 30483 zcmb__d%Rp#eeZ$fiS0PIwuRI1_{GGiRSUo6MOR*32Y1 z4oaawY4L%IdM{eA7R3~MjjdV)ZiTjD!N-lDQpPG&`Pe%ZTjWtBQR(-))?Saj*UaRc zpnq((IluLL|9?IrLj+2d2eY7$rwWOsE26NSrEd28xIvqHLK~ zNDJET5zFwf(rB@$5U8RjItk9voHd%O1l5|T=WAf4I~&l$+dAsuK%u-n6uIgE<8}}+ zRHaYcItdIcgn!raBSF0~l&gh7MU)GJIRS&&PtQUpe$_gV)*oLA~K`fYd9Qn9V~;DJCzE~|w@7{a9BdZ&{al9C*2 zBtc#r?IeoKwld#&iea3dGENg!qg^z@CVIhi&=Zno8wnjp6vsl8HXx3FCnd5I$c|jG z9(EF`i-suD#_@_YIus7p3+_acFBglLrqt}w%F{B3A&&%VuAxD@&`!!?t&=Z?Inlw< zy7&i0IcI{R$UHhyE^nu;=F?Vl;EQ`gx@1ZKFX_3!8Jy&0rRS{Dc~KDd zb55r7!z(?dI*Gj@mbaJRfl1Olwlhtn;9RXH7zmtt#a2AxQsvo!_T`%iwUg&y+w(SSz)*JCCtjO&7`8U?h%@)31tbK3GH7940buW zM|0yGLy9uT$Yq#Zi)s%st|At;H!UQ*_wvWzK6`ipN`1LlXkAW~EX87Bd#Fmk zpc)lQIo8MlNikm9gddhn4}EM=|Gj&TW%mW5e6FP1`hAgAF$Ov4)2 zg~Y{2;pFp{{PJgS+?O#gNq<|_+25in5?L)*$}@?zE+WPPF}|U1PtWIPF3Fmu95d>A z{G~OB2j-@Hd9;#~>mI93FDLZHE6D4Ss`gD^TL??xt_pIOpMISsv(6(nb*b6O2`6zC zm;e?FqSt6<)dqHl02G}?H=i9Shs=*w0xWf~Mmi0D6D2$fw>)vd*Y5xH{A}U8Z=QAC zf*C95$#OCBPxxzdK`<7D#i4jf66i|b&K@RKMvmFGd2ao`xJ-Qq3GbeS;k`0(B;@th zsIdE*ak`vJL!-F?#xvdF7pN+mv!d$eTov7wbA`xp5pB8ih=IFp;I4I!5-wk^qZn2z z={CZ7%Or6L#vO>lzOCaF#}8k%)@A<)O|9WTzEI9yR-em)S{d1-R49gl)I?Agfhs)$ zR*l{Hnm|ZY>x21Xt|~ipb0M&JA0CCpGyY}vhUZ?)n3smuhIK>FQ(@eRHLzr+hIDNI zP$)UmEu|A57F9K4ED3G8R#s~1>Ze5)d_OpjMmTJS95xgy$g=9$HmRU4a}<}n2P}C` zQI67OQJ18*v~!Lsh9xGaqiBg6X^F;7y{}AK`kRb-VRBkaiBF`TEC^5tUDUA1O?gZSWOC(S@|X0&C@0fzoKIpTvHG0vO3-ig3T$xHB~I8 zy0ZxS%u3PIwCOzAP7SId5FvF3xny6^i~7DeC#^digXJn7ZM6SKyyB6 zEIp}W1!XXq~hnj`6vzFDBk12r+WxL+>V3L%5k3n++4bxg# z^kau1+(qSg@xO*(4L=bdU2SA_cXn)TUF{!YSSCcaRi9@+RP*9b!9wc?cr+&mXqF}n zt|G=+L+O=W7Z*>e$TE!H7q$^pQ=mJ;U68|F_pR!f*=i#q%nf0V=B&MixoS$OT#JSe z(A!UP+~=i%AUBxr=L2gBRj%eo!u)oogGF@G&*D+|Y0v3TKfL9)8S?_&?pxN8Riebg zfXYYIB@xR75s`MloaFatk6*OM!hSx;;dJ7;ha9Q|^McMaFJv0xCj9#W41Rli5VyO$ zNrBk;>$Au2yY}{Z|DGx67pP)eM^%h(3qqA;EaFj{*xGC<)7BKV3(-arhs~tcPPT!R zzos6E*f9g;LjBnF+@1&`n~nV-f=^y_(9)^BQp|{ESBuVjImPOgoulqjA%O3K`dhEz z(VQM&YAo+8$?c0SQPfElB1DJ0I(}7n6H`On+umK=6mn;K5NCyzFsq_PkP<#@JoUNrp~w8QR4r=>++4v|;#pk^G?R8cKw1*l`9W*?Ehf z0L+d&e2JU({aC5&6xcFUYnltHo9i|9LN;R-VYeF7lyz{gfMbmy7p2#e$Jw^>D@NNV zwC-6y0ZwbfVgDr?)(2b1f=hCh@Zzv`L3RCTr8Y)pt%3b$M6TKJI-{`$g3QhjP~L@D zreY0_nyMNN7RRt&#VTn+bk@!gX0v3BJX28^bEfgGoA3=HPf- zzbnkwYpEXeV>h4q@%6dOvIYIZS^MByuPQ@^iW1GTf@Yb8texMEaFGlo;zg4+^pm2l zJ4jJW`SMVBu8`4U^{UHucNaq~)@C7QYWe@Tg<@;&?%}MVyEB z7?uFaF1G$O-6H+c5UdWMI~)97Jc$paL6j$TOxJ5)2kUfe%dgRqerdz6$BfUfH%lc| zefEU;-GBp3=pK<-Tk|}=|B)x}eAl7b!lU2%&qG)BPGP59Et)5>5S_|XPZ$JccNvr9 zY|A(})4N_+Z*Z-XE1FcVx51D$OU(7Dm7r3bGpRJhTa~4)Z+kG{Z`{dMcmJI}8;?W< z@veBQ#wcFccK@W!bcEeHz2K%|*>4ep#4wLF532m2TCY^}(GBa*N(5&#Rs3BHZ2c=o zuMBvqAYZT6%A@Go4Do~!TNCIo>jMbY#C)UhxdjC#N7?b2|>)f0F!ONIWY%<*+jg z90B%g$9c9wVk1}LNebas6vo0j6*k1bC_xwFKQe{KxgO@zFh-;6$>B=+q!lt43{hlB z7Kbd4y-xE>Df(A>d-QiaOQ5{!R^pqb#KYbn);{gJZSZfEBVSC>gMss{ObbUde4b*RQr) z4|^~M=Q6I)X2wTS&=shK2`|!)5xdG>Am@f^qE7K(403tKZyFBT1f!@q+9Ot@aoWIN z#T&#L#vuMAxn7`3*g(z@LAV1&WirX>VTC2fo&V|$i{@t=!fOL~Fos5sdrz}vUNs2} zBG{a4(u7XV4e^B1O&1uTpYTy4xt$?xgS4PFty-6YQgb2WlRcaY&Z*SmLySWSJuBYa z`B5h`T3-BH+L#_Z^#KQu5A)hS7-l?9$MKS}Smg}fzcoI*lIS^DSM`u1KaBf)ww{P} zsSuyi=9x`-V1_02b=1l{o@O9#=1&QC^_ZYZE*|GnOEV!&g0&r;Zh$?iqCC)xgF7me z1i3kbYLmxy{KV|kom`hb2hi3h$o+hEIxOMT{=SPdoCe}>@n3wfACPbWU33UO%7RX&IR=sVIpJOW9=yii`Z7S9hbi`?;0Zb=Sk*rI3X35!tITsoPT*%Pcl}yigO%1y!Dyu zPI~Igr)KpWVP}4&h?g>j+Pm|g{K8M)l%@XZd$M#Fdr8%`sZpFB*k*21?t;?G=)c3% zghL(LM>N@pHM;nLB5UoXFrhJSBhbU?!5EyZV#G6PoW@N)ziznj{c~19$zHSFgE2T+ z^@$&+ahga`a=n77zq8>qjKRr*Q2Zo~)9`TS&X>M%<*moJo9aCngOfYn;^s6?b5`Md zl}VZG2i5`S%Y<>G-h(kXnU=(Vr*RtBxOC3{`jbts5+_q%0;dOKaPs`Hcyk)33isDC zh1oxP?dZspX`HwH{RK~aX5{yo!neQn#E))y&n7u*$aMQ4;G6SP z={CK?>J2yKI9%*clQS-H&#?<1S#$%)`D*cCi1P|ylBe+0d9GgPM29jr79*;Y@!xQ> zpOUV-TW?gYkUx{riPA0;bbPvJg~xpxDN!!ytQ)kVNLLD zl8fGNW@RJCZo^p71M1MerRBkET4FMTT*AA0F~RoE-(P#ho=fQH@sif!^k6j3h{F3g z&0ML^!!!gNT?sxt7~)({ovfx#deJDU(t>uwU7a`W$rhe`<=>XCcx3=Mvs6PK@8`oYoMJD4cdW@L&wr?*u50_!o1dgF`u%BUXKCLQkAX z9#M$9z-Ywc7eqWIKw|@Yzb;x6iz+>%t0<;YK;4s-y`mTYA|Hj7bIxBB{cKj&yqMz^U0@7md1`fo;@nz0 zciOhqC+G93>vzLbnp zcX86)udjvdmmMXlDr%CcsR2%-%EPDMDpYzwx@|io=>s)(i=pX73hk8=!?HS?TD3w*@hXwlNI8xArhEqGvM6&4xRDx zOF6i5%tRL$;+#U|@PxV8u5hlxf7}y|m=dfL_7cXeqm)-UFa{?vW3L?@8ssLJO&>hE?@L@0jM`Cu@SaAEH?G zc<=`Q65vj);q!4Y4FG zMk5(>%kZ**1=tz#=-Bp?a%-2%TF-W*p?5E_Cb_|gmL4|gE2Vy$po<| zjng>YrBA$k{*yOlfs^+W6F5B>jk8q*y12?vxtn>;tg{L~{LbYw-nxGoa9*YOlxdJ& z=)-87;@!j<5I#2JOu{Q3(2W^+*DD{s>HjYW&PNF2%5Ku8hKXZ9tb|00%{a?S78E!N ztPP*|&X*s&cd6_{$(UJ7_zeyV4APioC$BWbpQKq(R(D{uSW%WZ|Mm}y^DV^rwLdIQ zId$HW#%a#6K0Ct6^PNz?m%lt1$o@L6&EX3YIbN$A;N1=$p2jI^dGq8BwoGQo>oxK| zTbo2^lF%<@g$p&Q4HxlE3g-HZGT2_d$NCfQlv?n|I)ZCoqkWIVfav?=Y9@|Ks=NHS z;+Npd#JYargeC`49I*;0z!&{eK1@AnUJRk%5I%NJ3vbemerbp|q;cv$#h(0-oWvv# zuPK#czdw@KDZsjI*_dpnUpIE_vR1#CmBop*hYt=mW=#B@~CsAI52&mMlcvi2hH(SZfTA<|Lk5cc%gq@TQ>x_~)qdU-R(-9tq#@;5B~nXN13% z@GgF%U)RRuN1OuIMSpw#u`Cqon(1G|{ z5N}Q{;R1}O{Qp4sf8^tz2=6+CnK(c8g~iw2@v+O0HN6?b zgCR~CU2N&F{S#gz+dpZ)N6y;v*xfh3vFGR%+lMQ*yTBU~vY`^IMb6E9n3YN4EAN>0 z?EH=WVEat)(k3>ro1%_j%*zTU1Ay3+q`Vrdn6qv?@yy0&R>}F*@j3&exY5c~Ac5MIOGr#rXky(*fl3PjiR!`wu_uu!yDbF8oS=b0{snrL)vxOq5 z$F#uAu!dlz1L@Xam`rd!j8Q7D_li-%E9qJ^NW39=M7GchKKj_<)zwhyiAo|H6-om* zEvR9%E0A09MhV`U18j-~r2sV3ZusIomj+iU1yca|firQKV!;~3xUXs(E&>Z5AdG7? z4~Ak%oV?{HKG+fGQHy8aICKb({b=~9oKxoVTO#&ocB#yKK2y4pw=)3bwue{ zWG1qOiI-2kcG)p!KYljM;G*_{%9pQdTFu9AzV$|TRr3W(klE-)IosF2}n6vGz2SAX|MWxlB<90d<`3NAvRvo7!&f5dGY@ok0QW+6*Woy`PsY^V=H}Nm;XRIC=us#5TnU4rSw}_W?9|U^Wci-x2RTWiOt} zl?E-2fr8yW(4J*ntW93i3>(hdBNSzUR15+01fX&uPr(^Ls+AkI9;nI@V%6#^da#YM2=)Px6m2n<0Y5SuBqOHkPk zT^jY$gK6Yn+uIIFKu_BKQ7JVVAbO6_ht1N-&3K8L@!tVj(WmLZQ&jl_=Ijtnukh2Q zH~!JJ)6WERLNV6`h5}2p{Kk%`65hqBR;Ji`;unneK2DtMQ9y|nrw3zjUJI9M&XZdV zOinuHKp`E>PR{HY4i1c=!0(ocixt-vr|}~lYaN@e@LvsMaPrF@;zJ6j4O^pKEiSuB zQgizayt^0X!5jP*-c1XNXH+Qq161^7iT?{|emCdw#ErhrQ^4id5!k?T=Z!Yh4n?aS zUmV^T);1Q)gSq1Q>PisqUjl{CfD_OSu~h|qnv4Tu$dF0#b`>uc-YCJwOK;kJwEpAr z_oa-V>x@0Mm?V275Q(GsI9l1t+TZk3H2sGE%~%hHIKh8>!u?`y9$?3AV4a$*&1&_cc5|(zf*qe?+!kVSTwqkfN5PRR!wLy^MN54#( z!Z%d4(W@Dm@0wHM@1DCUQek^0Q@H)6FMX-L`&>}mlvcEHV0GSewe082rlR=;ZgEUn z(PnLS*P4a9SAR_{ectg&z=c-6gbA?S;k4)Ng2vh|8Kp)4Ne?3_GNG zD*go+Y0khul*Z)0!Eq+zpWI)Ycs>*F8qaCxINA;ceelswN?P^CSCCrQO(?a>qrStU zJc{VyXJx>7bUW9F({V7chDC6>-48Z6o~OUTm(G+92o3Zes64ujh~<$(g_)PF+NGA1 zSXgBv^sR|=Y4dsTh9X4~65dTMrg3B&J6EJ}dN2m(Yv?s&0>aN{j2@f-*ha$GM3U$% z!6cN!Vwl{y5_s?p;Yr}Z7zrL`iOnfASL!5zm#q^fXio0%$=y_XmsbBKC5X}zm~m(p#<9)L_7J|g-08$pd8jyoB2Yd^-iS~2gcyMlsGR-<5VTY?=o=|Vjn?! zvz5zD_Z~d)r+nA7q%Kinp65Xim}-(!hzFCz>3N5NS5C|WOx^;_gGu7_>}B9b*F3yt z^IgYuE*3Y>mCwAA zaSmO_`s@Njob!Pqx(F+c)INwEW~u|Fi(f&lA6D#kfibB05l3-bnqBkF40RHHHdHV# zjnjiMI4=WEoUYbCPIGX(AMG&)WO3I5G;fa=8PvWeW9U{;KcMH-S$x-~?7>vvDg= z{XT>GT(2vmQn@p=Ghn1n_!B%@s!DmvX)gm!NH`cJ<7J~0^{vnlO`3Y7z6aPpU@ z#Bs_Ct=F2Cv^m?O6=!*HgTjL+ez{Z54}FV9+RM#Et%^15)&pK1l(4(|?gZk&8~ogf z7jLHfyLvW;`R(psKfZZTwJxNGF&&6nI$S(>;{O7;j`%T5u{*6F6M?#-Y}4EGV2Bfh z<9>tv;$i$5E7j9eO^4EuT;g@;+9__as?{$I5h#gl%;-NEm?ZT9(^)jjvhmY5$wj}k zp|AgBI?0{r;}%P)im&LzU+)-?{5mj9*St-6;K3y6ALD%oOc9o9=QTuIFshr(Q?v1v zW_2pyD!LI@sS5V_j9^uqSX$~_7oR3MT23thiv9Y$ z4w~)IM@xN#+KM5C}C>f2;b^-Sai!u3r)++IOM}4*ooZ z8yE4nay!JOX3(^WyDgJ6a3|90Hym8zb#7gCAnk=)o0X~uL((Bs8Exb=iXYa(kL}3Q zl4-|-S7}X|&lYuoF?j8doAj!}?Lts9Evm!{_`7tV@A2{f@JJX}lQwGgZ(6H)6^n2^ z5$P{Ov2Isa$Bgr9#Y6Zt_^bT!NAWX7)Xsorrq(Y2KaI#<;o~<7yiHB7Inn#AR!tg_ z$y$29)$*u7oU3af`WopSg7^}Lsz^MK|G-CV6D)W2x`m4WOaA$8K0b#BGRBW?#_*s}fDl7#{&e((36B)|Cm zXSMAGw?#TVThj(l9|G%xtkx8b&*DE_=d|X2X;GFD4>Rg1%0_AsTTyzkhi#kK8?jp- ziSJfd-*a>@t0j^jKmXjGBU?0CcrwDHEa>$JFw=?eTTY?bk?y=!P(|5hxp%MJt|*;0 zW`F{a9x(~ii-0$Ja`^bO4YRbwvCl579{Y5@CJRqScqZfd(whH>2RTEMb@4~+GnZaI ze_2iJJ&=3m>oZYJe)K6Pws-8ZnpyoOdFb0?QEf74r*z9CP$2)x`BNM7V<6U&U*5iy zpTAX;JHpUs)y$Exll!(@yr52wV@N9|#razWYQ(C)Db6dBoj*rQT=-#E*JI}wYVw0S z-eU)E`%=xczQ1$y;L@`-C`@YkS14h52ExAUw=JE3MxTH5TmN^5UsV&CMF)2Mbo^aa z7M_goOwMhGwCeh(8QuugBg4e&BQC+Leoh?HtGoT-7uCep!4+3Nlt)#$G5yt&oA(@6 zGr8uT-A(FSH5u-SNE@yXHf7g^T>vC3$6{ohLnQvk16-A$)Nt4zk?xu~4vNx;odTD>LZv9E1gb1NnF_g}hRU#{UY|*g^dN#q zOO?7qqY0F838sOoyt6>szRtNXAOUOwWMGCBG(rPZ!Xv1R9flZ1u+#FELy%vBJ%cRA z4OsXj(u;vJ;##~=Xr_1GXYQlb{ULg&8H)l}YPrD#wfhxG|tT*7V= zF;0rLBy)KiJ44CV@k7YdT_&$d<5ony2^>ulzSV$CEK3bpcTHifbS=lS{G;j=*Cpr#b-`&2DhQ}jDpo_Vvd^33HR70IZo8Y&hwVTo*gc<55Ih#Z312I) Au>b%7 literal 0 HcmV?d00001 diff --git a/tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/8c859046be41.ant.amazon.com.kernel_stats.pb b/tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/8c859046be41.ant.amazon.com.kernel_stats.pb new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/8c859046be41.ant.amazon.com.overview_page.pb b/tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/8c859046be41.ant.amazon.com.overview_page.pb new file mode 100644 index 0000000000000000000000000000000000000000..4fa3b28664e42b582ad792855d84c1b0fea9d5fa GIT binary patch literal 4137 zcmdT{ZEO@p819u1=>oA_{Nc42=MG4^74EnaG}%!0T1wj^rIfaS#o#fwJ9oEqZ)e?| z?Rf|q0#S%j6OGY7VieK%5rc}+fD--?i9gg}3^8irN0boM1dWIS!FTpcV$sl8_sHpCaWJEfqO7JmF%_i`JJ>Xd zxf79p$l8JES!)Kq_abSHy7CUjVqs;De2r$!#>l#+J#gcBjl2Jw3XxMiw z-lQm%(=4v7nvO^J^J;j*(4tH4%Li4t zG4{p$={xqT@l;Fi_Ga~sDi3!Aq*Yf38@KBGDgY9OWiV3a5D5K#FAwT&HQfEenTsTF$eEUD$z#GNeEpl!hY{J6Twk4x`esKnGYw_GrneU>KI>5bP3U zdHIxGRn6pnw#@OO-9C7RqTM!QC3Da82l*6!Jh+? zGSNA*oiYZ!M6}^H3!6pPaDxW}($pMvS#V!qhiM}Y^8QgTVq1K|!@ZYe!htgq$85$rOGvJLSJ_c9GG-PmC zrv;1itGvt2@?-97wEeYODKXKgjT*fNfkeBoAN)QbHP(sI`_CCXEOr`#bNtb}=KSSw zX;EJj8E3C~oV}LDIx)(J#3C#f$~Z5ykF&R`u}*vjgv25&7X9PAu<=SSteUZ+Z(V9l zZ_iM}buH82Ce(%_Pfq_|RZm*l>=#7JHf&fj8Cix2!OH%mA5!^i~bp+3}LXA9~pf3Ks? SwzVxYX9Vg4ktq-frREPVGSJol literal 0 HcmV?d00001 diff --git a/tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/8c859046be41.ant.amazon.com.tensorflow_stats.pb b/tests/profiler/resources/tfprofiler_timeline_traces/framework/tensorflow/detailed_profiling/2020080513/000000000/plugins/profile/2020_08_05_13_37_44/8c859046be41.ant.amazon.com.tensorflow_stats.pb new file mode 100644 index 0000000000000000000000000000000000000000..fad2811e801b86cfce470fae601a8e6c5f540148 GIT binary patch literal 19616 zcmdU$dq7mxzQ<=2bqn}L0mZ@*4G}^fqF}NI6%kZK5d;y34YPrz!^}7jluV;c$_nkI z$44hkt+1D!qgiIRtkm+=@v4QEcFnYhWIobT$<+Isz1LbZYZzd7=((3ahM(K<_x-N- zV}DfDPVmg+l-)hX8O)a69?4@SB&wv(U@k%dKc{?mY zu~xIFohVo)T6HRavpB~p>McmnX(Gb4qTVcOBEzSMIx7|}Pu(Sp+5$<`7D%GDKnVKx z^QK)Z|IVW=8A}fRIPHR?=mM@C7HyB^1{IkEEdo7_MKFrtF2&`q68WQ_V0j2+c?e{A z2n2866}#6TTh1eN?bWYdxaK6^zRB`Y;jH`zL>>ZJ9s*e&0^#)-zwq=AK>oh>@6(s8 zxae5=MM|y%*6%h&G#jj@0x`j8)RiS@g%VYe?K52Zuo|_3MXq+IZ{bQ%_*BW2U<)J{ zwm@=W3xt<``Q3MB$A@`zbwg@M`98+QN+ri(7oDaEdaa=(Lyrm#rV`Z%w)Ra1qrqyi zbpkmGH%Z^tSmm#WGyj+v>%0*ggFrS0fou!{q0gDcSGS~Cd9;1=(4{@EGRB@&aviZT zItqq0Gz#M4rXyp{kleVqu{!9yf)U1uNh}mC1;xaUAAY!Htd|(t$M*q*7j4gAG6+R|*7jr9dFW$7ZR{Jfw$-YW$|} zDyA_8zE*PH*g^kvqhK4ri}i7;=p$zV1o>MhV!%BCSp@dQH=vY1sxX6Z|s0_e|0 z1`{fPS*5`CRT%NMuZFvk#GUVuqA6D3qCp@R4Fb7n5D30gp8N3T--~#3>}0{o00UDr zGuH{1tgUSxR5daf;5$V0k>ywv0$CISSrh`H_RddUlm9FP(Org6-wH;wnCpy1@mNeR zwiXuZL>2o-qV#?xR)RoQfe1apE`n*h#;~!ns=*M~^GHZq5Y{b1FEqw$_T4-=oqW#U_mN)zbPNbu~jROj;dudUIGBkFwid8o6`_V`8zA>yAx$Cz(VT z45q|6Rzas4+(<8T{4%MG%MatqfIzMc2;|CuK=|8(n%9c=OybdiKK@yce#-dB<9c8l zboZk2Bs&_IAmp8lqAMu8um*mNh2y^^g*sh^3k88(COWht|yk125t~5 zS7#N2foq=%jip!?0$COUSr!7pFJQotZ>=djnu1jObjKN4Sj~Gm>P9xy#a+6HB!&N% z3A=zmb^(Fx0s>*1_kpz^bsWp1>XS=`UHX=B@tBhHB~5DEq)DT#@kK%2(K!w=#T@Jb z0@(uuvIhu+y@@_y$=+jl^iyDU@|Euy52c(Rw!m&BT{k=)=A`UKBWVF6X#pc?0i#-& z*EVeH-2Blzdf=^3e?IUNqYGvCcWi+nO>F_fR9=i{9F-{}|0qd}Q72L}AXr)h2m&k8s@T8HF zrje4Sk&>pVg|`yaxAIS^coY`0)H~%G-H#nf(^rO5CvJuPQ;B`6Ql5&9LV$aE zDmDsFgw^pv_O+Y+;0*l9)r06dV^q%tVxL_T%w|-i&lF5Z$k&OJjH)_Mg1g8~RX$!F zn<4#lB6{Q1$>fP(9R6~*zQY1}WIeHGWXTOi?|CKH4-3*A%g81z+|totDy%mKQ@Rcn zh-3q<08_SOzd|vBv=*2mPlTiT8D2@p@8?n0&|BAM*D|I|oCLUG zwNMQ%p3#a(RL5SrX!~hTDk4#xOr8jQR;ow!bblTloc8LWm_HfKr<7cO=Zc7GydolO z=OTkiEEP>tL?i4uf<5FmQH&+UC&NbpR^;LjPz#Zn+F!rl?Z=~wA8fnu7i&ew1z{(2 z@0cW7QpI@|mGc8K36!fzk5iL^tFapVm9S%UNY6}b4<4zt;_B5mnSxg+xdB+N^Ay-> zI&`=Yb{fLB6!J+O=2*w?5+jNjQtTMg>M^9)F=}Dr^x*;fe(T7ie$|ULioX~mu+$I4 zMqEttbr0?l^-uGPXyPK8xQHe$qR9cJ1CR3eTDH$*UF35QU=yCU(-j=0QVkZBLrnUb zB0?Rjl76o6cqxxiCzB_5Dfi&fJM-6^dz;mGQpp8l5$|-1328;GG;~y9jUD97^0zf+ zX{3>^22SLT)s2)EoRBAWCmus~`rl!lK%Eb2N}WeFQ0JklQ;=wN85GoYG#9=UT7(^Tfk^Xi)9<4fdWaDPm zVI~*KlwGO;TJk`WH<08FBzXhLigzU*`5jrM`I61MRmp`hiSrukTL~t(Q792Ds9=)O zf=XbvH9>0|Oj^-oYmW(NwwR!Y=|N`|;R47kno5zrNQ3m2m}m{NA#v06nuECkwXA7a zjUHr7tD9%4P-hSG}lL4eA14vBvnfoUdhe1i)XagxT!g4!H|--)xuH0-7xcPua62q*1|9!E zmn2Y;cYW1lNdgt}1mDYnN)38JeVSE)3!E6o#))XG3g?QbsMsQEH9ELw42Py{YEjDu zt~_>~HL;KzhCR3(X?^YIJQrh*>USko-IY{zS5nno$((s69_{E~yUp{C(?}nV^__P@ zy(MXBS+X7KUG981qDwmuM|5fD`8FQCF6=((=d2ssxUOyKZHexhmnz#l!0Kjmaiq=D z40G{Vhlp0Cn?+jkKhp*cWm8+Y5e+q^tM+ZGYFQsu+Gtiai+c#GI&LJQo0=JA|5`ek z%?t;~kvKKoNK9_MjYM1=yyq#hpPgvM-yj2-_N5yp4fN^V@8!bUZ1Ur=g0Ei|=O%G$+^ zKEZmiaLM&liXR;j^^LyEJRbI!NiVz1Y~vev(B-)Q@IIPSGpaweb7d8GwY5vYx`Ybvs$@{ z^;Dm~eaN)1)jIAP`ps{wY#EnYk8EGNENuPR)^nKh+$?eSV~3u*#!AEBom7vutHEo3 z8_RlWYTduCE*f`}bpp3~Y4td9b*o441*b}Bs*&0|wjX9sox)A7$A`nx=_G4w3sg5( z$c72Ph!OV8{%(~u2!lqqd9A^So&Q&LgjlqnN6UKjDmWSS ziz5PKrdS&LVz0%F*in6HP|y7vdGyG`x$o-o&O3@OpkNwdTw0wGTlC4&jX|R~0O3+t zac7-{;o_QusF6O3J7swzjM#uL&t6y*0`l*@tc_24l0nqq+LJJ9Ze2Co&V2Q}QEY?} z`ypp!)Uiz!JQ}zrV%1OIGcJG;Bc$5BXT*N$dOrTgipP00D={pzO{g;?M&LC%3hpu^ zmg1=_NbO(Fqvgpv#AQ0h4lrVMkhOckh|PTGMsC}~kMQWvIIAwait+R)#n#C9X&eV2 z9oW9qG{T5guX@pA%On5{Lo5pq9Xi1n07i^ZYX2V@u~COUJ9~byg-3huBvzeh%PbjY zO20|Fk=A3xc5d@;Gw-&UM?<1gRvsVBh!#^YjfKLR$B3m|*tGr5LZJKlo>Z>POktF? zlt!b2i@w&w7&4yX8e_z|pZ@aBk?n{_x87D3ch)j0@D84!Xia8 z@DtzdyAlg{^wH<Pk?qUiLZy^Y&#@$UuY_Y01wR+BE9@S)2y}R`wfD=4mtX2e`W;{&i2&UFNM$FJYvi!+;qj)r7!Mm!RKQOtils}WN0$Y(0 zle<&`k1~8auI^pK$Y$Env?eoR>4A#*KUeYq>(0F~)tklXV0|L+SrgZ%mM~&3#;*(L zKYIx9Ctfpl_+Mg53RnDuT9Z2Qf0Pl++EjTlZbCSZZeI0V(D^sUsGhQHb%?NLGh+X^ z8J-<9JQVaQ((W8#F+q4MPoOlqaNiR~tg7R9^}%g}dGyvZ9uq2mXDY!&xiuF&SPK}j zey>;_yxwIXkFJ^Kj=#uqjlhTzFpXBkU1Y>Uw=V0s?%4r6`t4v=&m@);)KOZE?j2jm zh&?xC(a=XO0wov}6mTnpr3B$Ni6Cjtm(s3olOpl|=5o5$A$!P17_nX}vy;-6_kv>2 zT(`=TB?DopC$O4}Nxn51G5LVv11Ru_i^vB?FM~AbRVipC}9AE9) zFTWdR#6BAESZDvYJ>e)XzV-CWY-V7@2!2LK^}T1rz`7fvVyKD(=o81>NosSQCp zSPxkgQp`>HsLl55Rj#AxQ}2A`nbZ03AQpV_gU<}hMZBdPcPJ&~-& zT*{$2PYez(y6a#EP3_RppDhr_v%!aE(0j()%Xa1NV3hazarWW;E4 z`! z#He)M*>6`|Vx?j5*2ky08oc(mvGojIH|zeNbMyzUvrgbvuRdDM)vX@E7wWs!bJNIe USBzMFteV5p=_D&LVs&%KuI+0i>nwI`=i&xIL!Q>o zG~ClYQLES982jH-?0GaL>QSUfd2(?B#5+BYQ$?~=tW#B_e!p064)@#g(+``w>c`!t zxhO7nhl?*R?y9C-mrZ$5{C;t}xw);Y!h$r8i?Y5s7}Up~Uf%8xP4W8Oj~Aam ze}OAA;!6TQ zGDBPKj(6}myTAx0=W!kG%lc5Qu0ib0zTRAy+tqcwKOA7|s`vMwE)Pw;SywRd?dG~F zt{y&JnGcR~5+RvpXvy!Z!)U;Bz;UR>lU%nLEwTxidwZpj~9 zXo3)(6TgKf0c*+>k;>a=Wb+pl1Z&cn+MBvuZ(tr*4Ll>4QxpAb1%Yw{GrfgxUVbdM z$LjK{dS9-u%R}?+Bh)KUGyG{YO1d6(nWv3|YjX`?Vr7V!zcL`0EwwTvbY-}|9}Ias zM8?Q@E`Ipt>(#qYt8ZWw@5<)(L29*n?d~BQ0Zc8+DsQU#5w7^tH=FI&)jr2Ssy5_b z_OLb+xB`J|*~3reb_3B*_1zAF<{v)YSFd*KcU$15cd(|&rvW38wMX*js@_#kLvh7m z5YJnPnTVoyHZYUHRm&jWRNJbleyDa3v;B9uTW@>R3!MdPyG4qc+*bfqD}Rd?lVOp_ zc&5k9@2B!n>A&7P@EoDH10N5WOCM{)OofHG%oD>cv$W=;7@Hf+)Cl2kECdBS>Z?HQ z?+)MXHq8cR?_brrM|;eqbcIC6*FJ`WaN#!MLWYxPw&8SUr9ey+!fA3V*@HGCjPjC< zSsE}YIT>KJK^HPb8L%@^1hB~Hz)HS?sI?B7 z9tX2+dUJriVjqOBwGo$DNm$Egi4QYT3`|J41mPPSVdkF61QAzGvVcugVSq;?E|~Jo z%?>>Hru?}#00JcZ^%TX>uZRdcUh7ZK(;h~5mx zQ^-Q*$y>%0z!wnvu(m3d_#6cSR}i07#%S6_L^F^O@C3LES&dcr#4!|%U$e?WXe+^N zDY;ivg3+{#h$0Iiz>zT*vW~nJLM5OJI`@vDFf*mdLXjECDD$^twqkH-E>0P3QxX1$l5~=+Fcyfs83=zpXVOc^;r9%2!DxzXWXr2d0`iLoV;Nu}H zulmCD2Hrwk#zLv`uP+44(gD5>g)4?vcs_>!{M=`qF(%(dOu!*pLKZS6ZJTcz4ndfe zppD2O0GFo-%3a*qUcy0)$W2<{%UBJwQogY`1b_zA91u~7-4PkHJm3OH!CW|t(2|%W z;%`a7w2%ag@vcK)tdg;%!ZjuXQoaOZR8men0dq6su87F!D)M%2BQEruNo_x8@reaf zc>?7c4pGkdgy}F6hK12OYiS;2@@;3wrUAgI=KIK(jO@2f&S78gLOKhV?FZ0^CKc z1|-HOC;*zInYn&XHS zv>}%W>3a2~PD>&H{o8i~(}1a~n^A8AfF>nm8PV&k*ox;WlQF zXg>h4&fvFTYQqy?Q2Aqn>NVk<1_94`nGokDCR)R1c_5O?5Ep+@p5vx%)xQ+3(d#MTzEkkeCMnNY-bZzHCt zgHd5^%<+s;(g3$g5Mi?%KM&T=umWHsAbPny#7fDf&wYVJ41q;#Fp@2GD;<^G+n$3( zseqIsn=Z>y*$uy=UK-)TM?+R5a)Vb3aT%jm#4i3Nt?N=n1G?K0OjpsN5?7SEF5r*0 z-Nj+Lwvgyisp>Y$&UVi>VlrZ`I<=8lbVYSLKGEb8rF4>ZQoxPwZ4z{Rwm=thVQD%( zsS4_)#LT;ID=7*y@Dg(lk9C;`%K}NKn6J z_S?4Nq0*3cPfTD>+0bndlWYnx#kAW@8lBZL+HEeK%W7MQ%h)>QX<03^-t83~2BzPEjhg%=2wd2q)+url8J?zugHW&ZX9vXC$Oo z4-qMO_TXbQ?IOk`qz-rj+=Z;h>XD2_t+@(_vrtwt6|x7CQU^E&rjGoUgBIdA?eqj& z$kb9aYBE8*k9&m< zjM%@lp3!cMS<|jBP4G5u=6P2YYFd`wgy&x)H62^oAMJTj`+a zS4MHW$1I*vvXIMU5kVYh<1I+N91F2<4r~O((Ec>c^yXftMo{y5-f)K5Dq`pAmfzEn zRnAlj)8#Cn3X90KaObKf6q`g`yyYU}$tnsy_*NXubZh2g$|!Xgds92e3WzB&EJI`? z#xlR+3iwm`jR+h}EPdB3NVHEerfEHH35mmZ?7Lzj*3!6LH;H{<)&)<18{OTA*k}J+ zh_Q;R02<&|FwX}aax3AKQgr$6Y)a`MMtU1c3wR-iWD)r^TvgDe?tJCl+Gw1Bt9!Rk ztgE2_c(f7~q3r64h>Y%6mGVPe=mF4a?>y#tmf&nSmpr5m0HLnrqnW+HgRCxW1Vpb) zAtE1RBQ9fnlAq0e&UslbMS?>WT>78F4V?*P@d^APTErOai-MFlu_y`~FAf_PCwvdIa$w~#MYAyyLA?Y;) zZa#WGavmjhz@yv-nT}cOOol8NTR5R&j)KPxSX+qkx`@r8 zoQ|By{SfZR5fU9g@~iDyh^Y#%9U+U@yGX)A0C&`iX0b{aJQ}W-)nFce3o#a3ZGkRi zC7nEOq-rhTj#QVzNF`u;d2q~pcM&61Yg@oeEo7@$6~9_5Smy<=aYCbj*CB=ufQtZI zX9&aOa^4A6jP6|#kJ!rCtXrISY9k_;dgdz=L$yV?OYkDq-6R2@KIr&EGpX4#PI9Ic!pEJJnsK|Bhxi1^QWv*~O) zsTMxDWF)qDNW$J?kQH)=1qzo=EOP`dxC?n39=&hbfCj`=h=^E?2SDK@)&Y)!=@nL?rgiDC%9fahzp4aBTLCq} z8)hM>4Z7M6VJdpOT`dNgbVNi(_u6O>8-k6vjIC=jr?D8ZcWSFpm^7(E;G_L^o6{=n z>#GLluqfWu`|Ij(a9tW~J!O3ZfB*Z1!MymAz>hXfRj+o(JNTSke8xyt=Y>1mm-V4q zU4z`4eQh?B)pflG2eNDG{r#uQLsM_o)p~WfzIs`QUGaMzmG~>*T48L>PhI?Y*qjdqwy2H%8Jz${GGo?I!Sle`OSV;y)L)gpZkW?Z&B{b zBbJ(E_I)fkBkPgmZ(|@>rm(R|Pi;h7w~*qNyQRS3VuwEIq;(1CvKByzmfY6@j9;H^ z0wCGCgY01)Zt8Nqfk|IA@K9Y&t;(;}=H|A6W!b_vFF%&sV|Dpey)W0-u%mzb2=xl| z99*$!YcX2pf7s$MX%~HQi=$$!4YTu3i5xvq9VB;p*EDiF;pl5v_#gIfs*m6#A-sFF zUcWA{Z>v97pWFv;IuC>U9?pKe$JxH!OF&qDb@>=+_dZB^_f#Llv)pDzC5n`H8yqqgi64`=G2yVg5mz zC`u6ROy#@%uLba{bPN@%SV%P7y4$OvId`5O-?W3AwO2wL`c zSGUIxA0V7*eR4G#dUL6bPK9)zn$c{ozzh>HF4l@%u!RxgNT)QDxnvLzYofC}*E(R5 zS#I5G@@?7NR`pL5%=hU_R}(UDmZXI$rJE|Ba^E~R?UYFdX{+-zvf&|uCfPuK+(KoF zX-^9x!YQNo4RCry@Fn&4`I%$E;M+Agc}dMtcA-EE|Jj8vJxxdr6aiYGlDXBfsR3#( zodRNBapW1%V9S%JIibGgS>n{-dWJVV9(^VX(tCDk7~4TkwXK>Qi?3){d`11Y z-m^-1=IzZGUuL%Q?7T;Q2Y+0GYQXisuYPNa&SzIg|42bUZ=YIuQ%xsMU>V`&MhZqG zr$iFz$30DKzRQ^=8VHWws>{CThRjk z8zB>9Nw9NWrI5 zhv%@fM$h~`D?#mdjiU3RNwdecZs)wZU*qG|>F19g>$UmG*AGAW8*B~cNLF*WygP0S z;6|=l&zVm9XnjZJu{zGUx2YIo91s5Bh46XqYKoK+BHD81&W@OEG~A(Z<^DbE@(XBbL3IO)H>w8r#+f-(+-FqSmwU~4zpA7D1z?Xg&Wb39b*hg7yZRZNEOvoaI8&m#`z z-lxOqs6wy`+A2?tMTz6{0O4J@4zK}O3lSl>M*L2J_?;eis)kGxF-*!Wg6?;C3J7p7 z3HGP*mNpi@2{_7CyxDyGe*b;x_S+3bHaMlumN2-f&A=DgDDy|L?irR2^7B#NPf!}n znPFj(i!<-{X8ltK=KS>{{u%fswh5ydou09D1j8u6dK5_W01E$*k8_>0;M}hqBERvl zSD9V2f72p4pRp{QT05tc(V#&Y4XS83V^Ai`;@@h*$NL9anaiMP)9y7RBR3TJ z#xUKMYJC|wmO3M6YhYKZ9(zi`v-NeAdb|H;F@)v0cS7DR^ee4_2xmECrZ-IIIidcO zZVqiSFUFq_yc(eeD!I3d1ja9EqA|*zhP*MxpAWsb!b0U{Yw~OqL9OBRfIxKadGkV! z$)!w7&RZlh;1ZK`?xU?*#2Wvz1WD{0K>d5X(v96{MtXu*r_PwpcN!H$uL{Ev^bBd# z&gr0oFsV}316tYzgwTUQn8>{|5YM{6KyrrC1)uKL-qEwAj?M=QzSF5W+N%)NLr)n( zA!@%h3K1}yY~`tj`}BpYjvTTwj5^XP3L@0;t}bWZWbCXQv)_m1kTDWQ#(Z;qlv~}t z*7?p(cP}`*;?4f`{_ei_ihd#nJD?-@f|icfG|H*wTo6SCEGyGlp3#o3ed9yvYIUMc>iYNWA4T!a7B-8X)uK#vYlA`)l`-%u z5a&rHvYd#H+Px#g>FqJcw%%r!u0B-F^=*$sumi@Qj~_n31GnAWZ5sG3^89qt)5V>c zfEnjn!gHFD&ZzpvjTjWwR%r3v?$DIR=Qh>)b$Q<$>uUXU3ce`bA9tpX@zuVr_rF$a zbIY1L9QO`CpZ;)5F-Ej|wgENUr?0K{c=s814w^}-v;vNxB_mu31``-(dKu+7NAxfX zVn(sAtB+OvzPhi<(I&`~URq?7<*)6oTx$A+tHMDzdC%{T|2rug?XRCbxRj37W8+~(m5B9%t3>ugPXXg zgU-7rZPK}C(Awuw^Kuw&^FlI*VF#Tt90ut=Yw0`&qv|B0lKcx^IPiq7kXxq` z!%= 230 + + +def test_tf2_profiler_by_time(tf2_profiler_config_parser_by_time, out_dir): + """ + This test executes a TF2 training script, enables detailed TF profiling by time, and + verifies the number of events. + """ + assert tf2_profiler_config_parser_by_time.profiling_enabled + + hook = Hook(out_dir=out_dir) + helper_keras_fit(trial_dir=out_dir, hook=hook, eager=True, steps=["train", "eval", "predict"]) + hook.close() + + # get tensorboard timeline files + files = [] + for path in Path(tf2_profiler_config_parser_by_time.config.local_path + "/framework").rglob( + f"*{TENSORBOARDTIMELINE_SUFFIX}" + ): + files.append(path) + + assert len(files) == 1 + + trace_file = str(files[0]) + t_events = TensorboardProfilerEvents() + + t_events.read_events_from_file(trace_file) + + all_trace_events = t_events.get_all_events() + num_trace_events = len(all_trace_events) + + print(f"Number of events read = {num_trace_events}") + + # The number of events is varying by a small number on + # consecutive runs. Hence, the approximation in the below asserts. + assert num_trace_events >= 700 diff --git a/tests/profiler/test_smtfprofiler_events.py b/tests/profiler/test_smtfprofiler_events.py deleted file mode 100644 index 51d59299cc..0000000000 --- a/tests/profiler/test_smtfprofiler_events.py +++ /dev/null @@ -1,32 +0,0 @@ -# First Party -from smdebug.profiler import SMTFProfilerEvents - - -def test_smtfprofiler_events(trace_file="./tests/profiler/smtf_profiler_trace.json"): - trace_json_file = trace_file - print(f"Reading the trace file {trace_json_file}") - t_events = SMTFProfilerEvents(trace_json_file) - - all_trace_events = t_events.get_all_events() - num_trace_events = len(all_trace_events) - - print(f"Number of events read = {num_trace_events}") - assert num_trace_events == 49 - - event_list = t_events.get_events_at(1589314018458800000) # nanoseconds - print(f"Number of events at 15013686 are {len(event_list)}") - assert len(event_list) == 1 - - completed_event_list = t_events.get_events_within_range(0, 1589314018470000000) # nanoseconds - print(f"Number of events occurred between 0 and 15013686 are {len(completed_event_list)}") - assert len(completed_event_list) == 34 - - start_time_sorted = t_events.get_events_start_time_sorted() - start_time_for_first_event = start_time_sorted[0].start_time - print(f"The first event started at {start_time_for_first_event}") - assert start_time_for_first_event == 1589314018458743000 - - end_time_sorted = t_events.get_events_end_time_sorted() - end_time_for_last_event = end_time_sorted[-1].end_time - print(f"The first event started at {end_time_for_last_event}") - assert end_time_for_last_event == 1589314018481947000 diff --git a/tests/profiler/test_tfprofiler_events.py b/tests/profiler/test_tfprofiler_events.py deleted file mode 100644 index 8801fa0a02..0000000000 --- a/tests/profiler/test_tfprofiler_events.py +++ /dev/null @@ -1,36 +0,0 @@ -# First Party -from smdebug.profiler import TFProfilerEvents - - -def test_tfprofiler_events(trace_file="./tests/profiler/ip-172-31-19-241.trace.json"): - trace_json_file = trace_file - print(f"Reading the trace file {trace_json_file}") - t_events = TFProfilerEvents(trace_json_file) - - all_trace_events = t_events.get_all_events() - num_trace_events = len(all_trace_events) - - print(f"Number of events read = {num_trace_events}") - assert num_trace_events == 256 - - event_list = t_events.get_events_at(15013686) # nanoseconds - print(f"Number of events at 15013686 are {len(event_list)}") - assert len(event_list) == 3 - - completed_event_list = t_events.get_events_within_range(0, 15013686) # nanoseconds - print(f"Number of events occurred between 0 and 15013686 are {len(completed_event_list)}") - assert len(completed_event_list) == 253 - - start_time_sorted = t_events.get_events_start_time_sorted() - start_time_for_first_event = start_time_sorted[0].start_time - print(f"The first event started at {start_time_for_first_event}") - assert start_time_for_first_event == 116457.0 - - end_time_sorted = t_events.get_events_end_time_sorted() - end_time_for_last_event = end_time_sorted[-1].end_time - print(f"The first event started at {end_time_for_last_event}") - assert end_time_for_last_event == 64045679.0 - - processes = t_events.get_processes() - print(f"Number of processes = {len(processes)}") - assert len(processes) == 9 diff --git a/tests/pytorch/test_distributed_training.py b/tests/pytorch/test_distributed_training.py index 031cda7914..a5051c69e8 100644 --- a/tests/pytorch/test_distributed_training.py +++ b/tests/pytorch/test_distributed_training.py @@ -7,8 +7,11 @@ torch.distributed.get_rank() - when manually spawning processes """ # Standard Library +import json import os import shutil +import time +from pathlib import Path # Third Party import numpy as nn @@ -23,10 +26,9 @@ # First Party import smdebug.pytorch as smd +from smdebug.profiler.profiler_constants import DEFAULT_PREFIX, HOROVODTIMELINE_SUFFIX from smdebug.trials import create_trial -out_dir = "/tmp/run" - class Net(nn.Module): """Returns f(x) = sigmoid(w*x + b)""" @@ -63,7 +65,16 @@ def train(model, device, optimizer, num_steps=10): optimizer.step() -def run(rank, size, include_workers="one", num_epochs=10, batch_size=128, num_batches=10): +def run( + out_dir, + rank, + size, + include_workers="one", + test_timeline=False, + num_epochs=10, + batch_size=128, + num_batches=10, +): """Distributed function to be implemented later.""" torch.manual_seed(1234) device = torch.device("cpu") @@ -83,6 +94,16 @@ def run(rank, size, include_workers="one", num_epochs=10, batch_size=128, num_ba for epoch in range(num_epochs): epoch_loss = 0.0 + start_time = time.time() + if test_timeline: + hook.record_trace_events( + training_phase="Training", + op_name="TrainingEpochStart", + phase="B", + timestamp=start_time, + rank=rank, + epoch=epoch, + ) for _ in range(num_batches): optimizer.zero_grad() data, target = dataset(batch_size) @@ -92,6 +113,17 @@ def run(rank, size, include_workers="one", num_epochs=10, batch_size=128, num_ba loss.backward() average_gradients(model) optimizer.step() + end_time = time.time() + if test_timeline: + hook.record_trace_events( + training_phase="Training", + op_name="TrainingEpochEnd", + phase="E", + timestamp=end_time, + rank=rank, + duration=end_time - start_time, + epoch=epoch, + ) # print(f"Rank {dist.get_rank()}, epoch {epoch}: {epoch_loss / num_batches}") assert hook._get_worker_name() == f"worker_{dist.get_rank()}" @@ -111,15 +143,15 @@ def average_gradients(model): param.grad.data /= size -def init_processes(rank, size, include_workers, fn, backend="gloo"): +def init_processes(out_dir, rank, size, include_workers, test_timeline, fn, backend="gloo"): """Initialize the distributed environment.""" os.environ["MASTER_ADDR"] = "127.0.0.1" os.environ["MASTER_PORT"] = "29500" dist.init_process_group(backend, rank=rank, world_size=size) - fn(rank, size, include_workers) + fn(out_dir, rank, size, include_workers, test_timeline) -def _run_net_distributed(include_workers="one"): +def _run_net_distributed(out_dir, include_workers="one", test_timeline=False): """Runs a single linear layer on 2 processes.""" # torch.distributed is empty on Mac on Torch <= 1.2 if not hasattr(dist, "is_initialized"): @@ -128,7 +160,9 @@ def _run_net_distributed(include_workers="one"): size = 2 processes = [] for rank in range(size): - p = Process(target=init_processes, args=(rank, size, include_workers, run)) + p = Process( + target=init_processes, args=(out_dir, rank, size, include_workers, test_timeline, run) + ) p.start() processes.append(p) @@ -139,13 +173,12 @@ def _run_net_distributed(include_workers="one"): # https://stackoverflow.com/questions/13400546/py-test-how-to-automatically-detect-an-exception-in-a-child-process assert all([not p.exitcode for p in processes]), f"Some processes failed. processes={processes}" - out_dir = "/tmp/run" trial = create_trial(path=out_dir) return trial @pytest.mark.slow # 0:05 to run -def test_run_net_single_process(): +def test_run_net_single_process(out_dir): """Runs a single linear layer.""" device = torch.device("cpu") model = Net().to(device) @@ -168,14 +201,45 @@ def test_run_net_single_process(): @pytest.mark.slow # 0:07 to run -def test_run_net_distributed_save_all_workers(): - trial = _run_net_distributed(include_workers="all") +def test_run_net_distributed_save_all_workers(out_dir): + trial = _run_net_distributed(out_dir, include_workers="all") assert len(trial.workers()) == 2, f"trial.workers() = {trial.workers()}" assert len(trial.steps()) == 3, f"trial.steps() = {trial.steps()}" @pytest.mark.slow # 0:07 to run -def test_run_net_distributed_save_one_worker(): - trial = _run_net_distributed(include_workers="one") +def test_run_net_distributed_save_one_worker(out_dir): + trial = _run_net_distributed(out_dir, include_workers="one") assert len(trial.workers()) == 1, f"trial.workers() = {trial.workers()}" assert len(trial.steps()) == 3, f"trial.steps() = {trial.steps()}" + + +@pytest.mark.slow +def test_run_net_distributed_save_all_test_timeline(set_up_smprofiler_config_path, out_dir): + """ + This test checks if any of the timestamps recorded are negative + """ + trial = _run_net_distributed(out_dir, include_workers="all", test_timeline=True) + assert len(trial.workers()) == 2, f"trial.workers() = {trial.workers()}" + assert len(trial.steps()) == 3, f"trial.steps() = {trial.steps()}" + + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob("*.json"): + files.append(path) + + assert len(files) >= 2 + + for file_name in files: + with open(file_name) as timeline_file: + events_dict = json.load(timeline_file) + for e in events_dict: + if e["name"].startswith("event"): + assert int(e["ts"]) >= 0 + + # ensure that no horovod timeline files are written to when horovod is + # not used + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob(f"*{HOROVODTIMELINE_SUFFIX}"): + files.append(path) + + assert not files diff --git a/tests/tensorflow2/test_keras.py b/tests/tensorflow2/test_keras.py index 0ea2159f80..443bf5007e 100644 --- a/tests/tensorflow2/test_keras.py +++ b/tests/tensorflow2/test_keras.py @@ -7,8 +7,10 @@ `python tests/tensorflow2/test_keras.py` from the main directory. """ # Standard Library +import json import re import time +from pathlib import Path # Third Party import pytest @@ -27,6 +29,7 @@ from smdebug.core.modes import ModeKeys from smdebug.core.reduction_config import ALLOWED_NORMS, ALLOWED_REDUCTIONS from smdebug.exceptions import TensorUnavailableForStep +from smdebug.profiler.profiler_constants import DEFAULT_PREFIX from smdebug.tensorflow import ReductionConfig, SaveConfig @@ -879,3 +882,21 @@ def test_save_layer_inputs_and_outputs(out_dir, tf_eager_mode): "dense_1/inputs" ).value(0) assert boolean_matrix.all() + + +def test_hook_timeline_file_write( + set_up_smprofiler_config_path, set_up_resource_config, out_dir, tf_eager_mode +): + hook = smd.KerasHook(out_dir=out_dir, save_all=False) + helper_keras_fit(trial_dir=out_dir, hook=hook, eager=tf_eager_mode, steps=["train", "eval"]) + + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob("*.json"): + files.append(path) + + assert len(files) == 1 + + with open(files[0]) as timeline_file: + events_dict = json.load(timeline_file) + + assert events_dict diff --git a/tests/zero_code_change/horovod_tests/pytorch/test_hvd.py b/tests/zero_code_change/horovod_tests/pytorch/test_hvd.py index eda445cfa0..83dddbe7c8 100644 --- a/tests/zero_code_change/horovod_tests/pytorch/test_hvd.py +++ b/tests/zero_code_change/horovod_tests/pytorch/test_hvd.py @@ -1,12 +1,20 @@ # Standard Library +import json +import os +from collections import defaultdict +from pathlib import Path + # Third Party +import pytest from tests.zero_code_change.horovod_tests.constants import HOROVOD_PYTORCH_TEST_MNIST_SCRIPT from tests.zero_code_change.horovod_tests.utils import launch_horovod_job from tests.zero_code_change.utils import build_json from torch.cuda import device_count # First Party +from smdebug.profiler.profiler_config_parser import ProfilerConfigParser +from smdebug.profiler.profiler_constants import DEFAULT_PREFIX, HOROVODTIMELINE_SUFFIX from smdebug.trials import create_trial """ @@ -99,3 +107,139 @@ def test_gpu_allworkers_saveall(out_dir): def test_cpu_allworkers_saveall(out_dir): mode_allworkers_saveall(out_dir, "cpu") + + +""" +HVD event file rotation tests +""" + + +@pytest.fixture +def user_disabled_profiler_config_parser(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "user_disabled_profile_config_parser.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + +@pytest.fixture +def hvd_rotation_profiler_config_parser(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "hvd_rotation_profiler_config_parser.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + +@pytest.mark.parametrize("mode", ["cpu", "gpu"]) +@pytest.mark.parametrize("worker_function", [mode_one_worker, mode_allworkers]) +def test_mode_workers_event_file_rotation( + out_dir, monkeypatch, hvd_rotation_profiler_config_parser, mode, worker_function +): + """ + This test is meant to verify the working of the Horovod trace file reader with + Horovod and PyTorch. + :param hvd_rotation_profiler_config_parser: Profiler Config Parser + :param mode: cpu or gpu + :param worker_function: one worker or all workers + """ + # check if Profiler config has been parsed and is enabled + assert hvd_rotation_profiler_config_parser.profiling_enabled + + # enable Horovod timeline + hvd_file = out_dir + "/hvd_timeline.json" + monkeypatch.setenv("HOROVOD_TIMELINE", hvd_file) + + # start the training job + worker_function(out_dir, mode) + + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob(f"*{HOROVODTIMELINE_SUFFIX}"): + files.append(path) + + # check if files have been split + assert files + + # after the training completes, read the Horovod timeline file and make note of all + # events. This will be used to verify reader functionality. + json_dict = [] + with open(hvd_file) as json_data: + for line in json_data: + try: + event = json.loads(line[:-2]) if line.endswith(",\n") else json.loads(line[:-1]) + json_dict.append(event) + except Exception as e: + # if JSON string is invalid, skip + pass + + # populate the horovod event dictionary + # key = phase, value = list of events of that particular phase + hvd_event_dict = defaultdict(list) + for event in json_dict: + hvd_event_dict[event["ph"]].append(event) + + # populate the events that were written by the timeline file writer + # in the rotated files. These events are the ones that the trace file reader + # thread read from the large Horovod timeline file. + rotated_event_dict = defaultdict(list) + for file_name in files: + with open(file_name) as timeline_file: + events_dict = json.load(timeline_file) + for idx, e in enumerate(events_dict): + if idx < 2: + # skip the 1st 2 events of each file as they are + # metadata filled by the timeline file writer + continue + rotated_event_dict[e["ph"]].append(e) + + # check if all timestamps are positive + if "ts" in e: + assert int(e["ts"]) >= 0 + + # check that the rotated files have the same number of event types + # as the original horovod timeline file + assert len(rotated_event_dict) == len(hvd_event_dict) + + # for every type of event, check that the rotated files have at least + # the same number of events of that type in the hvd file. + # We check for the condition >= because metadata events in rotated files + # could be more in number as metadata could be repeated in multiple files + # based on the time of rotation. + for key in hvd_event_dict: + assert len(rotated_event_dict[key]) >= len(hvd_event_dict[key]) + + +def test_event_file_rotation_profiler_disabled( + user_disabled_profiler_config_parser, out_dir, monkeypatch +): + """ + Test that timeline file rotation is disabled when profiler is disabled + """ + assert not user_disabled_profiler_config_parser.profiling_enabled + + hvd_file = out_dir + "/hvd_timeline.json" + monkeypatch.setenv("HOROVOD_TIMELINE", hvd_file) + + # start training + mode_one_worker(out_dir, "gpu") + + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob(f"*{HOROVODTIMELINE_SUFFIX}"): + files.append(path) + + # ensure that no horovod_timeline files have been generated + assert not files + + +def test_event_file_rotation_hvd_timeline_disabled(simple_profiler_config_parser, out_dir): + """ + Test that timeline file rotation is disabled when HOROVOD_TIMELINE file is not written to + """ + assert simple_profiler_config_parser.profiling_enabled + + # start training + mode_one_worker(out_dir, "gpu") + + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob(f"*{HOROVODTIMELINE_SUFFIX}"): + files.append(path) + + # ensure that no horovod_timeline files have been generated + assert not files diff --git a/tests/zero_code_change/horovod_tests/tensorflow2/test_keras_fit.py b/tests/zero_code_change/horovod_tests/tensorflow2/test_keras_fit.py index e61ee029c1..4a8318ab00 100644 --- a/tests/zero_code_change/horovod_tests/tensorflow2/test_keras_fit.py +++ b/tests/zero_code_change/horovod_tests/tensorflow2/test_keras_fit.py @@ -1,5 +1,13 @@ # First Party + +# Standard Library +import json +import os +from collections import defaultdict +from pathlib import Path + # Third Party +import pytest from tests.tensorflow2.utils import is_tf_2_2 from tests.zero_code_change.horovod_tests.constants import ( HOROVOD_KERAS_TEST_SCRIPT_ARGS, @@ -9,6 +17,8 @@ from tests.zero_code_change.horovod_tests.utils import launch_horovod_job from tests.zero_code_change.utils import build_json +from smdebug.profiler.profiler_config_parser import ProfilerConfigParser +from smdebug.profiler.profiler_constants import DEFAULT_PREFIX, HOROVODTIMELINE_SUFFIX from smdebug.trials import create_trial @@ -103,3 +113,93 @@ def test_gpu_allworkers_saveall(out_dir): def test_cpu_allworkers_saveall(out_dir): mode_allworkers_saveall(out_dir, "cpu") + + +""" +HVD event file rotation tests +""" + + +@pytest.fixture +def hvd_rotation_profiler_config_parser(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "hvd_rotation_profiler_config_parser.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + return ProfilerConfigParser() + + +@pytest.mark.parametrize("mode", ["cpu", "gpu"]) +@pytest.mark.parametrize("worker_function", [basic_test, mode_allworkers]) +def test_mode_workers_event_file_rotation( + out_dir, monkeypatch, hvd_rotation_profiler_config_parser, mode, worker_function +): + """ + This test is meant to verify the working of the Horovod trace file reader with + Horovod and Tensorflow 2.x. + :param hvd_rotation_profiler_config_parser: Profiler Config Parser + :param mode: cpu or gpu + :param worker_function: basic test (one worker) or all workers + """ + # check if Profiler config has been parsed and is enabled + assert hvd_rotation_profiler_config_parser.profiling_enabled + + # enable Horovod timeline + hvd_file = out_dir + "/hvd_timeline.json" + monkeypatch.setenv("HOROVOD_TIMELINE", hvd_file) + + # start the training job + worker_function(out_dir, mode) + + # check if files have been split + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob(f"*{HOROVODTIMELINE_SUFFIX}"): + files.append(path) + + assert files + + # after the training completes, read the Horovod timeline file and make note of all + # events. This will be used to verify reader functionality. + json_dict = [] + with open(hvd_file) as json_data: + for line in json_data: + try: + event = json.loads(line[:-2]) if line.endswith(",\n") else json.loads(line[:-1]) + json_dict.append(event) + except Exception as e: + # if JSON string is invalid, skip + pass + + # populate the horovod event dictionary + # key = phase, value = list of events of that particular phase + hvd_event_dict = defaultdict(list) + for event in json_dict: + hvd_event_dict[event["ph"]].append(event) + + # populate the events that were written by the timeline file writer + # in the rotated files. These events are the ones that the trace file reader + # thread read from the large Horovod timeline file. + rotated_event_dict = defaultdict(list) + for file_name in files: + with open(file_name) as timeline_file: + events_dict = json.load(timeline_file) + for idx, e in enumerate(events_dict): + if idx < 2: + # skip the 1st 2 events of each file as they are + # metadata filled by the timeline file writer + continue + rotated_event_dict[e["ph"]].append(e) + + # check if all timestamps are positive + if "ts" in e: + assert int(e["ts"]) >= 0 + + # check that the rotated files have the same number of event types + # as the original horovod timeline file + assert len(rotated_event_dict) == len(hvd_event_dict) + + # for every type of event, check that the rotated files have at least + # the same number of events of that type in the hvd file. + # We check for the condition >= because metadata events in rotated files + # could be more in number as metadata could be repeated in multiple files + # based on the time of rotation. + for key in hvd_event_dict: + assert len(rotated_event_dict[key]) >= len(hvd_event_dict[key]) diff --git a/tests/zero_code_change/smdataparallel_tests/__init__.py b/tests/zero_code_change/smdataparallel_tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/zero_code_change/smdataparallel_tests/constants.py b/tests/zero_code_change/smdataparallel_tests/constants.py new file mode 100644 index 0000000000..93bf07116a --- /dev/null +++ b/tests/zero_code_change/smdataparallel_tests/constants.py @@ -0,0 +1,6 @@ +SMDATAPARALLEL_PYTORCH_TEST_MNIST_SCRIPT = ( + "examples/pytorch/zero_code_change_examples/smdataparallel_mnist.py" +) +SMDATAPARALLEL_PYTORCH_TEST_MNIST_ARGS = ["--epochs", "1"] + +SMDATAPARALLEL_TF2_TEST_MNIST_SCRIPT = "examples/tensorflow2/scripts/smdataparallel_mnist_tf2.py" diff --git a/tests/zero_code_change/smdataparallel_tests/pytorch/__init__.py b/tests/zero_code_change/smdataparallel_tests/pytorch/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/zero_code_change/smdataparallel_tests/pytorch/test_smdataparallel.py b/tests/zero_code_change/smdataparallel_tests/pytorch/test_smdataparallel.py new file mode 100644 index 0000000000..f950d02365 --- /dev/null +++ b/tests/zero_code_change/smdataparallel_tests/pytorch/test_smdataparallel.py @@ -0,0 +1,148 @@ +# Standard Library + +import json +import os +from pathlib import Path + +# Third Party +import pytest +from tests.zero_code_change.smdataparallel_tests.constants import ( + SMDATAPARALLEL_PYTORCH_TEST_MNIST_ARGS, + SMDATAPARALLEL_PYTORCH_TEST_MNIST_SCRIPT, +) +from tests.zero_code_change.smdataparallel_tests.utils import launch_smdataparallel_job +from tests.zero_code_change.utils import build_json +from torch.cuda import device_count + +# First Party +from smdebug.profiler.profiler_config_parser import ProfilerConfigParser +from smdebug.profiler.profiler_constants import DEFAULT_PREFIX, SMDATAPARALLELTIMELINE_SUFFIX +from smdebug.profiler.tf_profiler_parser import SMDataParallelProfilerEvents +from smdebug.trials import create_trial + + +""" +Tested on current DLAMI p3.16xlarge when run from the main directory +""" + + +def mode_allworkers(out_dir, mode): + path = build_json(out_dir, include_workers="all", include_collections=["weights", "gradients"]) + print("build_json_path: ", path) + num_workers = 1 if bool(device_count()) is False else device_count() + mode_args = list(SMDATAPARALLEL_PYTORCH_TEST_MNIST_ARGS) + launch_smdataparallel_job( + script_file_path=SMDATAPARALLEL_PYTORCH_TEST_MNIST_SCRIPT, + script_args=mode_args, + num_workers=num_workers, + config_file_path=path, + mode=mode, + ) + tr = create_trial(out_dir) + assert len(tr.workers()) == num_workers + assert len(tr.tensor_names()) == 13 + assert len(tr.tensor(tr.tensor_names(collection="weights")[0]).workers(0)) == num_workers + + +@pytest.mark.skip( + reason="Requires SMDataParallel docker image which is private as of now. It would be available in general DLC sometime in mid of November 2020" +) +def test_gpu_allworkers(out_dir): + mode_allworkers(out_dir, "gpu") + + +@pytest.fixture +def smdataparallel_profiler_config_path(config_folder, monkeypatch): + config_path = os.path.join(config_folder, "smdataparallel_profiler_config.json") + monkeypatch.setenv("SMPROFILER_CONFIG_PATH", config_path) + yield config_path + if os.path.isfile(config_path): + os.remove(config_path) + + +@pytest.mark.skip( + reason="Requires SMDataParallel docker image which is private as of now. It would be available in general DLC sometime in mid of November 2020" +) +@pytest.mark.parametrize("mode", ["gpu"]) +@pytest.mark.parametrize("worker_function", [mode_allworkers]) +def test_mode_workers_dynamic_smdataparallel_profiler( + out_dir, smdataparallel_profiler_config_path, mode, worker_function +): + """ + This test is meant to verify dynamically turning ON/OFF SMDataParallel profiler with PyTorch. + :param mode: gpu + :param worker_function: basic test all workers + """ + + def _convert_to_string(item): + return '"{0}"'.format(item) if isinstance(item, str) else item + + def _convert_key_and_value(key, value): + return "{0}: {1}, ".format(_convert_to_string(key), _convert_to_string(value)) + + smdataparallel_profiler_config = "{" + smdataparallel_profiler_config += _convert_key_and_value("StartStep", 2) + smdataparallel_profiler_config += _convert_key_and_value("NumSteps", 1) + smdataparallel_profiler_config += "}" + + full_config = { + "ProfilingParameters": { + "ProfilerEnabled": True, + "SMDataparallelProfilingConfig": smdataparallel_profiler_config, + "localpath": out_dir, + } + } + + with open(smdataparallel_profiler_config_path, "w") as f: + json.dump(full_config, f) + + profiler_config_parser = ProfilerConfigParser() + assert profiler_config_parser.profiling_enabled + + # start the training job + worker_function(out_dir, mode) + + files = [] + for path in Path(out_dir + "/" + DEFAULT_PREFIX).rglob(f"*{SMDATAPARALLELTIMELINE_SUFFIX}"): + files.append(path) + + assert len(files) == 8 + + trace_file = str(files[0]) + t_events = SMDataParallelProfilerEvents() + + t_events.read_events_from_file(trace_file) + + all_trace_events = t_events.get_all_events() + num_trace_events = len(all_trace_events) + + print(f"Number of events read = {num_trace_events}") + + # The number of events is varying by a small number on + # consecutive runs. Hence, the approximation in the below asserts. + assert num_trace_events >= 8 + + +def mode_allworkers_saveall(out_dir, mode): + path = build_json(out_dir, include_workers="all", save_all=True) + num_workers = 1 if bool(device_count()) is False else device_count() + mode_args = list(SMDATAPARALLEL_PYTORCH_TEST_MNIST_ARGS) + launch_smdataparallel_job( + script_file_path=SMDATAPARALLEL_PYTORCH_TEST_MNIST_SCRIPT, + script_args=mode_args, + num_workers=num_workers, + config_file_path=path, + mode=mode, + ) + tr = create_trial(out_dir) + assert len(tr.workers()) == num_workers + assert len(tr.tensor_names()) > 25 + assert len(tr.tensor(tr.tensor_names(collection="weights")[0]).workers(0)) == num_workers + assert len(tr.tensor(tr.tensor_names(collection="losses")[0]).workers(0)) == num_workers + + +@pytest.mark.skip( + reason="Requires SMDataParallel docker image which is private as of now. It would be available in general DLC sometime in mid of November 2020" +) +def test_gpu_allworkers_saveall(out_dir): + mode_allworkers_saveall(out_dir, "gpu") diff --git a/tests/zero_code_change/smdataparallel_tests/tensorflow2/__init__.py b/tests/zero_code_change/smdataparallel_tests/tensorflow2/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/zero_code_change/smdataparallel_tests/tensorflow2/test_tf2_smdataparallel.py b/tests/zero_code_change/smdataparallel_tests/tensorflow2/test_tf2_smdataparallel.py new file mode 100644 index 0000000000..f98d06108f --- /dev/null +++ b/tests/zero_code_change/smdataparallel_tests/tensorflow2/test_tf2_smdataparallel.py @@ -0,0 +1,135 @@ +# Third Party +import pytest +import tensorflow.compat.v2 as tf +from tests.zero_code_change.smdataparallel_tests.constants import ( + SMDATAPARALLEL_TF2_TEST_MNIST_SCRIPT, +) +from tests.zero_code_change.smdataparallel_tests.utils import launch_smdataparallel_job +from tests.zero_code_change.tf_utils import get_available_gpus +from tests.zero_code_change.utils import build_json + +# First Party +from smdebug.tensorflow.constants import TF_DEFAULT_SAVED_COLLECTIONS +from smdebug.trials import create_trial + + +def basic_test(out_dir, mode): + path = build_json( + out_dir, include_workers="one", include_collections=["weights", "optimizer_variables"] + ) + num_workers = len(get_available_gpus()) + mode_args = ["--model_dir", out_dir] + launch_smdataparallel_job( + script_file_path=SMDATAPARALLEL_TF2_TEST_MNIST_SCRIPT, + script_args=mode_args, + num_workers=num_workers, + config_file_path=path, + mode=mode, + ) + + tr = create_trial(out_dir) + print(tr.tensor_names()) + assert len(tr.workers()) == 1 + assert len(tr.tensor_names()) == 5 + assert len(tr.tensor(tr.tensor_names(collection="weights")[0]).workers(0)) == 1 + + +@pytest.mark.skipif( + tf.__version__ < "2.3.0", + reason="smdistributed.dataparallel supports TF version 2.3.0 and above", +) +@pytest.mark.skip( + reason="Requires SMDataParallel docker image which is private as of now. It would be available in general DLC sometime in mid of November 2020" +) +def test_gpu(out_dir): + basic_test(out_dir, "gpu") + + +def mode_allworkers(out_dir, mode): + path = build_json( + out_dir, include_workers="all", include_collections=["weights", "optimizer_variables"] + ) + num_workers = len(get_available_gpus()) + mode_args = ["--model_dir", out_dir] + launch_smdataparallel_job( + script_file_path=SMDATAPARALLEL_TF2_TEST_MNIST_SCRIPT, + script_args=mode_args, + num_workers=num_workers, + config_file_path=path, + mode=mode, + ) + tr = create_trial(out_dir) + assert len(tr.workers()) == num_workers + print("tensor names: ", tr.tensor_names()) + assert len(tr.tensor_names()) == 5 + assert len(tr.tensor(tr.tensor_names(collection="weights")[0]).workers(0)) == num_workers + + +@pytest.mark.skipif( + tf.__version__ < "2.3.0", + reason="smdistributed.dataparallel supports TF version 2.3.0 and above", +) +@pytest.mark.skip( + reason="Requires SMDataParallel docker image which is private as of now. It would be available in general DLC sometime in mid of November 2020" +) +def test_gpu_allworkers(out_dir): + mode_allworkers(out_dir, "gpu") + + +def mode_allworkers_saveall(out_dir, mode): + path = build_json(out_dir, include_workers="all", save_all=True) + num_workers = len(get_available_gpus()) + mode_args = ["--model_dir", out_dir] + launch_smdataparallel_job( + script_file_path=SMDATAPARALLEL_TF2_TEST_MNIST_SCRIPT, + script_args=mode_args, + num_workers=num_workers, + config_file_path=path, + mode=mode, + ) + tr = create_trial(out_dir) + assert len(tr.workers()) == num_workers + assert len(tr.tensor_names()) == 35 + assert len(tr.tensor(tr.tensor_names(collection="weights")[0]).workers(0)) == num_workers + assert len(tr.tensor("loss").workers(0)) == num_workers + + +@pytest.mark.skipif( + tf.__version__ < "2.3.0", + reason="smdistributed.dataparallel supports TF version 2.3.0 and above", +) +@pytest.mark.skip( + reason="Requires SMDataParallel docker image which is private as of now. It would be available in general DLC sometime in mid of November 2020" +) +def test_gpu_allworkers_saveall(out_dir): + mode_allworkers_saveall(out_dir, "gpu") + + +def mode_allworkers_default_collections(out_dir, mode): + path = build_json( + out_dir, include_workers="all", include_collections=TF_DEFAULT_SAVED_COLLECTIONS + ) + num_workers = len(get_available_gpus()) + mode_args = ["--model_dir", out_dir] + launch_smdataparallel_job( + script_file_path=SMDATAPARALLEL_TF2_TEST_MNIST_SCRIPT, + script_args=mode_args, + num_workers=num_workers, + config_file_path=path, + mode=mode, + ) + tr = create_trial(out_dir) + assert len(tr.workers()) == num_workers + assert len(tr.tensor_names()) == 1 + assert len(tr.tensor(tr.tensor_names(collection="losses")[0]).workers(0)) == num_workers + + +@pytest.mark.skipif( + tf.__version__ < "2.3.0", + reason="smdistributed.dataparallel supports TF version 2.3.0 and above", +) +@pytest.mark.skip( + reason="Requires SMDataParallel docker image which is private as of now. It would be available in general DLC sometime in mid of November 2020" +) +def test_gpu_allworkers_default_collections(out_dir): + mode_allworkers_default_collections(out_dir, "gpu") diff --git a/tests/zero_code_change/smdataparallel_tests/utils.py b/tests/zero_code_change/smdataparallel_tests/utils.py new file mode 100644 index 0000000000..e436c6ed26 --- /dev/null +++ b/tests/zero_code_change/smdataparallel_tests/utils.py @@ -0,0 +1,12 @@ +# Standard Library +import os +import subprocess +import sys + + +def launch_smdataparallel_job(script_file_path, script_args, num_workers, config_file_path, mode): + command = ["smddpsinglenode"] + [sys.executable, script_file_path] + script_args + env_dict = os.environ.copy() + env_dict["SMDEBUG_CONFIG_FILE_PATH"] = f"{config_file_path}" + env_dict["PYTHONPATH"] = "/home/ubuntu/sagemaker-debugger/" + subprocess.check_call(command, env=env_dict) diff --git a/tests/zero_code_change/tf_utils.py b/tests/zero_code_change/tf_utils.py index 15b85aa6ec..b4ed6683da 100644 --- a/tests/zero_code_change/tf_utils.py +++ b/tests/zero_code_change/tf_utils.py @@ -5,6 +5,7 @@ import numpy as np import tensorflow.compat.v1 as tf import tensorflow_datasets as tfds +from tensorflow.python.client import device_lib tfds.disable_progress_bar() @@ -257,3 +258,8 @@ def get_keras_model_v1(): model = keras.Model(inputs=inputs, outputs=outputs, name="mnist_model") return model + + +def get_available_gpus(): + local_device_protos = device_lib.list_local_devices() + return [x.name for x in local_device_protos if x.device_type == "GPU"]