Skip to content

Commit

Permalink
Supporting changes to implement python pipeline for performance runs …
Browse files Browse the repository at this point in the history 
…on runtime repo (#4156)

* Make changes to python pipeline for runtime repo

* More fixes

* Clean up some parts

* Add latest changes from runtime repo

* Fix wasm bundle path and add arg support for local commit time

* Fix local commit time code

* Only set core_root for coreclr runs

* Fix ios hello world copying

* Fix wasm copying and prereq check on linux

* Fix wasm args quoting and PreparePayloadWorkItems usage

* Use new browser versions path

* Fix prerequisites script building

* Fix wasm args building

* Fix directory creation and prevent pip reinstalls

* Fix trailing space on wasm args

* More various fixes

* Fix file copy into root

* Fix FailOnTestFailure value

* Include recent changes from runtime repo

* Only build startup and SOD for runtime scenarios

* Fix infinite loop in zip creation

* Fix some directory issues

* Fix some copytree logic

* Copy ios apps into correct dir

* Fix iosSampleApp copying + add better logging

* Fix PDN download and error code

* Address PR comments

* Fix incorrect tfm condition check

* Address PR comments
  • Loading branch information
@caaavik-msft
caaavik-msft authored Sep 16, 2024
1 parent 297faee commit 254b737
Show file tree
Hide file tree
Showing 7 changed files with 491 additions and 287 deletions.
94 changes: 55 additions & 39 deletions eng/performance/helix.proj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,56 +11,72 @@
<PartitionCount Condition="'$(PartitionCount)' == ''">15</PartitionCount>
</PropertyGroup>
<ItemGroup>
<Partition Include="Partition0" Index="0" Condition="$(PartitionCount) &gt; 0" />
<Partition Include="Partition1" Index="1" Condition="$(PartitionCount) &gt; 1" />
<Partition Include="Partition2" Index="2" Condition="$(PartitionCount) &gt; 2" />
<Partition Include="Partition3" Index="3" Condition="$(PartitionCount) &gt; 3" />
<Partition Include="Partition4" Index="4" Condition="$(PartitionCount) &gt; 4" />
<Partition Include="Partition5" Index="5" Condition="$(PartitionCount) &gt; 5" />
<Partition Include="Partition6" Index="6" Condition="$(PartitionCount) &gt; 6" />
<Partition Include="Partition7" Index="7" Condition="$(PartitionCount) &gt; 7" />
<Partition Include="Partition8" Index="8" Condition="$(PartitionCount) &gt; 8" />
<Partition Include="Partition9" Index="9" Condition="$(PartitionCount) &gt; 9" />
<Partition Include="Partition10" Index="10" Condition="$(PartitionCount) &gt; 10" />
<Partition Include="Partition11" Index="11" Condition="$(PartitionCount) &gt; 11" />
<Partition Include="Partition12" Index="12" Condition="$(PartitionCount) &gt; 12" />
<Partition Include="Partition13" Index="13" Condition="$(PartitionCount) &gt; 13" />
<Partition Include="Partition14" Index="14" Condition="$(PartitionCount) &gt; 14" />
<Partition Include="Partition15" Index="15" Condition="$(PartitionCount) &gt; 15" />
<Partition Include="Partition16" Index="16" Condition="$(PartitionCount) &gt; 16" />
<Partition Include="Partition17" Index="17" Condition="$(PartitionCount) &gt; 17" />
<Partition Include="Partition18" Index="18" Condition="$(PartitionCount) &gt; 18" />
<Partition Include="Partition19" Index="19" Condition="$(PartitionCount) &gt; 19" />
<Partition Include="Partition20" Index="20" Condition="$(PartitionCount) &gt; 20" />
<Partition Include="Partition21" Index="21" Condition="$(PartitionCount) &gt; 21" />
<Partition Include="Partition22" Index="22" Condition="$(PartitionCount) &gt; 22" />
<Partition Include="Partition23" Index="23" Condition="$(PartitionCount) &gt; 23" />
<Partition Include="Partition24" Index="24" Condition="$(PartitionCount) &gt; 24" />
<Partition Include="Partition25" Index="25" Condition="$(PartitionCount) &gt; 25" />
<Partition Include="Partition26" Index="26" Condition="$(PartitionCount) &gt; 26" />
<Partition Include="Partition27" Index="27" Condition="$(PartitionCount) &gt; 27" />
<Partition Include="Partition28" Index="28" Condition="$(PartitionCount) &gt; 28" />
<Partition Include="Partition29" Index="29" Condition="$(PartitionCount) &gt; 29" />
<Partition Include="$(BuildConfig).Partition0" Index="0" Condition="$(PartitionCount) &gt; 0" />
</ItemGroup>

<!--
Partition the MicroBenchmarks project, but nothing else
TODO: Support Compare runs
This is achieved by doing a baseline corerun in a precommand, then running ResultsComparer in a postcommand
TODO: Check PERF_PREREQS_INSTALL_FAILED before running commands on non-windows
-->
<ItemGroup Condition="'$(OnlySanityCheck)' != 'true'">
<Partition Include="$(BuildConfig).Partition1" Index="1" Condition="$(PartitionCount) &gt; 1" />
<Partition Include="$(BuildConfig).Partition2" Index="2" Condition="$(PartitionCount) &gt; 2" />
<Partition Include="$(BuildConfig).Partition3" Index="3" Condition="$(PartitionCount) &gt; 3" />
<Partition Include="$(BuildConfig).Partition4" Index="4" Condition="$(PartitionCount) &gt; 4" />
<Partition Include="$(BuildConfig).Partition5" Index="5" Condition="$(PartitionCount) &gt; 5" />
<Partition Include="$(BuildConfig).Partition6" Index="6" Condition="$(PartitionCount) &gt; 6" />
<Partition Include="$(BuildConfig).Partition7" Index="7" Condition="$(PartitionCount) &gt; 7" />
<Partition Include="$(BuildConfig).Partition8" Index="8" Condition="$(PartitionCount) &gt; 8" />
<Partition Include="$(BuildConfig).Partition9" Index="9" Condition="$(PartitionCount) &gt; 9" />
<Partition Include="$(BuildConfig).Partition10" Index="10" Condition="$(PartitionCount) &gt; 10" />
<Partition Include="$(BuildConfig).Partition11" Index="11" Condition="$(PartitionCount) &gt; 11" />
<Partition Include="$(BuildConfig).Partition12" Index="12" Condition="$(PartitionCount) &gt; 12" />
<Partition Include="$(BuildConfig).Partition13" Index="13" Condition="$(PartitionCount) &gt; 13" />
<Partition Include="$(BuildConfig).Partition14" Index="14" Condition="$(PartitionCount) &gt; 14" />
<Partition Include="$(BuildConfig).Partition15" Index="15" Condition="$(PartitionCount) &gt; 15" />
<Partition Include="$(BuildConfig).Partition16" Index="16" Condition="$(PartitionCount) &gt; 16" />
<Partition Include="$(BuildConfig).Partition17" Index="17" Condition="$(PartitionCount) &gt; 17" />
<Partition Include="$(BuildConfig).Partition18" Index="18" Condition="$(PartitionCount) &gt; 18" />
<Partition Include="$(BuildConfig).Partition19" Index="19" Condition="$(PartitionCount) &gt; 19" />
<Partition Include="$(BuildConfig).Partition20" Index="20" Condition="$(PartitionCount) &gt; 20" />
<Partition Include="$(BuildConfig).Partition21" Index="21" Condition="$(PartitionCount) &gt; 21" />
<Partition Include="$(BuildConfig).Partition22" Index="22" Condition="$(PartitionCount) &gt; 22" />
<Partition Include="$(BuildConfig).Partition23" Index="23" Condition="$(PartitionCount) &gt; 23" />
<Partition Include="$(BuildConfig).Partition24" Index="24" Condition="$(PartitionCount) &gt; 24" />
<Partition Include="$(BuildConfig).Partition25" Index="25" Condition="$(PartitionCount) &gt; 25" />
<Partition Include="$(BuildConfig).Partition26" Index="26" Condition="$(PartitionCount) &gt; 26" />
<Partition Include="$(BuildConfig).Partition27" Index="27" Condition="$(PartitionCount) &gt; 27" />
<Partition Include="$(BuildConfig).Partition28" Index="28" Condition="$(PartitionCount) &gt; 28" />
<Partition Include="$(BuildConfig).Partition29" Index="29" Condition="$(PartitionCount) &gt; 29" />
</ItemGroup>

<!-- Partition the MicroBenchmarks project, but nothing else -->
<ItemGroup Condition="$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
<HelixWorkItem Include="@(Partition)">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
<Command>$(WorkItemCommand) --partition=%(HelixWorkItem.Index) --bdn-arguments="$(BenchmarkDotNetArguments) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</Command>
<PreCommands Condition="'$(Compare)' == 'true'">$(BaselineWorkItemCommand) --partition=%(HelixWorkItem.Index) --bdn-arguments="$(BaselineBenchmarkDotNetArguments) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</PreCommands>
<Command Condition="'$(TargetsWindows)' == 'true'">$(WorkItemCommand) --partition=%(HelixWorkItem.Index) --bdn-arguments="$(BenchmarkDotNetArguments) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</Command>
<Command Condition="'$(TargetsWindows)' != 'true'">
if [ "x$PERF_PREREQS_INSTALL_FAILED" = "x1" ]; then
echo "\n\n** Error: Failed to install prerequisites **\n\n"; (exit 1);
else
$(WorkItemCommand) --partition=%(HelixWorkItem.Index) --bdn-arguments="$(BenchmarkDotNetArguments) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)";
fi
</Command>
<PostCommands Condition="'$(Compare)' == 'true'">$(CompareCommand)</PostCommands>
<Timeout>$(WorkItemTimeout)</Timeout>
<DownloadFilesFromResults Condition="'$(DownloadFilesFromHelix)' == 'true'">Partition%(HelixWorkItem.Index)-combined-perf-lab-report.json</DownloadFilesFromResults>
</HelixWorkItem>
</ItemGroup>
<ItemGroup Condition="!$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
<HelixWorkItem Include="WorkItem">
<HelixWorkItem Include="$(BuildConfig).WorkItem">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
<Command>$(WorkItemCommand) --bdn-arguments="$(BenchmarkDotNetArguments)"</Command>
<PreCommands Condition="'$(Compare)' == 'true'">$(BaselineWorkItemCommand) --bdn-arguments="$(BaselineBenchmarkDotNetArguments)"</PreCommands>
<Command Condition="'$(TargetsWindows)' == 'true'">$(WorkItemCommand) --bdn-arguments="$(BenchmarkDotNetArguments)"</Command>
<Command Condition="'$(TargetsWindows)' != 'true'">
if [ "x$PERF_PREREQS_INSTALL_FAILED" = "x1" ]; then
echo "\n\n** Error: Failed to install prerequisites **\n\n"; (exit 1);
else
$(WorkItemCommand) --bdn-arguments="$(BenchmarkDotNetArguments)";
fi
</Command>
<PostCommands Condition="'$(Compare)' == 'true'">$(CompareCommand)</PostCommands>
<Timeout>$(WorkItemTimeout)</Timeout>
<DownloadFilesFromResults Condition="'$(DownloadFilesFromHelix)' == 'true'">combined-perf-lab-report.json</DownloadFilesFromResults>
</HelixWorkItem>
Expand Down
3 changes: 1 addition & 2 deletions scripts/ci_setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -440,8 +440,7 @@ def main(args: Any):
output_file += extension

dir_path = os.path.dirname(output_file)
if not os.path.isdir(dir_path):
os.mkdir(dir_path)
os.makedirs(dir_path, exist_ok=True)

if not framework.startswith('net4'):
target_framework_moniker = dotnet.FrameworkAction.get_target_framework_moniker(framework)
Expand Down
3 changes: 1 addition & 2 deletions scripts/dotnet.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -822,8 +822,7 @@ def install(

if not install_dir:
install_dir = __get_directory(architecture)
if not path.exists(install_dir):
makedirs(install_dir)
makedirs(install_dir, exist_ok=True)

getLogger().info("DotNet Install Path: '%s'", install_dir)

Expand Down
3 changes: 1 addition & 2 deletions scripts/performance/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,7 @@ def make_directory(path: str):
'''Creates a directory.'''
if not path:
raise TypeError('Undefined path.')
if not os.path.isdir(path):
os.makedirs(path)
os.makedirs(path, exist_ok=True)


def remove_directory(path: str) -> None:
Expand Down
3 changes: 1 addition & 2 deletions scripts/performance/logger.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,7 @@ def __initialize(verbose: bool):
def __generate_log_file_name(launch_datetime: datetime) -> str:
'''Generates a unique log file name for the current script.'''
log_dir = path.join(get_repo_root_path(), 'logs')
if not path.exists(log_dir):
makedirs(log_dir)
makedirs(log_dir, exist_ok=True)

if not hasattr(__main__, '__file__'):
script_name = 'python_interactive_mode'
Expand Down
Loading

0 comments on commit 254b737

Please sign in to comment.