Skip to content

Commit

Permalink
Initial Android cross-compiler (#43535)
Browse files Browse the repository at this point in the history
Produce cross compilers for Android targets, from Linux x64 and OSX x64 hosts. This should be reasonably solid groundwork for adding more targets and more hosts for cross-compile scenarios.
  • Loading branch information
directhex authored Jan 14, 2021
1 parent db1311e commit 48f4f29
Show file tree
Hide file tree
Showing 23 changed files with 399 additions and 35 deletions.
9 changes: 8 additions & 1 deletion eng/Subsets.props
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
<DefaultMonoSubsets Condition="'$(MonoEnableLLVM)' == 'true' and '$(MonoLLVMDir)' == ''">mono.llvm+</DefaultMonoSubsets>
<DefaultMonoSubsets Condition="'$(MonoAOTEnableLLVM)' == 'true' and '$(MonoAOTLLVMDir)' == ''">mono.llvm+</DefaultMonoSubsets>
<DefaultMonoSubsets Condition="'$(TargetOS)' == 'Browser'">$(DefaultMonoSubsets)mono.wasmruntime+</DefaultMonoSubsets>
<DefaultMonoSubsets Condition="'$(BuildMonoCrossAOT)' == 'true'">$(DefaultMonoSubsets)mono.aotcross+</DefaultMonoSubsets>
<DefaultMonoSubsets>$(DefaultMonoSubsets)mono.runtime+mono.corelib+mono.packages</DefaultMonoSubsets>

<DefaultLibrariesSubsets>libs.native+libs.ref+libs.src+libs.pretest+libs.packages</DefaultLibrariesSubsets>
Expand Down Expand Up @@ -92,6 +93,7 @@
<!-- Mono -->
<SubsetName Include="Mono" Description="The Mono runtime and CoreLib." />
<SubsetName Include="Mono.Runtime" Description="The Mono .NET runtime." />
<SubsetName Include="Mono.AotCross" Description="The cross-compiler runtime for Mono AOT." />
<SubsetName Include="Mono.CoreLib" Description="The managed System.Private.CoreLib library for Mono." />
<SubsetName Include="Mono.Packages" Description="The projects that produce NuGet packages for the Mono runtime." />
<SubsetName Include="Mono.WasmRuntime" Description="The WebAssembly runtime." />
Expand Down Expand Up @@ -201,7 +203,7 @@
</ItemGroup>

<!-- Mono sets -->
<ItemGroup Condition="$(_subset.Contains('+mono.llvm+')) or '$(TargetOS)' == 'iOS' or '$(TargetOS)' == 'tvOS' or '$(TargetOS)' == 'Browser'">
<ItemGroup Condition="$(_subset.Contains('+mono.llvm+')) or $(_subset.Contains('+mono.aotcross+')) or '$(TargetOS)' == 'iOS' or '$(TargetOS)' == 'tvOS' or '$(TargetOS)' == 'Android' or '$(TargetOS)' == 'Browser'">
<ProjectToBuild Include="$(MonoProjectRoot)llvm\llvm-init.proj" Category="mono" />
</ItemGroup>

Expand All @@ -213,6 +215,10 @@
<ProjectToBuild Include="$(MonoProjectRoot)mono.proj" Category="mono" />
</ItemGroup>

<ItemGroup Condition="$(_subset.Contains('+mono.aotcross+'))">
<ProjectToBuild Include="$(MonoProjectRoot)monoaotcross.proj" Category="mono" />
</ItemGroup>

<ItemGroup Condition="$(_subset.Contains('+mono.corelib+'))">
<ProjectToBuild Include="$(MonoProjectRoot)netcore\System.Private.CoreLib\System.Private.CoreLib.csproj" Category="mono" />
</ItemGroup>
Expand Down Expand Up @@ -282,6 +288,7 @@
<SharedFrameworkProjectToBuild Condition="'$(RuntimeFlavor)' != 'Mono'" Include="$(InstallerProjectRoot)pkg\sfx\installers\dotnet-hostfxr.proj" />
<SharedFrameworkProjectToBuild Condition="'$(RuntimeFlavor)' != 'Mono'" Include="$(InstallerProjectRoot)pkg\sfx\installers\dotnet-runtime-deps\*.proj" />
<SharedFrameworkProjectToBuild Condition="'$(RuntimeFlavor)' != 'Mono'" Include="$(InstallerProjectRoot)pkg\sfx\bundle\Microsoft.NETCore.App.Bundle.bundleproj" />
<SharedFrameworkProjectToBuild Condition="'$(BuildMonoCrossAOT)' == 'true'" Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\monocrossaot.sfxproj" />
<ProjectToBuild Include="@(SharedFrameworkProjectToBuild)" Category="packs" />
</ItemGroup>

Expand Down
4 changes: 3 additions & 1 deletion eng/liveBuilds.targets
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
<CoreCLRSharedFrameworkDir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'sharedFramework'))</CoreCLRSharedFrameworkDir>
<CoreCLRCrossgen2Dir>$([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)', 'crossgen2'))</CoreCLRCrossgen2Dir>

<MonoCrossAOTDir>$([MSBuild]::NormalizeDirectory('$(MonoArtifactsPath)', 'cross'))</MonoCrossAOTDir>

<LibrariesPackagesDir>$([MSBuild]::NormalizeDirectory('$(LibrariesArtifactsPath)', 'packages', '$(LibrariesConfiguration)'))</LibrariesPackagesDir>
<LibrariesShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'Shipping'))</LibrariesShippingPackagesDir>
<LibrariesNonShippingPackagesDir>$([MSBuild]::NormalizeDirectory('$(LibrariesPackagesDir)', 'NonShipping'))</LibrariesNonShippingPackagesDir>
Expand Down Expand Up @@ -105,7 +107,7 @@
</RuntimeFiles>

<MonoCrossFiles Condition="'$(TargetsMobile)' == 'true'"
Include="$(MonoArtifactsPath)\cross\*.*" />
Include="$(MonoArtifactsPath)\cross\**\*.*" />
<MonoIncludeFiles Condition="'$(TargetsMobile)' == 'true'"
Include="$(MonoArtifactsPath)\include\**\*.*" />
</ItemGroup>
Expand Down
16 changes: 15 additions & 1 deletion eng/pipelines/common/global-build-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ parameters:
variables: []
targetRid: ''
timeoutInMinutes: ''
dependsOn: ''
pool: ''
platform: ''
condition: true
Expand Down Expand Up @@ -40,8 +41,14 @@ jobs:
workspace:
clean: all

${{ if and(ne(parameters.useCheckoutBundle, true),ne(parameters.dependsOn,'')) }}:
dependsOn: ${{ parameters.dependsOn }}

${{ if eq(parameters.useCheckoutBundle, true) }}:
dependsOn: checkout
dependsOn:
- checkout
- ${{ if ne(parameters.dependsOn,'') }}:
- ${{ parameters.dependsOn }}

variables:
- name: _osParameter
Expand Down Expand Up @@ -82,6 +89,13 @@ jobs:
- ${{ if eq(parameters.isOfficialBuild, true) }}:
- template: /eng/pipelines/common/restore-internal-tools.yml

- ${{ if eq(parameters.monoCrossAOTTargetOS, 'Android') }}:
- task: DownloadPipelineArtifact@2
displayName: Download AOT offset files
inputs:
artifact: Mono_Offsets_Android
path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles'

- ${{ if in(parameters.osGroup, 'OSX', 'iOS', 'tvOS', 'Android') }}:
- script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} ${{ parameters.archType }} azDO
displayName: Install Build Dependencies
Expand Down
26 changes: 22 additions & 4 deletions eng/pipelines/mono/templates/build-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ parameters:
isOfficialBuild: false
crossBuild: false
crossrootfsDir: ''
dependsOn: ''
monoCrossAOTTargetOS: ''
useCheckoutBundle: false

### Product build
Expand All @@ -28,6 +30,7 @@ jobs:
pool: ${{ parameters.pool }}
runtimeVariant: ${{ parameters.runtimeVariant }}
crossBuild: ${{ parameters.crossBuild }}
monoCrossAOTTargetOS: ${{ parameters.monoCrossAOTTargetOS }}
crossrootfsDir: ${{ parameters.crossroofsDir }}
condition: ${{ parameters.condition }}
useCheckoutBundle: ${{ parameters.useCheckoutBundle }}
Expand All @@ -40,6 +43,7 @@ jobs:
# Note that the containers are defined in platform-matrix.yml
container: ${{ parameters.container }}

dependsOn: ${{ parameters.dependsOn }}
timeoutInMinutes: ${{ parameters.timeoutInMinutes }}

gatherAssetManifests: true
Expand All @@ -52,6 +56,10 @@ jobs:
value: ''
- name: osOverride
value: ''
- name: aotCrossParameter
value: ''
- name: llvmParameter
value: ''
- ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
- name: officialBuildIdArg
value: '/p:officialBuildId=$(Build.BuildNumber)'
Expand Down Expand Up @@ -85,6 +93,9 @@ jobs:
- ${{ if eq(parameters.runtimeVariant, 'llvmaot') }}:
- name: llvmParameter
value: /p:MonoEnableLLVM=true /p:MonoBundleLLVMOptimizer=true $(llvmCxxAbi)
- ${{ if eq(parameters.monoCrossAOTTargetOS, 'Android') }}:
- name: aotCrossParameter
value: /p:BuildMonoCrossAOT=true /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true
- ${{ parameters.variables }}

steps:
Expand All @@ -101,6 +112,13 @@ jobs:
- script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force
displayName: Install native dependencies

- ${{ if eq(parameters.monoCrossAOTTargetOS, 'Android') }}:
- task: DownloadPipelineArtifact@2
displayName: Download AOT offset files
inputs:
artifact: Mono_Offsets_Android
path: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles'

- ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}:
- script: |
du -sh $(Build.SourcesDirectory)/*
Expand All @@ -109,10 +127,10 @@ jobs:
# Build
- ${{ if ne(parameters.osGroup, 'windows') }}:
- script: ./build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(llvmParameter)
- script: ./build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter)
displayName: Build product
- ${{ if eq(parameters.osGroup, 'windows') }}:
- script: build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(llvmParameter)
- script: build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter)
displayName: Build product

- ${{ if in(parameters.osGroup, 'OSX', 'iOS','tvOS') }}:
Expand All @@ -134,10 +152,10 @@ jobs:

# Build packages
- ${{ if ne(parameters.osGroup, 'windows') }}:
- script: ./build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(llvmParameter) -pack $(OutputRidArg)
- script: ./build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter) -pack $(OutputRidArg)
displayName: Build nupkg
- ${{ if eq(parameters.osGroup, 'windows') }}:
- script: build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(llvmParameter) -pack $(OutputRidArg)
- script: build$(scriptExt) -subset mono -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) $(aotCrossParameter) $(llvmParameter) -pack $(OutputRidArg)
displayName: Build nupkg

# Publish official build
Expand Down
94 changes: 94 additions & 0 deletions eng/pipelines/mono/templates/generate-offsets.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
parameters:
buildConfig: 'Debug'
osGroup: ''
platform: ''
container: ''
timeoutInMinutes: ''
variables: {}
pool: ''
condition: true
isOfficialBuild: false

### Product build
jobs:
- template: xplat-pipeline-job.yml
parameters:
buildConfig: ${{ parameters.buildConfig }}
osGroup: ${{ parameters.osGroup }}
helixType: 'build/product/'
enableMicrobuild: true
pool: ${{ parameters.pool }}
condition: ${{ parameters.condition }}

${{ if eq(parameters.useCheckoutBundle, true) }}:
dependsOn: checkout

# Compute job name from template parameters
name: ${{ format('mono_{0}_offsets', parameters.osGroup) }}
displayName: ${{ format('Mono {0} AOT offsets', parameters.osGroup) }}

# Run all steps in the container.
# Note that the containers are defined in platform-matrix.yml
container: ${{ parameters.container }}

timeoutInMinutes: ${{ parameters.timeoutInMinutes }}

gatherAssetManifests: true
variables:
- name: osGroup
value: ${{ parameters.osGroup }}
- name: officialBuildIdArg
value: ''
- ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}:
- name: officialBuildIdArg
value: '/p:officialBuildId=$(Build.BuildNumber)'
- name: osOverride
value: -os Linux
- name: archType
value: x64
- ${{ parameters.variables }}

steps:

# Install native dependencies
# Linux builds use docker images with dependencies preinstalled,
# and FreeBSD builds use a build agent with dependencies
# preinstalled, so we only need this step for OSX and Windows.
- ${{ if in(parameters.osGroup, 'OSX', 'iOS', 'tvOS') }}:
- script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh $(osGroup) ${{ parameters.archType }} azDO
displayName: Install native dependencies
- ${{ if eq(parameters.osGroup, 'windows') }}:
# Necessary to install python
- script: $(Build.SourcesDirectory)\eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force
displayName: Install native dependencies

# Build
- ${{ if ne(parameters.osGroup, 'windows') }}:
- script: ./build$(scriptExt) -subset mono.aotcross -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) /p:MonoGenerateOffsetsOnly=true
displayName: Generate AOT offsets
- ${{ if eq(parameters.osGroup, 'windows') }}:
- script: build$(scriptExt) -subset mono.aotcross -c $(buildConfig) -arch $(archType) $(osOverride) -ci $(officialBuildIdArg) /p:MonoGenerateOffsetsOnly=true
displayName: Generate AOT offsets

# Upload offset files
- task: CopyFiles@2
displayName: Collect offset files
inputs:
sourceFolder: '$(Build.SourcesDirectory)/artifacts/obj/mono/'
contents: '**/offsets-*.h'
targetFolder: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles/'

- task: PublishPipelineArtifact@1
displayName: Upload offset files
inputs:
targetPath: '$(Build.SourcesDirectory)/artifacts/obj/mono/offsetfiles'
artifactName: 'Mono_Offsets_$(osGroup)'

# Publish Logs
- task: PublishPipelineArtifact@1
displayName: Publish Logs
inputs:
targetPath: $(Build.SourcesDirectory)/artifacts/log
artifactName: 'BuildLogs_Mono_Offsets_$(osGroup)'
continueOnError: true
condition: always()
31 changes: 31 additions & 0 deletions eng/pipelines/runtime-official.yml
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,37 @@ stages:
extraStepsParameters:
name: MonoRuntimePacks

# Build Mono AOT offset headers once, for consumption elsewhere
#
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/mono/templates/generate-offsets.yml
buildConfig: release
platforms:
- Android_x64

#
# Build Mono release Android AOT cross-compiler
#
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/common/global-build-job.yml
runtimeFlavor: mono
buildConfig: release
platforms:
- OSX_x64
- Linux_x64
jobParameters:
buildArgs: -s mono+libs+host+packs -c $(_BuildConfig)
/p:BuildMonoCrossAOT=true /p:SkipMonoCrossJitConfigure=true /p:BuildMonoAOTCrossCompilerOnly=true
nameSuffix: AndroidAOT_Mono
runtimeVariant: crossandroid
dependsOn: mono_android_offsets
monoCrossAOTTargetOS: Android
isOfficialBuild: ${{ variables.isOfficialBuild }}
extraStepsTemplate: /eng/pipelines/common/upload-intermediate-artifacts-step.yml
extraStepsParameters:
name: MonoRuntimePacks
#
# Build Mono LLVM runtime packs
#
Expand Down
44 changes: 44 additions & 0 deletions eng/pipelines/runtime.yml
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,22 @@ jobs:
eq(dependencies.checkout.outputs['SetPathVars_coreclr.containsChange'], true),
eq(variables['isFullMatrix'], true))
# Build Mono AOT offset headers once, for consumption elsewhere
# Only when mono changed
#
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/mono/templates/generate-offsets.yml
buildConfig: release
platforms:
- Android_x64
jobParameters:
condition: >-
or(
eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true),
eq(dependencies.checkout.outputs['SetPathVars_installer.containsChange'], true),
eq(variables['isFullMatrix'], true))
# Build the whole product using Mono runtime
# Only when libraries, mono or installer are changed
#
Expand Down Expand Up @@ -459,6 +475,34 @@ jobs:
eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true),
eq(variables['isFullMatrix'], true))
#
# Build Mono release Android AOT cross-compiler
# Only when mono changed
#
- template: /eng/pipelines/common/platform-matrix.yml
parameters:
jobTemplate: /eng/pipelines/mono/templates/build-job.yml
runtimeFlavor: mono
buildConfig: release
platforms:
- OSX_x64
- Linux_x64
# - Linux_arm64
# - Linux_musl_arm64
# - windows_x64
# - windows_x86
# - windows_arm
# - windows_arm64
jobParameters:
runtimeVariant: crossandroid
dependsOn: mono_android_offsets
monoCrossAOTTargetOS: Android
condition: >-
or(
eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true),
eq(dependencies.checkout.outputs['SetPathVars_installer.containsChange'], true),
eq(variables['isFullMatrix'], true))
#
# Build Mono release
# Only when libraries or mono changed
Expand Down
4 changes: 2 additions & 2 deletions eng/testing/tests.mobile.targets
Original file line number Diff line number Diff line change
Expand Up @@ -102,14 +102,14 @@
</ItemGroup>

<MonoAOTCompiler Condition="'$(RunAOTCompilation)' == 'true'"
CompilerBinaryPath="$(MicrosoftNetCoreAppRuntimePackNativeDir)cross\mono-aot-cross"
CompilerBinaryPath="$(MicrosoftNetCoreAppRuntimePackNativeDir)cross\$(PackageRID)\mono-aot-cross"
Mode="Full"
OutputType="AsmOnly"
Assemblies="@(AotInputAssemblies)"
AotModulesTablePath="$(BundleDir)\modules.m"
AotModulesTableLanguage="ObjC"
UseLLVM="$(MonoEnableLLVM)"
LLVMPath="$(MicrosoftNetCoreAppRuntimePackNativeDir)cross">
LLVMPath="$(MicrosoftNetCoreAppRuntimePackNativeDir)cross\$(PackageRID)">
<Output TaskParameter="CompiledAssemblies" ItemName="BundleAssemblies" />
</MonoAOTCompiler>

Expand Down
Loading

0 comments on commit 48f4f29

Please sign in to comment.