diff --git a/src/SourceBuild/tarball/content/.gitignore b/src/SourceBuild/tarball/content/.gitignore index 6694e6a2d15a..a5da6c81c157 100644 --- a/src/SourceBuild/tarball/content/.gitignore +++ b/src/SourceBuild/tarball/content/.gitignore @@ -1,8 +1,8 @@ /.dotnet /artifacts -/packages /eng/tools/**/bin /eng/tools/**/obj +/prereqs/packages /src/linker/src/ILLink.Tasks/ILLink.Tasks.nuspec /src/nuget-client/NuGet.config /test/Microsoft.DotNet.SourceBuild.SmokeTests/bin diff --git a/src/SourceBuild/tarball/content/Directory.Build.props b/src/SourceBuild/tarball/content/Directory.Build.props index dc550accf499..ff5ceda7d5ea 100644 --- a/src/SourceBuild/tarball/content/Directory.Build.props +++ b/src/SourceBuild/tarball/content/Directory.Build.props @@ -39,8 +39,9 @@ $([MSBuild]::NormalizeDirectory('$(DOTNET_INSTALL_DIR)')) $(ProjectDir).dotnet/ $(DotNetCliToolDir)dotnet + $(ProjectDir)prereqs/packages/ $(NuGetPackageRoot) - $(ProjectDir)packages/restored/ + $(PrereqsPackagesDir)restored/ $(PackagesDir)ArcadeBootstrapPackage/ $(NETCoreSdkVersion) @@ -105,9 +106,9 @@ $(IntermediatePath)blob-feed/ $(SourceBuiltBlobFeedDir)packages/ $(SourceBuiltBlobFeedDir)assets/ - $(ProjectDir)packages/prebuilt/ - $(ProjectDir)packages/previouslyRestored/ - $(ProjectDir)packages/previously-source-built/ + $(PrereqsPackagesDir)prebuilt/ + $(PrereqsPackagesDir)previouslyRestored/ + $(PrereqsPackagesDir)previously-source-built/ $(CustomPrebuiltSourceBuiltPackagesPath)/ $(OutputPath) $(LocalBlobStorageRoot)Sdk/ @@ -138,8 +139,8 @@ $(PackageReportDir)poisoned.txt $(BaseOutputPath)conflict-report/ $(PackageReportDir)PrebuiltBurndownData.csv - $(ProjectDir)packages/archive/ - $(ProjectDir)packages/reference/ + $(PrereqsPackagesDir)archive/ + $(PrereqsPackagesDir)reference/ Private.SourceBuilt.Artifacts Private.SourceBuilt.Prebuilts https://dotnetcli.azureedge.net/source-built-artifacts/assets/ diff --git a/src/SourceBuild/tarball/content/build.sh b/src/SourceBuild/tarball/content/build.sh index 0a7763e46a24..a7d8113d49aa 100755 --- a/src/SourceBuild/tarball/content/build.sh +++ b/src/SourceBuild/tarball/content/build.sh @@ -23,6 +23,10 @@ CUSTOM_REF_PACKAGES_DIR='' CUSTOM_PACKAGES_DIR='' alternateTarget=false runningSmokeTests=false +packagesDir="$SCRIPT_ROOT/prereqs/packages/" +packagesArchiveDir="${packagesDir}archive/" +packagesRestoredDir="${packagesDir}restored/" +packagesPreviouslySourceBuiltDir="${packagesDir}previously-source-built/" CUSTOM_SDK_DIR='' while :; do @@ -95,14 +99,14 @@ if [ "$CUSTOM_PACKAGES_DIR" != "" ]; then fi fi -if [ -f "$SCRIPT_ROOT/packages/archive/archiveArtifacts.txt" ]; then +if [ -f "${packagesArchiveDir}archiveArtifacts.txt" ]; then ARCHIVE_ERROR=0 if [ ! -d "$SCRIPT_ROOT/.dotnet" ] && [ "$CUSTOM_SDK_DIR" == "" ]; then echo "ERROR: SDK not found at $SCRIPT_ROOT/.dotnet" ARCHIVE_ERROR=1 fi - if [ ! -f $SCRIPT_ROOT/packages/archive/Private.SourceBuilt.Artifacts*.tar.gz ] && [ "$CUSTOM_PACKAGES_DIR" == "" ]; then - echo "ERROR: Private.SourceBuilt.Artifacts artifact not found at $SCRIPT_ROOT/packages/archive/ - Either run prep.sh or pass --with-packages parameter" + if [ ! -f ${packagesArchiveDir}Private.SourceBuilt.Artifacts*.tar.gz ] && [ "$CUSTOM_PACKAGES_DIR" == "" ]; then + echo "ERROR: Private.SourceBuilt.Artifacts artifact not found at $packagesArchiveDir - Either run prep.sh or pass --with-packages parameter" ARCHIVE_ERROR=1 fi if [ $ARCHIVE_ERROR == 1 ]; then @@ -131,14 +135,13 @@ else fi packageVersionsPath='' -restoredPackagesDir="$SCRIPT_ROOT/packages/restored" if [[ "$CUSTOM_PACKAGES_DIR" != "" && -f "$CUSTOM_PACKAGES_DIR/PackageVersions.props" ]]; then packageVersionsPath="$CUSTOM_PACKAGES_DIR/PackageVersions.props" -elif [ -d "$SCRIPT_ROOT/packages/archive" ]; then - sourceBuiltArchive=`find $SCRIPT_ROOT/packages/archive -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'` - if [ -f "$SCRIPT_ROOT/packages/previously-source-built/PackageVersions.props" ]; then - packageVersionsPath=$SCRIPT_ROOT/packages/previously-source-built/PackageVersions.props +elif [ -d "$packagesArchiveDir" ]; then + sourceBuiltArchive=`find $packagesArchiveDir -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'` + if [ -f "${packagesPreviouslySourceBuiltDir}}PackageVersions.props" ]; then + packageVersionsPath=${packagesPreviouslySourceBuiltDir}PackageVersions.props elif [ -f "$sourceBuiltArchive" ]; then tar -xzf "$sourceBuiltArchive" -C /tmp PackageVersions.props packageVersionsPath=/tmp/PackageVersions.props @@ -147,7 +150,7 @@ fi if [ ! -f "$packageVersionsPath" ]; then echo "Cannot find PackagesVersions.props. Debugging info:" - echo " Attempted archive path: $SCRIPT_ROOT/packages/archive" + echo " Attempted archive path: $packagesArchiveDir" echo " Attempted custom PVP path: $CUSTOM_PACKAGES_DIR/PackageVersions.props" exit 1 fi @@ -162,7 +165,7 @@ if [[ $arcadeSdkLine =~ $versionPattern ]]; then # projects overwrite this so that they use the source-built Arcade SDK instad. export SOURCE_BUILT_SDK_ID_ARCADE=Microsoft.DotNet.Arcade.Sdk export SOURCE_BUILT_SDK_VERSION_ARCADE=$ARCADE_BOOTSTRAP_VERSION - export SOURCE_BUILT_SDK_DIR_ARCADE=$restoredPackagesDir/ArcadeBootstrapPackage/microsoft.dotnet.arcade.sdk/$ARCADE_BOOTSTRAP_VERSION + export SOURCE_BUILT_SDK_DIR_ARCADE=$packagesRestoredDir/ArcadeBootstrapPackage/microsoft.dotnet.arcade.sdk/$ARCADE_BOOTSTRAP_VERSION fi sourceLinkLine=`grep -m 1 'MicrosoftSourceLinkCommonVersion' "$packageVersionsPath"` @@ -174,7 +177,7 @@ fi echo "Found bootstrap SDK $SDK_VERSION, bootstrap Arcade $ARCADE_BOOTSTRAP_VERSION, bootstrap SourceLink $SOURCE_LINK_BOOTSTRAP_VERSION" export DOTNET_CLI_TELEMETRY_OPTOUT=1 -export NUGET_PACKAGES=$restoredPackagesDir/ +export NUGET_PACKAGES=$packagesRestoredDir/ LogDateStamp=$(date +"%m%d%H%M%S") diff --git a/src/SourceBuild/tarball/content/prep.sh b/src/SourceBuild/tarball/content/prep.sh index aaba4f130bac..4fcb7e1bb3cb 100755 --- a/src/SourceBuild/tarball/content/prep.sh +++ b/src/SourceBuild/tarball/content/prep.sh @@ -52,14 +52,15 @@ fi # Check if Private.SourceBuilt artifacts archive exists artifactsBaseFileName="Private.SourceBuilt.Artifacts" -if [ -f $SCRIPT_ROOT/packages/archive/$artifactsBaseFileName.*.tar.gz ]; then +packagesArchiveDir="$SCRIPT_ROOT/prereqs/packages/archive/" +if [ -f ${packagesArchiveDir}${artifactsBaseFileName}.*.tar.gz ]; then echo " Private.SourceBuilt.Artifacts.*.tar.gz exists...it will not be downloaded" downloadArtifacts=false fi # Check if Private.SourceBuilt prebuilts archive exists prebuiltsBaseFileName="Private.SourceBuilt.Prebuilts" -if [ -f $SCRIPT_ROOT/packages/archive/$prebuiltsBaseFileName.*.tar.gz ]; then +if [ -f ${packagesArchiveDir}${prebuiltsBaseFileName}.*.tar.gz ]; then echo " Private.SourceBuilt.Prebuilts.*.tar.gz exists...it will not be downloaded" downloadPrebuilts=false fi @@ -85,7 +86,7 @@ function DownloadArchive { if [[ $archiveVersionLine =~ $versionPattern ]]; then archiveUrl="${sourceBuiltArtifactsTarballUrl}${baseFileName}.${BASH_REMATCH[1]}.tar.gz" echo " Downloading source-built $archiveType from $archiveUrl..." - (cd $SCRIPT_ROOT/packages/archive/ && curl --retry 5 -O $archiveUrl) + (cd $packagesArchiveDir && curl --retry 5 -O $archiveUrl) elif [ "$isRequired" == "true" ]; then echo " ERROR: $notFoundMessage" exit -1 @@ -128,13 +129,13 @@ if [ "$buildBootstrap" == "true" ]; then # Get PackageVersions.props from existing prev-sb archive echo " Retrieving PackageVersions.props from existing archive" - sourceBuiltArchive=`find $SCRIPT_ROOT/packages/archive -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'` + sourceBuiltArchive=`find $packagesArchiveDir -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'` if [ -f "$sourceBuiltArchive" ]; then tar -xzf "$sourceBuiltArchive" -C $workingDir PackageVersions.props fi # Run restore on project to initiate download of bootstrap packages - $DOTNET_SDK_PATH/dotnet restore $workingDir/buildBootstrapPreviouslySB.csproj /bl:artifacts/prep/bootstrap.binlog /fileLoggerParameters:LogFile=artifacts/prep/bootstrap.log /p:ArchiveDir="$SCRIPT_ROOT/packages/archive/" /p:BootstrapOverrideVersionsProps="$SCRIPT_ROOT/eng/bootstrap/OverrideBootstrapVersions.props" + $DOTNET_SDK_PATH/dotnet restore $workingDir/buildBootstrapPreviouslySB.csproj /bl:artifacts/prep/bootstrap.binlog /fileLoggerParameters:LogFile=artifacts/prep/bootstrap.log /p:ArchiveDir="$packagesArchiveDir" /p:BootstrapOverrideVersionsProps="$SCRIPT_ROOT/eng/bootstrap/OverrideBootstrapVersions.props" # Remove working directory rm -rf $workingDir diff --git a/src/SourceBuild/tarball/content/prereqs/packages/archive/_ b/src/SourceBuild/tarball/content/prereqs/packages/archive/_ new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/SourceBuild/tarball/content/prereqs/packages/prebuilt/_ b/src/SourceBuild/tarball/content/prereqs/packages/prebuilt/_ new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md index 416492a6e37c..5cd7d8e867f1 100644 --- a/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md +++ b/src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md @@ -7,4 +7,4 @@ Some prerelease scenarios, usually security updates, require non-source-built packages which are not publicly available. Specify the directory where these packages can be found via the `SMOKE_TESTS_PREREQS_PATH` environment variable when running tests via `build.sh --run-smoke-test` e.g. -`SMOKE_TESTS_PREREQS_PATH=packages/smoke-test-prereqs`. +`SMOKE_TESTS_PREREQS_PATH=prereqs/packages/smoke-test-prereqs`.