Use asset manifests exclusively to produce/consume previously source built artifacts

src/SourceBuild/content/eng/bootstrap/buildBootstrapPreviouslySB.csproj

@@ -26,12 +26,16 @@
     <UnixRid Include="osx-arm64" />
   </ItemGroup>
 
-  <!-- These packages will be replaced with ms-built packages downloaded from official package feeds-->
+  <!-- 
+    These packages will be replaced with ms-built packages downloaded from official package feeds.
+    The runtime packs do not have a RIDless package, so we'll use the version of their corresponding ref pack
+    to determine the version.
+  -->
   <ItemGroup>
-    <RuntimePack Include="Microsoft.Aspnetcore.App.Runtime" Version="[$(MicrosoftAspNetCoreAppRuntimeVersion)]" />
-    <RuntimePack Include="Microsoft.NETCore.App.Crossgen2" Version="[$(MicrosoftNETCoreAppCrossgen2Version)]" />
-    <RuntimePack Include="Microsoft.NETCore.App.Host" Version="[$(MicrosoftNETCoreAppHostPackageVersion)]" />
-    <RuntimePack Include="Microsoft.NETCore.App.Runtime" Version="[$(MicrosoftNETCoreAppRuntimeVersion)]" />
+    <RuntimePack Include="Microsoft.AspNetCore.App.Runtime" Version="[$(MicrosoftAspNetCoreAppRefPackageVersion)]" />
+    <RuntimePack Include="Microsoft.NETCore.App.Crossgen2" Version="[$(MicrosoftNETCoreAppRefPackageVersion)]" />
+    <RuntimePack Include="Microsoft.NETCore.App.Host" Version="[$(MicrosoftNETCoreAppRefPackageVersion)]" />
+    <RuntimePack Include="Microsoft.NETCore.App.Runtime" Version="[$(MicrosoftNETCoreAppRefPackageVersion)]" />
 
     <PortablePackage Include="Microsoft.DotNet.ILCompiler" Version="[$(MicrosoftDotNetILCompilerVersion)]" />
     <PortablePackage Include="Microsoft.NETCore.DotNetAppHost" Version="[$(MicrosoftNETCoreDotNetAppHostVersion)]" />
@@ -63,7 +67,7 @@
     <ItemGroup>
       <!--
             Generate package names for runtime packs by concatenating the base name with the Unix RID
-            (e.g. Microsoft.Aspnetcore.App.Runtime.linux-x64)
+            (e.g. Microsoft.AspNetCore.App.Runtime.linux-x64)
         -->
       <PackageWithName Include="@(RuntimePackWithUnixRid)">
         <PackageName>%(RuntimePackWithUnixRid.Identity).%(RuntimePackWithUnixRid.UnixRid)</PackageName>
@@ -153,7 +157,7 @@
 
     <!-- Repack tarball with new bootstrap name -->
     <Message Text="  Repacking tarball to $(NewTarballName)" Importance="High" />
-    <Exec Command="tar --numeric-owner -czf $(NewTarballName) *.nupkg *.props SourceBuildReferencePackages/" WorkingDirectory="$(UnpackedTarPath)" />
+    <Exec Command="tar --numeric-owner -czf $(NewTarballName) *.nupkg *.props *.xml SourceBuildReferencePackages/" WorkingDirectory="$(UnpackedTarPath)" />
 
   </Target>
 

src/SourceBuild/content/eng/finish-source-only.proj

@@ -200,11 +200,19 @@
     </PropertyGroup>
   </Target>
 
+  <!-- Discover the produced packages from the merged asset manifest -->
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.GetKnownPackagesFromAssetManifests" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
+  <Target Name="GetProducedPackages" Returns="@(ProducedPackage)">
+    <GetKnownPackagesFromAssetManifests AssetManifests="$(MergedAssetManifestOutputPath)">
+      <Output TaskParameter="KnownPackages" ItemName="ProducedPackage" />
+    </GetKnownPackagesFromAssetManifests>
+  </Target>
+
   <!-- Create the SourceBuilt.Private.Artifacts archive when building source-only. -->
   <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.WritePackageVersionsProps" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="CreatePrivateSourceBuiltArtifactsArchive"
           AfterTargets="Build"
-          DependsOnTargets="GetInputsOutputForCreatePrivateSourceBuiltArtifactsArchive"
+          DependsOnTargets="GetInputsOutputForCreatePrivateSourceBuiltArtifactsArchive;GetProducedPackages"
           Inputs="@(ArtifactsPackageToBundle);@(ReferencePackageToBundle);@(MergedAssetManifest)"
           Outputs="$(SourceBuiltTarballName);$(SourceBuiltVersionName);$(AllPackageVersionsPropsName);$(SourceBuiltMergedAssetManifestName)">
     <!-- Copy packages to layout directory. Since there are a large number of files,
@@ -226,20 +234,12 @@
                       Overwrite="true" />
 
     <!-- Copy the merged asset manifest into the tarball -->
-    <Copy SourceFiles="$(MergedAssetManifestOutputPath)"
+    <Copy SourceFiles="@(MergedAssetManifest)"
           DestinationFolder="$(SourceBuiltLayoutDir)"
           UseSymbolicLinksIfPossible="true" />
 
-    <!-- non-rid-specific versions of RID-specific version variables to use for bootstrapping -->
-    <ItemGroup>
-      <ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppRuntimeVersion" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />
-      <ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppHostPackageVersion" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />
-      <ExtraPackageVersionPropsPackageInfo Include="MicrosoftAspNetCoreAppRuntimeVersion" Version="%24(MicrosoftAspNetCoreAppRefPackageVersion)" />
-      <ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppCrossgen2Version" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />  
-    </ItemGroup>
-
     <!-- Create a PackageVersions.props file that includes entries for all packages. -->
-    <WritePackageVersionsProps NuGetPackages="@(ArtifactsPackageToBundle)"
+    <WritePackageVersionsProps KnownPackages="@(ProducedPackage)"
                                ExtraProperties="@(ExtraPackageVersionPropsPackageInfo)"
                                VersionPropsFlowType="AllPackages"
                                OutputPath="$(AllPackageVersionsPropsName)" />

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/GetKnownPackagesFromAssetManifests.cs

@@ -0,0 +1,34 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Xml.Linq;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+
+namespace Microsoft.DotNet.UnifiedBuild.Tasks
+{
+    /// <summary>
+    /// Get a list of MSBuild Items that represent the packages described in the asset manifests.
+    /// </summary>
+    public sealed class GetKnownPackagesFromAssetManifests : Task
+    {
+        [Required]
+        public ITaskItem[] AssetManifests { get; set; }
+
+        [Output]
+        public ITaskItem[] KnownPackages { get; set; }
+
+        public override bool Execute()
+        {
+            var knownPackages = from assetManifest in AssetManifests
+                                let doc = XDocument.Load(assetManifest.ItemSpec)
+                                from package in doc.Root.Descendants("Package")
+                                select new TaskItem(package.Attribute("Id").Value, new Dictionary<string, string>{ { "Version", package.Attribute("Version").Value } });
+            KnownPackages = knownPackages.ToArray();
+            return true;
+        }
+    }
+}

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/WritePackageVersionsProps.cs

@@ -4,8 +4,6 @@
 
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
-using NuGet.Packaging;
-using NuGet.Packaging.Core;
 using NuGet.Versioning;
 using System;
 using System.Collections.Generic;
@@ -49,17 +47,13 @@ public class WritePackageVersionsProps : Microsoft.Build.Utilities.Task
         private const string DependenciesOnlyVersionPropsFlowType = "DependenciesOnly";
         private const string DefaultVersionPropsFlowType = AllPackagesVersionPropsFlowType;
 
-        /// <summary>
-        /// Set of input nuget package files to generate version properties for.
-        /// </summary>
-        public ITaskItem[] NuGetPackages { get; set; }
-
         /// <summary>
         /// Set of packages built by dependencies of this repo during this build.
         ///
         /// %(Identity): Package identity.
         /// %(Version): Package version.
         /// </summary>
+        [Required]
         public ITaskItem[] KnownPackages { get; set; }
 
         /// <summary>
@@ -176,25 +170,10 @@ public override bool Execute()
                 return !Log.HasLoggedErrors;
             }
 
-            NuGetPackages ??= Array.Empty<ITaskItem>();
             KnownPackages ??= Array.Empty<ITaskItem>();
 
             // First, obtain version information from the packages and additional assets that
             // are provided.
-            var latestPackages = NuGetPackages
-                .Select(item =>
-                {
-                    using (var reader = new PackageArchiveReader(item.GetMetadata("FullPath")))
-                    {
-                        return reader.GetIdentity();
-                    }
-                })
-                .Select(identity => new VersionEntry()
-                {
-                    Name = identity.Id,
-                    Version = identity.Version
-                });
-
             var knownPackages = KnownPackages
                 .Select(item => new VersionEntry()
                     {
@@ -205,7 +184,7 @@ public override bool Execute()
             // We may have multiple versions of the same package. We'll keep the latest one.
             // This can even happen in the KnownPackages list, as a repo (such as source-build-reference-packages)
             // may have multiple versions of the same package.
-            IEnumerable<VersionEntry> packageElementsToWrite = latestPackages.Concat(knownPackages)
+            IEnumerable<VersionEntry> packageElementsToWrite = knownPackages
                 .GroupBy(identity => identity.Name)
                 .Select(g => g.OrderByDescending(id => id.Version).First())
                 .OrderBy(id => id.Name);

src/SourceBuild/content/prep-source-build.sh

@@ -231,10 +231,16 @@ if [ "$removeBinaries" == true ]; then
       echo "  Unpacking Private.SourceBuilt.Artifacts.*.tar.gz into $packagesDir"
       mkdir -p "$packagesDir"
       tar -xzf "$sourceBuiltArchive" -C "$packagesDir"
-    elif [ ! -f "$packagesDir/PackageVersions.props" ] && [ -f "$sourceBuiltArchive" ]; then
-      echo "  Creating $packagesDir/PackageVersions.props..."
-      tar -xzf "$sourceBuiltArchive" -C "$packagesDir" PackageVersions.props
-    elif [ ! -f "$sourceBuiltArchive" ]; then
+    elif [ -f "$sourceBuiltArchive" ]; then
+      if [ ! -f "$packagesDir/PackageVersions.props" ]; then
+        echo "  Creating $packagesDir/PackageVersions.props..."
+        tar -xzf "$sourceBuiltArchive" -C "$packagesDir" PackageVersions.props
+      fi
+      if [ ! -f "$packagesDir/VerticalManifest.xml" ]; then
+        echo "  Unpacking Asset manifests into $packagesDir..."
+        tar -xzf "$sourceBuiltArchive" -C "$packagesDir" VerticalManifest.xml
+      fi
+    else
       echo "  ERROR: Private.SourceBuilt.Artifacts.*.tar.gz does not exist..."\
             "Cannot remove non-SB allowed binaries. Either pass --with-packages or download the artifacts."
       exit 1

src/SourceBuild/content/repo-projects/Directory.Build.targets

@@ -255,14 +255,21 @@
   <!-- Before a repository builds, set up the version property files that override the repo's defaults.
        There are 3 files generated -->
   <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.WritePackageVersionsProps" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.GetKnownPackagesFromAssetManifests" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="CreateBuildInputProps"
           DependsOnTargets="GetProducedPackagesFromTransitiveReferences"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)CreateBuildInputProps.complete">
+
+    <!-- Get the previously-built-source-built package information from the manifest from that build. -->
     <ItemGroup>
-      <_PreviouslyBuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" />
+      <_PrebuiltSourceBuiltAssetManifests Include="$(PrebuiltSourceBuiltPackagesPath)VerticalManifest.xml" />
     </ItemGroup>
 
+    <GetKnownPackagesFromAssetManifests AssetManifests="@(_PrebuiltSourceBuiltAssetManifests)">
+      <Output TaskParameter="KnownPackages" ItemName="_PreviouslyBuiltSourceBuiltPackages" />
+    </GetKnownPackagesFromAssetManifests>
+
     <Error Condition="'$(PackageVersionPropsFlowType)' != 'AllPackages' and '$(PackageVersionPropsFlowType)' != 'DependenciesOnly'"
       Text="Invalid PackageVersionPropsFlowType '$(PackageVersionPropsFlowType)'. Must be 'AllPackages' or 'DependenciesOnly'." />
 
@@ -278,7 +285,7 @@
                                OutputPath="$(CurrentSourceBuiltPackageVersionPropsPath)" />
 
     <!-- Create previously source-built inputs info -->
-    <WritePackageVersionsProps NuGetPackages="@(_PreviouslyBuiltSourceBuiltPackages)"
+    <WritePackageVersionsProps KnownPackages="@(_PreviouslyBuiltSourceBuiltPackages)"
                                VersionPropsFlowType="$(PackageVersionPropsFlowType)"
                                VersionDetails="$(_VersionDetailsXml)"
                                OutputPath="$(PreviouslySourceBuiltPackageVersionPropsPath)" />
@@ -470,28 +477,16 @@
     <Message Importance="High" Text="  -> %(RepoManifestArtifact.Identity)" />
   </Target>
 
-  <!-- Discover the produced packages from all repo asset manifests for this repository. -->
-  <Target Name="DiscoverProducedPackages" DependsOnTargets="GetRepoAssetManifests" Inputs="@(RepoAssetManifest)" Outputs="%(Identity).ForBatching">
-    <XmlPeek XmlInputPath="%(RepoAssetManifest.Identity)" Query="/Build/*[self::Package]">
-      <Output TaskParameter="Result" ItemName="ProducedPackageEntry" />
-    </XmlPeek>
-  </Target>
-
-  <Target Name="ProcessPackageEntries" DependsOnTargets="DiscoverProducedPackages" Inputs="@(ProducedPackageEntry)" Outputs="%(Identity).ForBatching">
-    <XmlPeek XmlContent="%(ProducedPackageEntry.Identity)" Query="/Package/@Id">
-      <Output TaskParameter="Result" PropertyName="ProducedPackageId" />
-    </XmlPeek>
-    <XmlPeek XmlContent="%(ProducedPackageEntry.Identity)" Query="/Package/@Version">
-      <Output TaskParameter="Result" PropertyName="ProducedPackageVersion" />
-    </XmlPeek>
-
+  <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.GetKnownPackagesFromAssetManifests" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
+  <Target Name="GetProducedPackages" DependsOnTargets="GetRepoAssetManifests" Returns="@(ProducedPackage)">
+    <GetKnownPackagesFromAssetManifests AssetManifests="@(RepoAssetManifest)">
+      <Output TaskParameter="KnownPackages" ItemName="ProducedPackage" />
+    </GetKnownPackagesFromAssetManifests>
     <ItemGroup>
-      <ProducedPackage Include="$(ProducedPackageId)" Version="$(ProducedPackageVersion)" ReferenceOnly="$([MSBuild]::ValueOrDefault('$(ReferenceOnlyRepoArtifacts)', 'false'))" />
+      <ProducedPackage ReferenceOnly="$([MSBuild]::ValueOrDefault('$(ReferenceOnlyRepoArtifacts)', 'false'))" />
     </ItemGroup>
   </Target>
 
-  <Target Name="GetProducedPackages" DependsOnTargets="GetRepoAssetManifests;DiscoverProducedPackages;ProcessPackageEntries" Returns="@(ProducedPackage)" />
-
   <Target Name="GetProducedPackagesFromTransitiveReferences" DependsOnTargets="GetTransitiveRepositoryReferences" Returns="@(_DependencyProducedPackage)">
     <MSBuild Projects="@(TransitiveRepositoryReference->'%(Identity).proj')"
              Targets="GetProducedPackages"