Use asset manifests exclusively to produce/consume previously source built artifacts

src/SourceBuild/content/eng/bootstrap/buildBootstrapPreviouslySB.csproj

@@ -26,12 +26,16 @@
     <UnixRid Include="osx-arm64" />
   </ItemGroup>
 
-  <!-- These packages will be replaced with ms-built packages downloaded from official package feeds-->
+  <!-- 
+    These packages will be replaced with ms-built packages downloaded from official package feeds.
+    The runtime packs do not have a RIDless package, so we'll use the version of their corresponding ref pack
+    to determine the version.
+  -->
   <ItemGroup>
-    <RuntimePack Include="Microsoft.Aspnetcore.App.Runtime" Version="[$(MicrosoftAspNetCoreAppRuntimeVersion)]" />
-    <RuntimePack Include="Microsoft.NETCore.App.Crossgen2" Version="[$(MicrosoftNETCoreAppCrossgen2Version)]" />
-    <RuntimePack Include="Microsoft.NETCore.App.Host" Version="[$(MicrosoftNETCoreAppHostPackageVersion)]" />
-    <RuntimePack Include="Microsoft.NETCore.App.Runtime" Version="[$(MicrosoftNETCoreAppRuntimeVersion)]" />
+    <RuntimePack Include="Microsoft.AspNetCore.App.Runtime" Version="[$(MicrosoftAspNetCoreAppRefPackageVersion)]" />
+    <RuntimePack Include="Microsoft.NETCore.App.Crossgen2" Version="[$(MicrosoftNETCoreAppRefPackageVersion)]" />
+    <RuntimePack Include="Microsoft.NETCore.App.Host" Version="[$(MicrosoftNETCoreAppRefPackageVersion)]" />
+    <RuntimePack Include="Microsoft.NETCore.App.Runtime" Version="[$(MicrosoftNETCoreAppRefPackageVersion)]" />
 
     <PortablePackage Include="Microsoft.DotNet.ILCompiler" Version="[$(MicrosoftDotNetILCompilerVersion)]" />
     <PortablePackage Include="Microsoft.NETCore.DotNetAppHost" Version="[$(MicrosoftNETCoreDotNetAppHostVersion)]" />
@@ -63,7 +67,7 @@
     <ItemGroup>
       <!--
             Generate package names for runtime packs by concatenating the base name with the Unix RID
-            (e.g. Microsoft.Aspnetcore.App.Runtime.linux-x64)
+            (e.g. Microsoft.AspNetCore.App.Runtime.linux-x64)
         -->
       <PackageWithName Include="@(RuntimePackWithUnixRid)">
         <PackageName>%(RuntimePackWithUnixRid.Identity).%(RuntimePackWithUnixRid.UnixRid)</PackageName>

src/SourceBuild/content/eng/finish-source-only.proj

@@ -200,11 +200,33 @@
     </PropertyGroup>
   </Target>
 
+  <!-- Discover the produced packages from all repo asset manifests for this repository. -->
+  <Target Name="DiscoverProducedPackages" DependsOnTargets="ResolveProjectReferences">
+    <XmlPeek XmlInputPath="$(MergedAssetManifestOutputPath)" Query="/Build/*[self::Package]">
+      <Output TaskParameter="Result" ItemName="ProducedPackageEntry" />
+    </XmlPeek>
+  </Target>
+
+  <Target Name="ProcessPackageEntries" DependsOnTargets="DiscoverProducedPackages" Inputs="@(ProducedPackageEntry)" Outputs="%(Identity).ForBatching">
+    <XmlPeek XmlContent="%(ProducedPackageEntry.Identity)" Query="/Package/@Id">
+      <Output TaskParameter="Result" PropertyName="ProducedPackageId" />
+    </XmlPeek>
+    <XmlPeek XmlContent="%(ProducedPackageEntry.Identity)" Query="/Package/@Version">
+      <Output TaskParameter="Result" PropertyName="ProducedPackageVersion" />
+    </XmlPeek>
+
+    <ItemGroup>
+      <ProducedPackage Include="$(ProducedPackageId)" Version="$(ProducedPackageVersion)" ReferenceOnly="$([MSBuild]::ValueOrDefault('$(ReferenceOnlyRepoArtifacts)', 'false'))" />
+    </ItemGroup>
+  </Target>
+
+  <Target Name="GetProducedPackages" DependsOnTargets="DiscoverProducedPackages;ProcessPackageEntries" Returns="@(ProducedPackage)" />
+
   <!-- Create the SourceBuilt.Private.Artifacts archive when building source-only. -->
   <UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.WritePackageVersionsProps" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
   <Target Name="CreatePrivateSourceBuiltArtifactsArchive"
           AfterTargets="Build"
-          DependsOnTargets="GetInputsOutputForCreatePrivateSourceBuiltArtifactsArchive"
+          DependsOnTargets="GetInputsOutputForCreatePrivateSourceBuiltArtifactsArchive;GetProducedPackages"
           Inputs="@(ArtifactsPackageToBundle);@(ReferencePackageToBundle);@(MergedAssetManifest)"
           Outputs="$(SourceBuiltTarballName);$(SourceBuiltVersionName);$(AllPackageVersionsPropsName);$(SourceBuiltMergedAssetManifestName)">
     <!-- Copy packages to layout directory. Since there are a large number of files,
@@ -226,20 +248,12 @@
                       Overwrite="true" />
 
     <!-- Copy the merged asset manifest into the tarball -->
-    <Copy SourceFiles="$(MergedAssetManifestOutputPath)"
+    <Copy SourceFiles="@(MergedAssetManifest)"
           DestinationFolder="$(SourceBuiltLayoutDir)"
           UseSymbolicLinksIfPossible="true" />
 
-    <!-- non-rid-specific versions of RID-specific version variables to use for bootstrapping -->
-    <ItemGroup>
-      <ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppRuntimeVersion" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />
-      <ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppHostPackageVersion" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />
-      <ExtraPackageVersionPropsPackageInfo Include="MicrosoftAspNetCoreAppRuntimeVersion" Version="%24(MicrosoftAspNetCoreAppRefPackageVersion)" />
-      <ExtraPackageVersionPropsPackageInfo Include="MicrosoftNETCoreAppCrossgen2Version" Version="%24(MicrosoftNETCoreAppRefPackageVersion)" />  
-    </ItemGroup>
-
     <!-- Create a PackageVersions.props file that includes entries for all packages. -->
-    <WritePackageVersionsProps NuGetPackages="@(ArtifactsPackageToBundle)"
+    <WritePackageVersionsProps KnownPackages="@(ProducedPackage)"
                                ExtraProperties="@(ExtraPackageVersionPropsPackageInfo)"
                                VersionPropsFlowType="AllPackages"
                                OutputPath="$(AllPackageVersionsPropsName)" />

src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/WritePackageVersionsProps.cs

@@ -4,8 +4,6 @@
 
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
-using NuGet.Packaging;
-using NuGet.Packaging.Core;
 using NuGet.Versioning;
 using System;
 using System.Collections.Generic;
@@ -49,17 +47,13 @@ public class WritePackageVersionsProps : Microsoft.Build.Utilities.Task
         private const string DependenciesOnlyVersionPropsFlowType = "DependenciesOnly";
         private const string DefaultVersionPropsFlowType = AllPackagesVersionPropsFlowType;
 
-        /// <summary>
-        /// Set of input nuget package files to generate version properties for.
-        /// </summary>
-        public ITaskItem[] NuGetPackages { get; set; }
-
         /// <summary>
         /// Set of packages built by dependencies of this repo during this build.
         ///
         /// %(Identity): Package identity.
         /// %(Version): Package version.
         /// </summary>
+        [Required]
         public ITaskItem[] KnownPackages { get; set; }
 
         /// <summary>
@@ -176,25 +170,10 @@ public override bool Execute()
                 return !Log.HasLoggedErrors;
             }
 
-            NuGetPackages ??= Array.Empty<ITaskItem>();
             KnownPackages ??= Array.Empty<ITaskItem>();
 
             // First, obtain version information from the packages and additional assets that
             // are provided.
-            var latestPackages = NuGetPackages
-                .Select(item =>
-                {
-                    using (var reader = new PackageArchiveReader(item.GetMetadata("FullPath")))
-                    {
-                        return reader.GetIdentity();
-                    }
-                })
-                .Select(identity => new VersionEntry()
-                {
-                    Name = identity.Id,
-                    Version = identity.Version
-                });
-
             var knownPackages = KnownPackages
                 .Select(item => new VersionEntry()
                     {
@@ -205,7 +184,7 @@ public override bool Execute()
             // We may have multiple versions of the same package. We'll keep the latest one.
             // This can even happen in the KnownPackages list, as a repo (such as source-build-reference-packages)
             // may have multiple versions of the same package.
-            IEnumerable<VersionEntry> packageElementsToWrite = latestPackages.Concat(knownPackages)
+            IEnumerable<VersionEntry> packageElementsToWrite = knownPackages
                 .GroupBy(identity => identity.Name)
                 .Select(g => g.OrderByDescending(id => id.Version).First())
                 .OrderBy(id => id.Name);

src/SourceBuild/content/repo-projects/Directory.Build.targets

@@ -259,10 +259,18 @@
           DependsOnTargets="GetProducedPackagesFromTransitiveReferences"
           Inputs="$(MSBuildProjectFullPath)"
           Outputs="$(BaseIntermediateOutputPath)CreateBuildInputProps.complete">
+
     <ItemGroup>
-      <_PreviouslyBuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" />
+      <_PrebuiltAssetManifestProject Include="$(MSBuildThisFileFullPath)" AdditionalProperties="RepoAssetManifestsDir=$(PrebuiltSourceBuiltPackagesPath)" />
     </ItemGroup>
 
+    <!-- Get the previously-built-source-built package information from the manifest from that build. -->
+    <MSBuild Projects="@(_PrebuiltAssetManifestProject)"
+             Targets="GetProducedPackages"
+             BuildInParallel="true">
+      <Output TaskParameter="TargetOutputs" ItemName="_PreviouslyBuiltSourceBuiltPackages" />
+    </MSBuild>
+
     <Error Condition="'$(PackageVersionPropsFlowType)' != 'AllPackages' and '$(PackageVersionPropsFlowType)' != 'DependenciesOnly'"
       Text="Invalid PackageVersionPropsFlowType '$(PackageVersionPropsFlowType)'. Must be 'AllPackages' or 'DependenciesOnly'." />
 
@@ -278,7 +286,7 @@
                                OutputPath="$(CurrentSourceBuiltPackageVersionPropsPath)" />
 
     <!-- Create previously source-built inputs info -->
-    <WritePackageVersionsProps NuGetPackages="@(_PreviouslyBuiltSourceBuiltPackages)"
+    <WritePackageVersionsProps KnownPackages="@(_PreviouslyBuiltSourceBuiltPackages)"
                                VersionPropsFlowType="$(PackageVersionPropsFlowType)"
                                VersionDetails="$(_VersionDetailsXml)"
                                OutputPath="$(PreviouslySourceBuiltPackageVersionPropsPath)" />