Installed package tracking (#1614)

The beginning of tracking packages installed by winget specifically.  This adds the infrastructure to create a local index of installed packages per source, as well as the workflow updates to write installs, upgrades, and uninstalls.  Future work will leverage this data to better correlate installed packages with their source.

src/AppInstallerCLICore/Commands/UninstallCommand.cpp

@@ -136,6 +136,7 @@ namespace AppInstaller::CLI
             Workflow::ReportDependencies(Resource::String::UninstallCommandReportDependencies) <<
             Workflow::ReportExecutionStage(ExecutionStage::Execution) <<
             Workflow::ExecuteUninstaller <<
-            Workflow::ReportExecutionStage(ExecutionStage::PostExecution);
+            Workflow::ReportExecutionStage(ExecutionStage::PostExecution) <<
+            Workflow::RecordUninstall;
     }
 }

src/AppInstallerCLICore/Workflows/InstallFlow.cpp

@@ -11,19 +11,23 @@
 #include "MsiInstallFlow.h"
 #include "WorkflowBase.h"
 #include "Workflows/DependenciesFlow.h"
+#include <winget/PackageTrackingCatalog.h>
 
 #include <AppInstallerDeployment.h>
 
+using namespace winrt::Windows::ApplicationModel::Store::Preview::InstallControl;
+using namespace winrt::Windows::Foundation;
+using namespace winrt::Windows::Foundation::Collections;
+using namespace winrt::Windows::Management::Deployment;
+using namespace AppInstaller::CLI::Execution;
+using namespace AppInstaller::Manifest;
+using namespace AppInstaller::Repository;
+using namespace AppInstaller::Settings;
+using namespace AppInstaller::Utility;
+
+
 namespace AppInstaller::CLI::Workflow
 {
-    using namespace winrt::Windows::ApplicationModel::Store::Preview::InstallControl;
-    using namespace winrt::Windows::Foundation;
-    using namespace winrt::Windows::Foundation::Collections;
-    using namespace winrt::Windows::Management::Deployment;
-    using namespace AppInstaller::Manifest;
-    using namespace AppInstaller::Repository;
-    using namespace AppInstaller::Settings;
-
     namespace
     {
         bool MightWriteToARP(InstallerTypeEnum type)
@@ -370,6 +374,7 @@ namespace AppInstaller::CLI::Workflow
             Workflow::ExecuteInstaller <<
             Workflow::ReportExecutionStage(ExecutionStage::PostExecution) <<
             Workflow::ReportARPChanges <<
+            Workflow::RecordInstall <<
             Workflow::RemoveInstaller;
     }
 
@@ -655,5 +660,23 @@ namespace AppInstaller::CLI::Workflow
             );
         }
     }
-    CATCH_LOG()
+    CATCH_LOG();
+
+    void RecordInstall(Context& context)
+    {
+        // Local manifest installs won't have a package version, and tracking them doesn't provide much
+        // value currently. If we ever do use our own database as a primary source of packages that we
+        // maintain, this decision will probably have to be reconsidered.
+        if (!context.Contains(Data::PackageVersion))
+        {
+            return;
+        }
+
+        auto trackingCatalog = PackageTrackingCatalog::CreateForSource(context.Get<Data::PackageVersion>()->GetSource());
+
+        trackingCatalog.RecordInstall(
+            context.Get<Data::Manifest>(),
+            context.Get<Data::Installer>().value(),
+            WI_IsFlagSet(context.GetFlags(), ContextFlag::InstallerExecutionUseUpdate));
+    }
 }

src/AppInstallerCLICore/Workflows/InstallFlow.h

@@ -160,4 +160,10 @@ namespace AppInstaller::CLI::Workflow
     // Inputs: ARPSnapshot?, Manifest, PackageVersion
     // Outputs: None
     void ReportARPChanges(Execution::Context& context);
+
+    // Records the installation to the tracking catalog.
+    // Required Args: None
+    // Inputs: PackageVersion?, Manifest, Installer
+    // Outputs: None
+    void RecordInstall(Execution::Context& context);
 }

src/AppInstallerCLICore/Workflows/UninstallFlow.cpp

@@ -7,13 +7,51 @@
 #include "AppInstallerMsixInfo.h"
 
 #include <AppInstallerDeployment.h>
+#include <winget/PackageTrackingCatalog.h>
 
+using namespace AppInstaller::CLI::Execution;
 using namespace AppInstaller::Manifest;
 using namespace AppInstaller::Msix;
 using namespace AppInstaller::Repository;
 
 namespace AppInstaller::CLI::Workflow
 {
+    namespace
+    {
+        // Helper for RecordUninstall
+        struct UninstallCorrelatedSources
+        {
+            struct Item
+            {
+                Utility::LocIndString Identifier;
+                std::shared_ptr<const ISource> Source;
+                std::string SourceIdentifier;
+            };
+
+            void AddIfRemoteAndNotPresent(const std::shared_ptr<IPackageVersion>& packageVersion)
+            {
+                auto source = packageVersion->GetSource();
+                const auto& details = source->GetDetails();
+                if (!ContainsAvailablePackages(details.Origin))
+                {
+                    return;
+                }
+
+                for (const auto& item : Items)
+                {
+                    if (item.SourceIdentifier == details.Identifier)
+                    {
+                        return;
+                    }
+                }
+
+                Items.emplace_back(Item{ packageVersion->GetProperty(PackageVersionProperty::Id), std::move(source), details.Identifier });
+            }
+
+            std::vector<Item> Items;
+        };
+    }
+
     void GetUninstallInfo(Execution::Context& context)
     {
         auto installedPackageVersion = context.Get<Execution::Data::InstalledPackageVersion>();
@@ -124,4 +162,27 @@ namespace AppInstaller::CLI::Workflow
 
         context.Reporter.Info() << Resource::String::UninstallFlowUninstallSuccess << std::endl;
     }
-}
+
+    void RecordUninstall(Context& context)
+    {
+        // In order to report an uninstall to every correlated tracking catalog, we first need to find them all.
+        auto package = context.Get<Data::Package>();
+        UninstallCorrelatedSources correlatedSources;
+
+        // Start with the installed version
+        correlatedSources.AddIfRemoteAndNotPresent(package->GetInstalledVersion());
+
+        // Then look through all available versions
+        for (const auto& versionKey : package->GetAvailableVersionKeys())
+        {
+            correlatedSources.AddIfRemoteAndNotPresent(package->GetAvailableVersion(versionKey));
+        }
+
+        // Finally record the uninstall for each found value
+        for (const auto& item : correlatedSources.Items)
+        {
+            auto trackingCatalog = PackageTrackingCatalog::CreateForSource(item.Source);
+            trackingCatalog.RecordUninstall(item.Identifier);
+        }
+    }
+}

src/AppInstallerCLICore/Workflows/UninstallFlow.h

@@ -22,4 +22,10 @@ namespace AppInstaller::CLI::Workflow
     // Inputs: PackageFamilyNames
     // Outputs: None
     void MsixUninstall(Execution::Context& context);
-}
+
+    // Records the uninstall to the tracking catalog.
+    // Required Args: None
+    // Inputs: Package
+    // Outputs: None
+    void RecordUninstall(Execution::Context& context);
+}

src/AppInstallerCLITests/AppInstallerCLITests.vcxproj

@@ -198,6 +198,7 @@
     <ClCompile Include="MsixInfo.cpp" />
     <ClCompile Include="NameNormalization.cpp" />
     <ClCompile Include="PackageCollection.cpp" />
+    <ClCompile Include="PackageTrackingCatalog.cpp" />
     <ClCompile Include="PredefinedInstalledSource.cpp" />
     <ClCompile Include="PreIndexedPackageSource.cpp" />
     <ClCompile Include="Regex.cpp" />

src/AppInstallerCLITests/AppInstallerCLITests.vcxproj.filters

@@ -176,6 +176,9 @@
     <ClCompile Include="RestInterface_1_1.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="PackageTrackingCatalog.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <None Include="PropertySheet.props" />

src/AppInstallerCLITests/PackageTrackingCatalog.cpp

@@ -0,0 +1,174 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+#include "pch.h"
+#include "TestCommon.h"
+#include <Microsoft/SQLiteIndexSource.h>
+#include <winget/ManifestYamlParser.h>
+#include <winget/PackageTrackingCatalog.h>
+
+using namespace std::string_literals;
+using namespace TestCommon;
+using namespace AppInstaller::Manifest;
+using namespace AppInstaller::Repository;
+using namespace AppInstaller::Repository::Microsoft;
+using namespace AppInstaller::Repository::SQLite;
+using namespace AppInstaller::Utility;
+
+static std::shared_ptr<SQLiteIndexSource> SimpleTestSetup(const std::string& filePath, SourceDetails& details, Manifest& manifest, std::string& relativePath)
+{
+    SQLiteIndex index = SQLiteIndex::CreateNew(filePath, Schema::Version::Latest());
+
+    TestDataFile testManifest("Manifest-Good.yaml");
+    manifest = YamlParser::CreateFromPath(testManifest);
+
+    relativePath = testManifest.GetPath().filename().u8string();
+
+    index.AddManifest(manifest, relativePath);
+
+    details.Name = "TestName";
+    details.Type = "TestType";
+    details.Arg = testManifest.GetPath().parent_path().u8string();
+    details.Data = "";
+
+    auto result = std::make_shared<SQLiteIndexSource>(details, "*SimpleTestSetup", std::move(index));
+
+    PackageTrackingCatalog::RemoveForSource(result->GetIdentifier());
+
+    return result;
+}
+
+TEST_CASE("TrackingCatalog_Create", "[tracking_catalog]")
+{
+    TempFile tempFile{ "repolibtest_tempdb"s, ".db"s };
+    INFO("Using temporary file named: " << tempFile.GetPath());
+
+    SourceDetails details;
+    Manifest manifest;
+    std::string relativePath;
+    std::shared_ptr<SQLiteIndexSource> source = SimpleTestSetup(tempFile, details, manifest, relativePath);
+
+    PackageTrackingCatalog catalog = PackageTrackingCatalog::CreateForSource(source);
+}
+
+TEST_CASE("TrackingCatalog_Install", "[tracking_catalog]")
+{
+    TempFile tempFile{ "repolibtest_tempdb"s, ".db"s };
+    INFO("Using temporary file named: " << tempFile.GetPath());
+
+    SourceDetails details;
+    Manifest manifest;
+    std::string relativePath;
+    std::shared_ptr<SQLiteIndexSource> source = SimpleTestSetup(tempFile, details, manifest, relativePath);
+
+    PackageTrackingCatalog catalog = PackageTrackingCatalog::CreateForSource(source);
+
+    SearchRequest request;
+    request.Filters.emplace_back(PackageMatchField::Id, MatchType::Exact, manifest.Id);
+
+    SearchResult resultBefore = catalog.Search(request);
+    REQUIRE(resultBefore.Matches.size() == 0);
+
+    catalog.RecordInstall(manifest, manifest.Installers[0], false);
+
+    SearchResult resultAfter = catalog.Search(request);
+    REQUIRE(resultAfter.Matches.size() == 1);
+
+    auto trackingVersion = resultAfter.Matches[0].Package->GetLatestAvailableVersion();
+    REQUIRE(trackingVersion);
+
+    auto metadata = trackingVersion->GetMetadata();
+    REQUIRE(metadata.find(PackageVersionMetadata::TrackingWriteTime) != metadata.end());
+}
+
+TEST_CASE("TrackingCatalog_Reinstall", "[tracking_catalog]")
+{
+    TempFile tempFile{ "repolibtest_tempdb"s, ".db"s };
+    INFO("Using temporary file named: " << tempFile.GetPath());
+
+    SourceDetails details;
+    Manifest manifest;
+    std::string relativePath;
+    std::shared_ptr<SQLiteIndexSource> source = SimpleTestSetup(tempFile, details, manifest, relativePath);
+
+    PackageTrackingCatalog catalog = PackageTrackingCatalog::CreateForSource(source);
+
+    SearchRequest request;
+    request.Filters.emplace_back(PackageMatchField::Id, MatchType::Exact, manifest.Id);
+
+    catalog.RecordInstall(manifest, manifest.Installers[0], false);
+
+    SearchResult resultBefore = catalog.Search(request);
+    REQUIRE(resultBefore.Matches.size() == 1);
+    REQUIRE(resultBefore.Matches[0].Package->GetLatestAvailableVersion()->GetProperty(PackageVersionProperty::Name) ==
+        manifest.DefaultLocalization.Get<Localization::PackageName>());
+
+    // Change name
+    std::string newName = "New Package Name";
+    manifest.DefaultLocalization.Add<Localization::PackageName>(newName);
+
+    catalog.RecordInstall(manifest, manifest.Installers[0], false);
+
+    SearchResult resultAfter = catalog.Search(request);
+    REQUIRE(resultAfter.Matches.size() == 1);
+    REQUIRE(resultBefore.Matches[0].Package->GetLatestAvailableVersion()->GetProperty(PackageVersionProperty::Name) ==
+        newName);
+}
+
+TEST_CASE("TrackingCatalog_Upgrade", "[tracking_catalog]")
+{
+    TempFile tempFile{ "repolibtest_tempdb"s, ".db"s };
+    INFO("Using temporary file named: " << tempFile.GetPath());
+
+    SourceDetails details;
+    Manifest manifest;
+    std::string relativePath;
+    std::shared_ptr<SQLiteIndexSource> source = SimpleTestSetup(tempFile, details, manifest, relativePath);
+
+    PackageTrackingCatalog catalog = PackageTrackingCatalog::CreateForSource(source);
+
+    SearchRequest request;
+    request.Filters.emplace_back(PackageMatchField::Id, MatchType::Exact, manifest.Id);
+
+    catalog.RecordInstall(manifest, manifest.Installers[0], false);
+
+    SearchResult resultBefore = catalog.Search(request);
+    REQUIRE(resultBefore.Matches.size() == 1);
+    REQUIRE(resultBefore.Matches[0].Package->GetLatestAvailableVersion()->GetProperty(PackageVersionProperty::Version) ==
+        manifest.Version);
+
+    // Change name
+    manifest.Version = "99.1.2.3";
+
+    catalog.RecordInstall(manifest, manifest.Installers[0], true);
+
+    SearchResult resultAfter = catalog.Search(request);
+    REQUIRE(resultAfter.Matches.size() == 1);
+    REQUIRE(resultBefore.Matches[0].Package->GetLatestAvailableVersion()->GetProperty(PackageVersionProperty::Version) ==
+        manifest.Version);
+}
+
+TEST_CASE("TrackingCatalog_Uninstall", "[tracking_catalog]")
+{
+    TempFile tempFile{ "repolibtest_tempdb"s, ".db"s };
+    INFO("Using temporary file named: " << tempFile.GetPath());
+
+    SourceDetails details;
+    Manifest manifest;
+    std::string relativePath;
+    std::shared_ptr<SQLiteIndexSource> source = SimpleTestSetup(tempFile, details, manifest, relativePath);
+
+    PackageTrackingCatalog catalog = PackageTrackingCatalog::CreateForSource(source);
+
+    SearchRequest request;
+    request.Filters.emplace_back(PackageMatchField::Id, MatchType::Exact, manifest.Id);
+
+    catalog.RecordInstall(manifest, manifest.Installers[0], false);
+
+    SearchResult resultBefore = catalog.Search(request);
+    REQUIRE(resultBefore.Matches.size() == 1);
+
+    catalog.RecordUninstall(LocIndString{ manifest.Id });
+
+    SearchResult resultAfter = catalog.Search(request);
+    REQUIRE(resultAfter.Matches.size() == 0);
+}