Add alternate url support for some predefined sources (#2970)

Adds an alternate base URL for the sources behind `cdn.winget.microsoft.com`.  In the event that the files hosted there are inaccessible or corrupted, the alternate URL will be used instead.  This mechanism does rely on the fact that these are basically two names for the exact same set of data, which is true as the alternate is the upstream store for the CDN itself.

Additionally, we now require preindexed packaged sources to contain the manifest hash, rather than just using it if present.

Finally, this change adds a setting that can be used to disable the alternate URL for the `winget` source.  I intentionally did not add it to the schema or documentation, as I don't think it is something that should actually be set by anyone unless there is an explicit reason to do so (such as for testing purposes).

src/AppInstallerCommonCore/Public/winget/UserSettings.h

@@ -89,6 +89,7 @@ namespace AppInstaller::Settings
         // Network
         NetworkDownloader,
         NetworkDOProgressTimeoutInSeconds,
+        NetworkWingetAlternateSourceURL,
         // Logging
         LoggingLevelPreference,
         // Uninstall behavior
@@ -160,6 +161,7 @@ namespace AppInstaller::Settings
         // Network
         SETTINGMAPPING_SPECIALIZATION(Setting::NetworkDownloader, std::string, InstallerDownloader, InstallerDownloader::Default, ".network.downloader"sv);
         SETTINGMAPPING_SPECIALIZATION(Setting::NetworkDOProgressTimeoutInSeconds, uint32_t, std::chrono::seconds, 60s, ".network.doProgressTimeoutInSeconds"sv);
+        SETTINGMAPPING_SPECIALIZATION(Setting::NetworkWingetAlternateSourceURL, bool, bool, true, ".network.enableWingetAlternateSourceURL"sv);
         // Debug
         SETTINGMAPPING_SPECIALIZATION(Setting::EnableSelfInitiatedMinidump, bool, bool, false, ".debugging.enableSelfInitiatedMinidump"sv);
         // Logging

src/AppInstallerCommonCore/UserSettings.cpp

@@ -268,6 +268,7 @@ namespace AppInstaller::Settings
         WINGET_VALIDATE_PASS_THROUGH(InstallIgnoreWarnings)
         WINGET_VALIDATE_PASS_THROUGH(DisableInstallNotes)
         WINGET_VALIDATE_PASS_THROUGH(UninstallPurgePortablePackage)
+        WINGET_VALIDATE_PASS_THROUGH(NetworkWingetAlternateSourceURL)
 
         WINGET_VALIDATE_SIGNATURE(PortablePackageUserRoot)
         {

src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp

@@ -21,19 +21,82 @@ namespace AppInstaller::Repository::Microsoft
         // TODO: This being hard coded to force using the Public directory name is not ideal.
         static constexpr std::string_view s_PreIndexedPackageSourceFactory_IndexFilePath = "Public\\index.db"sv;
 
-        // Construct the package location from the given details.
-        // Currently expects that the arg is an https uri pointing to the root of the data.
-        std::string GetPackageLocation(const SourceDetails& details)
+        struct PreIndexedPackageInfo
         {
-            THROW_HR_IF(E_INVALIDARG, details.Arg.empty());
-            std::string result = details.Arg;
-            if (result.back() != '/')
+            template <typename LocationCheck>
+            PreIndexedPackageInfo(const SourceDetails& details, LocationCheck&& locationCheck)
             {
-                result += '/';
+                // Get both locations to force the alternate location check
+                m_packageLocation = GetPrimaryPackageLocation(details);
+                locationCheck(m_packageLocation);
+
+                std::string alternateLocation = GetAlternatePackageLocation(details);
+                if (!alternateLocation.empty())
+                {
+                    locationCheck(alternateLocation);
+                }
+
+                // Try getting the primary location's info
+                HRESULT primaryHR = S_OK;
+
+                try
+                {
+                    m_msixInfo = std::make_unique<Msix::MsixInfo>(m_packageLocation);
+                    return;
+                }
+                catch (...)
+                {
+                    if (alternateLocation.empty())
+                    {
+                        throw;
+                    }
+                    primaryHR = LOG_CAUGHT_EXCEPTION_MSG("PreIndexedPackageInfo failed on primary location");
+                }
+
+                // Try alternate location
+                m_packageLocation = std::move(alternateLocation);
+
+                try
+                {
+                    m_msixInfo = std::make_unique<Msix::MsixInfo>(m_packageLocation);
+                    return;
+                }
+                CATCH_LOG_MSG("PreIndexedPackageInfo failed on alternate location");
+
+                THROW_HR(primaryHR);
             }
-            result += s_PreIndexedPackageSourceFactory_PackageFileName;
-            return result;
-        }
+
+            const std::string& PackageLocation() const { return m_packageLocation; }
+            Msix::MsixInfo& MsixInfo() { return *m_msixInfo; }
+
+        private:
+            std::string m_packageLocation;
+            std::unique_ptr<Msix::MsixInfo> m_msixInfo;
+
+            std::string GetPrimaryPackageLocation(const SourceDetails& details)
+            {
+                THROW_HR_IF(E_INVALIDARG, details.Arg.empty());
+                return GetPackageLocation(details.Arg);
+            }
+
+            std::string GetAlternatePackageLocation(const SourceDetails& details)
+            {
+                return (details.AlternateArg.empty() ? std::string{} : GetPackageLocation(details.AlternateArg));
+            }
+
+            // Construct the package location from the given details.
+            // Currently expects that the arg is an https uri pointing to the root of the data.
+            std::string GetPackageLocation(const std::string& basePath)
+            {
+                std::string result = basePath;
+                if (result.back() != '/')
+                {
+                    result += '/';
+                }
+                result += s_PreIndexedPackageSourceFactory_PackageFileName;
+                return result;
+            }
+        };
 
         // Gets the package family name from the details.
         std::string GetPackageFamilyNameFromDetails(const SourceDetails& details)
@@ -75,16 +138,16 @@ namespace AppInstaller::Repository::Microsoft
                     THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type());
                 }
 
-                std::string packageLocation = GetPackageLocation(details);
-
-                THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation));
+                PreIndexedPackageInfo packageInfo(details, [](const std::string& packageLocation)
+                    {
+                        THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation));
+                    });
 
-                AICLI_LOG(Repo, Info, << "Initializing source from: " << details.Name << " => " << packageLocation);
+                AICLI_LOG(Repo, Info, << "Initializing source from: " << details.Name << " => " << packageInfo.PackageLocation());
 
-                Msix::MsixInfo packageInfo(packageLocation);
-                THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.GetIsBundle());
+                THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle());
 
-                auto fullName = packageInfo.GetPackageFullName();
+                auto fullName = packageInfo.MsixInfo().GetPackageFullName();
                 AICLI_LOG(Repo, Info, << "Found package full name: " << details.Name << " => " << fullName);
 
                 details.Data = Msix::GetPackageFamilyNameFromFullName(fullName);
@@ -96,7 +159,7 @@ namespace AppInstaller::Repository::Microsoft
                     return false;
                 }
 
-                return UpdateInternal(packageLocation, packageInfo, details, progress);
+                return UpdateInternal(packageInfo.PackageLocation(), packageInfo.MsixInfo(), details, progress);
             }
 
             bool Update(const SourceDetails& details, IProgressCallback& progress) override final
@@ -143,15 +206,14 @@ namespace AppInstaller::Repository::Microsoft
             {
                 THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type());
 
-                std::string packageLocation = GetPackageLocation(details);
-                Msix::MsixInfo packageInfo(packageLocation);
+                PreIndexedPackageInfo packageInfo(details, [](const std::string&){});
 
                 // The package should not be a bundle
-                THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.GetIsBundle());
+                THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle());
 
                 // Ensure that family name has not changed
                 THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE,
-                    GetPackageFamilyNameFromDetails(details) != Msix::GetPackageFamilyNameFromFullName(packageInfo.GetPackageFullName()));
+                    GetPackageFamilyNameFromDetails(details) != Msix::GetPackageFamilyNameFromFullName(packageInfo.MsixInfo().GetPackageFullName()));
 
                 if (progress.IsCancelled())
                 {
@@ -165,7 +227,7 @@ namespace AppInstaller::Repository::Microsoft
                     return false;
                 }
 
-                return UpdateInternal(packageLocation, packageInfo, details, progress);
+                return UpdateInternal(packageInfo.PackageLocation(), packageInfo.MsixInfo(), details, progress);
             }
         };
 
@@ -219,7 +281,7 @@ namespace AppInstaller::Repository::Microsoft
                 // We didn't use to store the source identifier, so we compute it here in case it's
                 // missing from the details.
                 m_details.Identifier = GetPackageFamilyNameFromDetails(m_details);
-                return std::make_shared<SQLiteIndexSource>(m_details, std::move(index), std::move(lock));
+                return std::make_shared<SQLiteIndexSource>(m_details, std::move(index), std::move(lock), false, true);
             }
 
         private:
@@ -394,7 +456,7 @@ namespace AppInstaller::Repository::Microsoft
                 // We didn't use to store the source identifier, so we compute it here in case it's
                 // missing from the details.
                 m_details.Identifier = GetPackageFamilyNameFromDetails(m_details);
-                return std::make_shared<SQLiteIndexSource>(m_details, std::move(index), std::move(lock));
+                return std::make_shared<SQLiteIndexSource>(m_details, std::move(index), std::move(lock), false, true);
             }
 
         private:

src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp

@@ -74,13 +74,37 @@ namespace AppInstaller::Repository::Microsoft
                 THROW_HR_IF(E_NOT_SET, !relativePathOpt);
 
                 std::optional<std::string> manifestHashString = source->GetIndex().GetPropertyByManifestId(m_manifestId, PackageVersionProperty::ManifestSHA256Hash);
+                THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, source->RequireManifestHash() && !manifestHashString);
+
                 SHA256::HashBuffer manifestSHA256;
                 if (manifestHashString)
                 {
                     manifestSHA256 = SHA256::ConvertToBytes(manifestHashString.value());
                 }
 
-                return GetManifestFromArgAndRelativePath(source->GetDetails().Arg, relativePathOpt.value(), manifestSHA256);
+                // Try the primary location 
+                HRESULT primaryHR = S_OK;
+                try
+                {
+                    return GetManifestFromArgAndRelativePath(source->GetDetails().Arg, relativePathOpt.value(), manifestSHA256);
+                }
+                catch (...)
+                {
+                    if (source->GetDetails().AlternateArg.empty())
+                    {
+                        throw;
+                    }
+                    primaryHR = LOG_CAUGHT_EXCEPTION_MSG("GetManifest failed on primary location");
+                }
+
+                // Try alternate location
+                try
+                {
+                    return GetManifestFromArgAndRelativePath(source->GetDetails().AlternateArg, relativePathOpt.value(), manifestSHA256);
+                }
+                CATCH_LOG_MSG("GetManifest failed on alternate location");
+
+                THROW_HR(primaryHR);
             }
 
             Source GetSource() const override
@@ -345,8 +369,13 @@ namespace AppInstaller::Repository::Microsoft
         };
     }
 
-    SQLiteIndexSource::SQLiteIndexSource(const SourceDetails& details, SQLiteIndex&& index, Synchronization::CrossProcessReaderWriteLock&& lock, bool isInstalledSource) :
-        m_details(details), m_lock(std::move(lock)), m_isInstalled(isInstalledSource), m_index(std::move(index))
+    SQLiteIndexSource::SQLiteIndexSource(
+        const SourceDetails& details,
+        SQLiteIndex&& index,
+        Synchronization::CrossProcessReaderWriteLock&& lock,
+        bool isInstalledSource,
+        bool requireManifestHash) :
+        m_details(details), m_lock(std::move(lock)), m_isInstalled(isInstalledSource), m_index(std::move(index)), m_requireManifestHash(requireManifestHash)
     {
     }
 

src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.h

@@ -17,7 +17,8 @@ namespace AppInstaller::Repository::Microsoft
             const SourceDetails& details,
             SQLiteIndex&& index,
             Synchronization::CrossProcessReaderWriteLock&& lock = {},
-            bool isInstalledSource = false);
+            bool isInstalledSource = false, 
+            bool requireManifestHash = false);
 
         SQLiteIndexSource(const SQLiteIndexSource&) = delete;
         SQLiteIndexSource& operator=(const SQLiteIndexSource&) = delete;
@@ -45,11 +46,14 @@ namespace AppInstaller::Repository::Microsoft
         // Determines if the other source refers to the same as this.
         bool IsSame(const SQLiteIndexSource* other) const;
 
+        bool RequireManifestHash() const { return m_requireManifestHash; }
+
     private:
         std::shared_ptr<SQLiteIndexSource> NonConstSharedFromThis() const;
 
         SourceDetails m_details;
         Synchronization::CrossProcessReaderWriteLock m_lock;
+        bool m_requireManifestHash;
         bool m_isInstalled;
 
     protected:

src/AppInstallerRepositoryCore/Public/winget/RepositorySource.h

@@ -121,6 +121,10 @@ namespace AppInstaller::Repository
 
         // The configuration of how the server certificate will be validated.
         Certificates::PinningConfiguration CertificatePinningConfiguration;
+
+        // This value is used as an alternative to the `Arg` value if it is failing to function properly.
+        // The alternate location must point to identical data or inconsistencies may arise.
+        std::string AlternateArg;
     };
 
     // Individual source agreement entry. Label will be highlighted in the display as the key of the agreement entry.

src/AppInstallerRepositoryCore/SourceList.cpp

@@ -33,6 +33,7 @@ namespace AppInstaller::Repository
 
         constexpr std::string_view s_Source_WingetCommunityDefault_Name = "winget"sv;
         constexpr std::string_view s_Source_WingetCommunityDefault_Arg = "https://cdn.winget.microsoft.com/cache"sv;
+        constexpr std::string_view s_Source_WingetCommunityDefault_AlternateArg = "https://winget.azureedge.net/cache"sv;
         constexpr std::string_view s_Source_WingetCommunityDefault_Data = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv;
         constexpr std::string_view s_Source_WingetCommunityDefault_Identifier = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv;
 
@@ -42,6 +43,7 @@ namespace AppInstaller::Repository
 
         constexpr std::string_view s_Source_DesktopFrameworks_Name = "microsoft.builtin.desktop.frameworks"sv;
         constexpr std::string_view s_Source_DesktopFrameworks_Arg = "https://cdn.winget.microsoft.com/platform"sv;
+        constexpr std::string_view s_Source_DesktopFrameworks_AlternateArg = "https://winget.azureedge.net/platform"sv;
         constexpr std::string_view s_Source_DesktopFrameworks_Data = "Microsoft.Winget.Platform.Source_8wekyb3d8bbwe"sv;
         constexpr std::string_view s_Source_DesktopFrameworks_Identifier = "Microsoft.Winget.Platform.Source_8wekyb3d8bbwe"sv;
 
@@ -284,6 +286,10 @@ namespace AppInstaller::Repository
             details.Name = s_Source_WingetCommunityDefault_Name;
             details.Type = Microsoft::PreIndexedPackageSourceFactory::Type();
             details.Arg = s_Source_WingetCommunityDefault_Arg;
+            if (Settings::User().Get<Settings::Setting::NetworkWingetAlternateSourceURL>())
+            {
+                details.AlternateArg = s_Source_WingetCommunityDefault_AlternateArg;
+            }
             details.Data = s_Source_WingetCommunityDefault_Data;
             details.Identifier = s_Source_WingetCommunityDefault_Identifier;
             details.TrustLevel = SourceTrustLevel::Trusted | SourceTrustLevel::StoreOrigin;
@@ -325,6 +331,7 @@ namespace AppInstaller::Repository
             details.Name = s_Source_DesktopFrameworks_Name;
             details.Type = Microsoft::PreIndexedPackageSourceFactory::Type();
             details.Arg = s_Source_DesktopFrameworks_Arg;
+            details.AlternateArg = s_Source_DesktopFrameworks_AlternateArg;
             details.Data = s_Source_DesktopFrameworks_Data;
             details.Identifier = s_Source_DesktopFrameworks_Identifier;
             details.TrustLevel = SourceTrustLevel::Trusted | SourceTrustLevel::StoreOrigin;
@@ -354,7 +361,7 @@ namespace AppInstaller::Repository
             }
             else
             {
-                AICLI_LOG(Repo, Info, << "GetCurrentSourceRefs: Source named '" << s.Name << "' from origin " << ToString(s.Origin) << " is hidden and is dropped.");
+                AICLI_LOG(Repo, Verbose, << "GetCurrentSourceRefs: Source named '" << s.Name << "' from origin " << ToString(s.Origin) << " is hidden and is dropped.");
             }
         }