From 428ffb50527fbb20e39443e13cae2cfa7ec6cdcc Mon Sep 17 00:00:00 2001 From: JohnMcPMS Date: Thu, 16 Feb 2023 11:48:20 -0800 Subject: [PATCH] Add alternate url support for some predefined sources (#2970) Adds an alternate base URL for the sources behind `cdn.winget.microsoft.com`. In the event that the files hosted there are inaccessible or corrupted, the alternate URL will be used instead. This mechanism does rely on the fact that these are basically two names for the exact same set of data, which is true as the alternate is the upstream store for the CDN itself. Additionally, we now require preindexed packaged sources to contain the manifest hash, rather than just using it if present. Finally, this change adds a setting that can be used to disable the alternate URL for the `winget` source. I intentionally did not add it to the schema or documentation, as I don't think it is something that should actually be set by anyone unless there is an explicit reason to do so (such as for testing purposes). --- .../Public/winget/UserSettings.h | 2 + src/AppInstallerCommonCore/UserSettings.cpp | 1 + .../PreIndexedPackageSourceFactory.cpp | 112 ++++++++++++++---- .../Microsoft/SQLiteIndexSource.cpp | 35 +++++- .../Microsoft/SQLiteIndexSource.h | 6 +- .../Public/winget/RepositorySource.h | 4 + src/AppInstallerRepositoryCore/SourceList.cpp | 9 +- 7 files changed, 139 insertions(+), 30 deletions(-) diff --git a/src/AppInstallerCommonCore/Public/winget/UserSettings.h b/src/AppInstallerCommonCore/Public/winget/UserSettings.h index 508be5fe81..22ef9fd602 100644 --- a/src/AppInstallerCommonCore/Public/winget/UserSettings.h +++ b/src/AppInstallerCommonCore/Public/winget/UserSettings.h @@ -89,6 +89,7 @@ namespace AppInstaller::Settings // Network NetworkDownloader, NetworkDOProgressTimeoutInSeconds, + NetworkWingetAlternateSourceURL, // Logging LoggingLevelPreference, // Uninstall behavior @@ -160,6 +161,7 @@ namespace AppInstaller::Settings // Network SETTINGMAPPING_SPECIALIZATION(Setting::NetworkDownloader, std::string, InstallerDownloader, InstallerDownloader::Default, ".network.downloader"sv); SETTINGMAPPING_SPECIALIZATION(Setting::NetworkDOProgressTimeoutInSeconds, uint32_t, std::chrono::seconds, 60s, ".network.doProgressTimeoutInSeconds"sv); + SETTINGMAPPING_SPECIALIZATION(Setting::NetworkWingetAlternateSourceURL, bool, bool, true, ".network.enableWingetAlternateSourceURL"sv); // Debug SETTINGMAPPING_SPECIALIZATION(Setting::EnableSelfInitiatedMinidump, bool, bool, false, ".debugging.enableSelfInitiatedMinidump"sv); // Logging diff --git a/src/AppInstallerCommonCore/UserSettings.cpp b/src/AppInstallerCommonCore/UserSettings.cpp index 2299e30501..29b837ab4d 100644 --- a/src/AppInstallerCommonCore/UserSettings.cpp +++ b/src/AppInstallerCommonCore/UserSettings.cpp @@ -268,6 +268,7 @@ namespace AppInstaller::Settings WINGET_VALIDATE_PASS_THROUGH(InstallIgnoreWarnings) WINGET_VALIDATE_PASS_THROUGH(DisableInstallNotes) WINGET_VALIDATE_PASS_THROUGH(UninstallPurgePortablePackage) + WINGET_VALIDATE_PASS_THROUGH(NetworkWingetAlternateSourceURL) WINGET_VALIDATE_SIGNATURE(PortablePackageUserRoot) { diff --git a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp index b75f8f8aaa..8dee1a7270 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/PreIndexedPackageSourceFactory.cpp @@ -21,19 +21,82 @@ namespace AppInstaller::Repository::Microsoft // TODO: This being hard coded to force using the Public directory name is not ideal. static constexpr std::string_view s_PreIndexedPackageSourceFactory_IndexFilePath = "Public\\index.db"sv; - // Construct the package location from the given details. - // Currently expects that the arg is an https uri pointing to the root of the data. - std::string GetPackageLocation(const SourceDetails& details) + struct PreIndexedPackageInfo { - THROW_HR_IF(E_INVALIDARG, details.Arg.empty()); - std::string result = details.Arg; - if (result.back() != '/') + template + PreIndexedPackageInfo(const SourceDetails& details, LocationCheck&& locationCheck) { - result += '/'; + // Get both locations to force the alternate location check + m_packageLocation = GetPrimaryPackageLocation(details); + locationCheck(m_packageLocation); + + std::string alternateLocation = GetAlternatePackageLocation(details); + if (!alternateLocation.empty()) + { + locationCheck(alternateLocation); + } + + // Try getting the primary location's info + HRESULT primaryHR = S_OK; + + try + { + m_msixInfo = std::make_unique(m_packageLocation); + return; + } + catch (...) + { + if (alternateLocation.empty()) + { + throw; + } + primaryHR = LOG_CAUGHT_EXCEPTION_MSG("PreIndexedPackageInfo failed on primary location"); + } + + // Try alternate location + m_packageLocation = std::move(alternateLocation); + + try + { + m_msixInfo = std::make_unique(m_packageLocation); + return; + } + CATCH_LOG_MSG("PreIndexedPackageInfo failed on alternate location"); + + THROW_HR(primaryHR); } - result += s_PreIndexedPackageSourceFactory_PackageFileName; - return result; - } + + const std::string& PackageLocation() const { return m_packageLocation; } + Msix::MsixInfo& MsixInfo() { return *m_msixInfo; } + + private: + std::string m_packageLocation; + std::unique_ptr m_msixInfo; + + std::string GetPrimaryPackageLocation(const SourceDetails& details) + { + THROW_HR_IF(E_INVALIDARG, details.Arg.empty()); + return GetPackageLocation(details.Arg); + } + + std::string GetAlternatePackageLocation(const SourceDetails& details) + { + return (details.AlternateArg.empty() ? std::string{} : GetPackageLocation(details.AlternateArg)); + } + + // Construct the package location from the given details. + // Currently expects that the arg is an https uri pointing to the root of the data. + std::string GetPackageLocation(const std::string& basePath) + { + std::string result = basePath; + if (result.back() != '/') + { + result += '/'; + } + result += s_PreIndexedPackageSourceFactory_PackageFileName; + return result; + } + }; // Gets the package family name from the details. std::string GetPackageFamilyNameFromDetails(const SourceDetails& details) @@ -75,16 +138,16 @@ namespace AppInstaller::Repository::Microsoft THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); } - std::string packageLocation = GetPackageLocation(details); - - THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); + PreIndexedPackageInfo packageInfo(details, [](const std::string& packageLocation) + { + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_NOT_SECURE, Utility::IsUrlRemote(packageLocation) && !Utility::IsUrlSecure(packageLocation)); + }); - AICLI_LOG(Repo, Info, << "Initializing source from: " << details.Name << " => " << packageLocation); + AICLI_LOG(Repo, Info, << "Initializing source from: " << details.Name << " => " << packageInfo.PackageLocation()); - Msix::MsixInfo packageInfo(packageLocation); - THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.GetIsBundle()); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); - auto fullName = packageInfo.GetPackageFullName(); + auto fullName = packageInfo.MsixInfo().GetPackageFullName(); AICLI_LOG(Repo, Info, << "Found package full name: " << details.Name << " => " << fullName); details.Data = Msix::GetPackageFamilyNameFromFullName(fullName); @@ -96,7 +159,7 @@ namespace AppInstaller::Repository::Microsoft return false; } - return UpdateInternal(packageLocation, packageInfo, details, progress); + return UpdateInternal(packageInfo.PackageLocation(), packageInfo.MsixInfo(), details, progress); } bool Update(const SourceDetails& details, IProgressCallback& progress) override final @@ -143,15 +206,14 @@ namespace AppInstaller::Repository::Microsoft { THROW_HR_IF(E_INVALIDARG, details.Type != PreIndexedPackageSourceFactory::Type()); - std::string packageLocation = GetPackageLocation(details); - Msix::MsixInfo packageInfo(packageLocation); + PreIndexedPackageInfo packageInfo(details, [](const std::string&){}); // The package should not be a bundle - THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.GetIsBundle()); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_PACKAGE_IS_BUNDLE, packageInfo.MsixInfo().GetIsBundle()); // Ensure that family name has not changed THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, - GetPackageFamilyNameFromDetails(details) != Msix::GetPackageFamilyNameFromFullName(packageInfo.GetPackageFullName())); + GetPackageFamilyNameFromDetails(details) != Msix::GetPackageFamilyNameFromFullName(packageInfo.MsixInfo().GetPackageFullName())); if (progress.IsCancelled()) { @@ -165,7 +227,7 @@ namespace AppInstaller::Repository::Microsoft return false; } - return UpdateInternal(packageLocation, packageInfo, details, progress); + return UpdateInternal(packageInfo.PackageLocation(), packageInfo.MsixInfo(), details, progress); } }; @@ -219,7 +281,7 @@ namespace AppInstaller::Repository::Microsoft // We didn't use to store the source identifier, so we compute it here in case it's // missing from the details. m_details.Identifier = GetPackageFamilyNameFromDetails(m_details); - return std::make_shared(m_details, std::move(index), std::move(lock)); + return std::make_shared(m_details, std::move(index), std::move(lock), false, true); } private: @@ -394,7 +456,7 @@ namespace AppInstaller::Repository::Microsoft // We didn't use to store the source identifier, so we compute it here in case it's // missing from the details. m_details.Identifier = GetPackageFamilyNameFromDetails(m_details); - return std::make_shared(m_details, std::move(index), std::move(lock)); + return std::make_shared(m_details, std::move(index), std::move(lock), false, true); } private: diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp index 58b737c851..6d390afe05 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.cpp @@ -74,13 +74,37 @@ namespace AppInstaller::Repository::Microsoft THROW_HR_IF(E_NOT_SET, !relativePathOpt); std::optional manifestHashString = source->GetIndex().GetPropertyByManifestId(m_manifestId, PackageVersionProperty::ManifestSHA256Hash); + THROW_HR_IF(APPINSTALLER_CLI_ERROR_SOURCE_DATA_INTEGRITY_FAILURE, source->RequireManifestHash() && !manifestHashString); + SHA256::HashBuffer manifestSHA256; if (manifestHashString) { manifestSHA256 = SHA256::ConvertToBytes(manifestHashString.value()); } - return GetManifestFromArgAndRelativePath(source->GetDetails().Arg, relativePathOpt.value(), manifestSHA256); + // Try the primary location + HRESULT primaryHR = S_OK; + try + { + return GetManifestFromArgAndRelativePath(source->GetDetails().Arg, relativePathOpt.value(), manifestSHA256); + } + catch (...) + { + if (source->GetDetails().AlternateArg.empty()) + { + throw; + } + primaryHR = LOG_CAUGHT_EXCEPTION_MSG("GetManifest failed on primary location"); + } + + // Try alternate location + try + { + return GetManifestFromArgAndRelativePath(source->GetDetails().AlternateArg, relativePathOpt.value(), manifestSHA256); + } + CATCH_LOG_MSG("GetManifest failed on alternate location"); + + THROW_HR(primaryHR); } Source GetSource() const override @@ -345,8 +369,13 @@ namespace AppInstaller::Repository::Microsoft }; } - SQLiteIndexSource::SQLiteIndexSource(const SourceDetails& details, SQLiteIndex&& index, Synchronization::CrossProcessReaderWriteLock&& lock, bool isInstalledSource) : - m_details(details), m_lock(std::move(lock)), m_isInstalled(isInstalledSource), m_index(std::move(index)) + SQLiteIndexSource::SQLiteIndexSource( + const SourceDetails& details, + SQLiteIndex&& index, + Synchronization::CrossProcessReaderWriteLock&& lock, + bool isInstalledSource, + bool requireManifestHash) : + m_details(details), m_lock(std::move(lock)), m_isInstalled(isInstalledSource), m_index(std::move(index)), m_requireManifestHash(requireManifestHash) { } diff --git a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.h b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.h index 5f1b4f18e5..b1191940a3 100644 --- a/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.h +++ b/src/AppInstallerRepositoryCore/Microsoft/SQLiteIndexSource.h @@ -17,7 +17,8 @@ namespace AppInstaller::Repository::Microsoft const SourceDetails& details, SQLiteIndex&& index, Synchronization::CrossProcessReaderWriteLock&& lock = {}, - bool isInstalledSource = false); + bool isInstalledSource = false, + bool requireManifestHash = false); SQLiteIndexSource(const SQLiteIndexSource&) = delete; SQLiteIndexSource& operator=(const SQLiteIndexSource&) = delete; @@ -45,11 +46,14 @@ namespace AppInstaller::Repository::Microsoft // Determines if the other source refers to the same as this. bool IsSame(const SQLiteIndexSource* other) const; + bool RequireManifestHash() const { return m_requireManifestHash; } + private: std::shared_ptr NonConstSharedFromThis() const; SourceDetails m_details; Synchronization::CrossProcessReaderWriteLock m_lock; + bool m_requireManifestHash; bool m_isInstalled; protected: diff --git a/src/AppInstallerRepositoryCore/Public/winget/RepositorySource.h b/src/AppInstallerRepositoryCore/Public/winget/RepositorySource.h index 435e2879c4..d4a003216d 100644 --- a/src/AppInstallerRepositoryCore/Public/winget/RepositorySource.h +++ b/src/AppInstallerRepositoryCore/Public/winget/RepositorySource.h @@ -121,6 +121,10 @@ namespace AppInstaller::Repository // The configuration of how the server certificate will be validated. Certificates::PinningConfiguration CertificatePinningConfiguration; + + // This value is used as an alternative to the `Arg` value if it is failing to function properly. + // The alternate location must point to identical data or inconsistencies may arise. + std::string AlternateArg; }; // Individual source agreement entry. Label will be highlighted in the display as the key of the agreement entry. diff --git a/src/AppInstallerRepositoryCore/SourceList.cpp b/src/AppInstallerRepositoryCore/SourceList.cpp index 33099861aa..afaae89b34 100644 --- a/src/AppInstallerRepositoryCore/SourceList.cpp +++ b/src/AppInstallerRepositoryCore/SourceList.cpp @@ -33,6 +33,7 @@ namespace AppInstaller::Repository constexpr std::string_view s_Source_WingetCommunityDefault_Name = "winget"sv; constexpr std::string_view s_Source_WingetCommunityDefault_Arg = "https://cdn.winget.microsoft.com/cache"sv; + constexpr std::string_view s_Source_WingetCommunityDefault_AlternateArg = "https://winget.azureedge.net/cache"sv; constexpr std::string_view s_Source_WingetCommunityDefault_Data = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; constexpr std::string_view s_Source_WingetCommunityDefault_Identifier = "Microsoft.Winget.Source_8wekyb3d8bbwe"sv; @@ -42,6 +43,7 @@ namespace AppInstaller::Repository constexpr std::string_view s_Source_DesktopFrameworks_Name = "microsoft.builtin.desktop.frameworks"sv; constexpr std::string_view s_Source_DesktopFrameworks_Arg = "https://cdn.winget.microsoft.com/platform"sv; + constexpr std::string_view s_Source_DesktopFrameworks_AlternateArg = "https://winget.azureedge.net/platform"sv; constexpr std::string_view s_Source_DesktopFrameworks_Data = "Microsoft.Winget.Platform.Source_8wekyb3d8bbwe"sv; constexpr std::string_view s_Source_DesktopFrameworks_Identifier = "Microsoft.Winget.Platform.Source_8wekyb3d8bbwe"sv; @@ -284,6 +286,10 @@ namespace AppInstaller::Repository details.Name = s_Source_WingetCommunityDefault_Name; details.Type = Microsoft::PreIndexedPackageSourceFactory::Type(); details.Arg = s_Source_WingetCommunityDefault_Arg; + if (Settings::User().Get()) + { + details.AlternateArg = s_Source_WingetCommunityDefault_AlternateArg; + } details.Data = s_Source_WingetCommunityDefault_Data; details.Identifier = s_Source_WingetCommunityDefault_Identifier; details.TrustLevel = SourceTrustLevel::Trusted | SourceTrustLevel::StoreOrigin; @@ -325,6 +331,7 @@ namespace AppInstaller::Repository details.Name = s_Source_DesktopFrameworks_Name; details.Type = Microsoft::PreIndexedPackageSourceFactory::Type(); details.Arg = s_Source_DesktopFrameworks_Arg; + details.AlternateArg = s_Source_DesktopFrameworks_AlternateArg; details.Data = s_Source_DesktopFrameworks_Data; details.Identifier = s_Source_DesktopFrameworks_Identifier; details.TrustLevel = SourceTrustLevel::Trusted | SourceTrustLevel::StoreOrigin; @@ -354,7 +361,7 @@ namespace AppInstaller::Repository } else { - AICLI_LOG(Repo, Info, << "GetCurrentSourceRefs: Source named '" << s.Name << "' from origin " << ToString(s.Origin) << " is hidden and is dropped."); + AICLI_LOG(Repo, Verbose, << "GetCurrentSourceRefs: Source named '" << s.Name << "' from origin " << ToString(s.Origin) << " is hidden and is dropped."); } }