From d50d48e82723fd9358f06d8205f8183fd06a5375 Mon Sep 17 00:00:00 2001 From: donid Date: Tue, 27 Jul 2021 08:34:58 +0200 Subject: [PATCH] fixed download problems for some URLs - bumped version to 0.2.1 --- src/WingetRepoBrowser/MainForm.Designer.cs | 2 +- src/WingetRepoBrowser/MainForm.cs | 33 +++--------- src/WingetRepoBrowser/ManifestPackageVM.cs | 2 +- .../Properties/licenses.licx | 8 +-- .../WingetRepoBrowser.csproj | 16 +++--- src/WingetRepoBrowserCore/Helpers.cs | 4 +- .../InstallerDownloader.cs | 52 +++++++++++-------- .../WingetRepoBrowserCore.csproj | 2 +- .../DownloadTests.cs | 33 ++++++++++++ src/WingetRepoBrowserCoreTests/Program.cs | 5 +- .../WingetRepoBrowserCoreTests.csproj | 7 +-- src/WingetRepoBrowserCoreTests/YamlTest.cs | 3 +- 12 files changed, 97 insertions(+), 70 deletions(-) create mode 100644 src/WingetRepoBrowserCoreTests/DownloadTests.cs diff --git a/src/WingetRepoBrowser/MainForm.Designer.cs b/src/WingetRepoBrowser/MainForm.Designer.cs index 9948375..547c4e6 100644 --- a/src/WingetRepoBrowser/MainForm.Designer.cs +++ b/src/WingetRepoBrowser/MainForm.Designer.cs @@ -771,7 +771,7 @@ private void InitializeComponent() this.ClientSize = new System.Drawing.Size(1012, 639); this.Controls.Add(this.layoutControl1); this.Name = "MainForm"; - this.Text = "WingetRepo Browser 0.2.0"; + this.Text = "WingetRepo Browser 0.2.1"; ((System.ComponentModel.ISupportInitialize)(this.gridView2)).EndInit(); ((System.ComponentModel.ISupportInitialize)(this.gridControl1)).EndInit(); ((System.ComponentModel.ISupportInitialize)(this.gridView1)).EndInit(); diff --git a/src/WingetRepoBrowser/MainForm.cs b/src/WingetRepoBrowser/MainForm.cs index d82075c..0785d82 100644 --- a/src/WingetRepoBrowser/MainForm.cs +++ b/src/WingetRepoBrowser/MainForm.cs @@ -324,34 +324,15 @@ private static List FindNewDownloads(YamlFileHelper yamlFileHelper, string idFileFolder = Path.GetDirectoryName(idFilePath); string versionFolder = Path.Combine(idFileFolder, ConvertVersionToDirectoryName(manifestPackage.Version)); // illegal chars in version shouldn't be a problem, because yaml files are stored in folders with version as name bool exists = versionsToIgnoreDownload.Any(v => v == manifestPackage.Version) || Directory.Exists(versionFolder); - //if (manifestPackage.Version == "latest" && exists) - if (false)//TODO: the following code crashes when downloaded yaml-files are not version 1.0.0 + if (!exists) { - string downloadedYamlFilePath = Path.Combine(versionFolder, "latest.yaml"); - ManifestPackage_1_0_0 downloadedManifestPackage = yamlFileHelper.ReadYamlFile(downloadedYamlFilePath).Manifest; - //TODO test all installers when winget supports multiple installers - if (manifestPackage.Installers[0].Sha256 != downloadedManifestPackage.Installers[0].InstallerSha256) + NewDownload dl = new NewDownload() { - FileInfo fi = new FileInfo(downloadedYamlFilePath); - string versionSuffix = fi.LastWriteTime.ToString("_yyyy-MM-dd"); - downloadedManifestPackage.PackageVersion += versionSuffix; - yamlFileHelper.WriteYamlFile(downloadedYamlFilePath, downloadedManifestPackage); - Directory.Move(versionFolder, versionFolder + versionSuffix); - exists = Directory.Exists(versionFolder); - } - } - if (manifestPackage.Version != "latest")//TODO!!!! - { - if (!exists) - { - NewDownload dl = new NewDownload() - { - MultiFileYaml = manifestPackage.MultiFileYaml, - VersionFolder = versionFolder, - IdFilePath = idFilePath - }; - result.Add(dl); - } + MultiFileYaml = manifestPackage.MultiFileYaml, + VersionFolder = versionFolder, + IdFilePath = idFilePath + }; + result.Add(dl); } } return result; diff --git a/src/WingetRepoBrowser/ManifestPackageVM.cs b/src/WingetRepoBrowser/ManifestPackageVM.cs index 5332596..fec849a 100644 --- a/src/WingetRepoBrowser/ManifestPackageVM.cs +++ b/src/WingetRepoBrowser/ManifestPackageVM.cs @@ -97,7 +97,7 @@ static string SafeJoin(string separator, IEnumerable arr) private string GetDebuggerDisplay() { - return ToString(); + return Id; } diff --git a/src/WingetRepoBrowser/Properties/licenses.licx b/src/WingetRepoBrowser/Properties/licenses.licx index 269fc4d..e1521de 100644 --- a/src/WingetRepoBrowser/Properties/licenses.licx +++ b/src/WingetRepoBrowser/Properties/licenses.licx @@ -1,4 +1,4 @@ -DevExpress.XtraEditors.TextEdit, DevExpress.XtraEditors.v21.1, Version=21.1.3.0, Culture=neutral, PublicKeyToken=b88d1754d700e49a -DevExpress.XtraEditors.Repository.RepositoryItemButtonEdit, DevExpress.XtraEditors.v21.1, Version=21.1.3.0, Culture=neutral, PublicKeyToken=b88d1754d700e49a -DevExpress.XtraGrid.GridControl, DevExpress.XtraGrid.v21.1, Version=21.1.3.0, Culture=neutral, PublicKeyToken=b88d1754d700e49a -DevExpress.XtraLayout.LayoutControl, DevExpress.XtraLayout.v21.1, Version=21.1.3.0, Culture=neutral, PublicKeyToken=b88d1754d700e49a +DevExpress.XtraEditors.Repository.RepositoryItemButtonEdit, DevExpress.XtraEditors.v21.1, Version=21.1.4.0, Culture=neutral, PublicKeyToken=b88d1754d700e49a +DevExpress.XtraGrid.GridControl, DevExpress.XtraGrid.v21.1, Version=21.1.4.0, Culture=neutral, PublicKeyToken=b88d1754d700e49a +DevExpress.XtraLayout.LayoutControl, DevExpress.XtraLayout.v21.1, Version=21.1.4.0, Culture=neutral, PublicKeyToken=b88d1754d700e49a +DevExpress.XtraEditors.TextEdit, DevExpress.XtraEditors.v21.1, Version=21.1.4.0, Culture=neutral, PublicKeyToken=b88d1754d700e49a diff --git a/src/WingetRepoBrowser/WingetRepoBrowser.csproj b/src/WingetRepoBrowser/WingetRepoBrowser.csproj index 4744a67..7e35800 100644 --- a/src/WingetRepoBrowser/WingetRepoBrowser.csproj +++ b/src/WingetRepoBrowser/WingetRepoBrowser.csproj @@ -67,14 +67,14 @@ Properties\app.manifest - - - - - - - - + + + + + + + + diff --git a/src/WingetRepoBrowserCore/Helpers.cs b/src/WingetRepoBrowserCore/Helpers.cs index 3a6b3c1..0839077 100644 --- a/src/WingetRepoBrowserCore/Helpers.cs +++ b/src/WingetRepoBrowserCore/Helpers.cs @@ -44,7 +44,9 @@ public static string[] GetVersionsToIgnoreDownload(string idFilePath) return result; } - + //TODO: Retry if this occurs: + // happens regularly on a network share (due to malware scanner?) + // Error: Calculating Sha256-Hash: I/O Exception: The process cannot access the file '...\Firefox\89.0.1\x86_en-US\Firefox Setup 89.0.1.msi' because it is being used by another process. public static CalculateFileHashResult CalculateSha256HashFromFile(string downloadFilePath) { using (SHA256 mySHA256 = SHA256.Create()) diff --git a/src/WingetRepoBrowserCore/InstallerDownloader.cs b/src/WingetRepoBrowserCore/InstallerDownloader.cs index dbf8037..da5e999 100644 --- a/src/WingetRepoBrowserCore/InstallerDownloader.cs +++ b/src/WingetRepoBrowserCore/InstallerDownloader.cs @@ -10,37 +10,36 @@ public class InstallerDownloader : IDisposable { // newer, but very specific: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0"; // or "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)" - const string cUserAgent = "Mozilla/4.0 (Windows NT 10.0; Win64; x64)"; + //const string cUserAgent = "Mozilla/4.0 (Windows NT 10.0; Win64; x64)"; + public const string cUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0"; WebClient _webClient; public InstallerDownloader() { - _webClient = new WebClient(); + _webClient = new MyWebClient(); //without this: "https://fossies.org/windows/misc/audacity-win-2.4.1.exe" throws WebException 403 forbidden _webClient.Headers.Add("user-agent", cUserAgent); - } public string GetFileNameFromUrl(string url, out Uri responseUri) { responseUri = null; HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url); - req.Method = "HEAD"; - //without this: "https://fossies.org/windows/misc/audacity-win-2.4.1.exe" -> WebException 403 forbidden - //req.Headers.Add("user-agent" -> System.ArgumentException: 'The 'user-agent' header must be modified using the appropriate property or method. Parameter name: name' + req.Method = "HEAD"; //without this: "https://fossies.org/windows/misc/audacity-win-2.4.1.exe" -> WebException 403 forbidden + req.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip; // avoid 404 for "https://dl.pstmn.io/download/version/8.9.0/win32" // https://github.com/HandBrake/HandBrake/releases/download/1.3.2/HandBrake-1.3.2-x86_64-Win_GUI.exe - // this and other github release URIs throws 403 even with useragent, but webclient.downloadfileworks + // this and other github release URIs throws 403 even with useragent, but webclient.downloadfile works - if (req.Host.ToLower() != "sourceforge.net")//HACK!! sourceforge does not remove 'download' at the end of ResponseUri when we provide UserAgent + if (!string.Equals(req.Host, "sourceforge.net", StringComparison.OrdinalIgnoreCase))//HACK!! sourceforge does not remove 'download' at the end of ResponseUri when we provide UserAgent { req.UserAgent = cUserAgent; } - WebResponse resp = null; + WebResponse response; try { - resp = req.GetResponse(); + response = req.GetResponse(); } catch (WebException /*ex*/) { @@ -51,30 +50,25 @@ public string GetFileNameFromUrl(string url, out Uri responseUri) } return fileNameFromUrl; } - using (resp) + + using (response) { - responseUri = resp.ResponseUri; + responseUri = response.ResponseUri; // Try to extract the filename from the Content-Disposition header - string contDisp = resp.Headers["Content-Disposition"]; + string contDisp = response.Headers["Content-Disposition"]; string filenameFromCd = GetFilenameFromContentDisposition(contDisp); if (filenameFromCd != null) { return filenameFromCd; } - string location = resp.Headers["Location"]; + string location = response.Headers["Location"]; if (location != null) { return Path.GetFileName(location); } - //string fileNameFromUrl = Path.GetFileName(url); - //if (fileNameFromUrl.Contains('?') || fileNameFromUrl.Contains('=') ) - //{ - // return Path.GetFileName(resp.ResponseUri.ToString()); - //} - - string potentialFilename = Path.GetFileName(resp.ResponseUri.ToString()); + string potentialFilename = Path.GetFileName(response.ResponseUri.ToString()); int foundIndex = potentialFilename.IndexOfAny(new[] { '?' }); if (foundIndex == -1) { @@ -117,8 +111,8 @@ static string GetFilenameFromContentDisposition(string contDisp) static string GetFilenameFromContentDispositionWorkaround(string contDisp) { string[] parts = contDisp.Split(';'); - string filenamePart=parts.Select(i => i.Trim()).FirstOrDefault(i=>i.StartsWith("filename=",StringComparison.InvariantCultureIgnoreCase)); - if (filenamePart==null) + string filenamePart = parts.Select(i => i.Trim()).FirstOrDefault(i => i.StartsWith("filename=", StringComparison.InvariantCultureIgnoreCase)); + if (filenamePart == null) { return null; } @@ -153,4 +147,16 @@ public void Dispose() ((IDisposable)_webClient).Dispose(); } } + + class MyWebClient : WebClient + { + protected override WebRequest GetWebRequest(Uri address) + { + HttpWebRequest request = (HttpWebRequest)base.GetWebRequest(address); + request.UserAgent = InstallerDownloader.cUserAgent; // avoid 403 for some URLs - see InstallerDownloader + request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip; // avoid 404 for "https://dl.pstmn.io/download/version/8.9.0/win32" + return request; + } + } + } diff --git a/src/WingetRepoBrowserCore/WingetRepoBrowserCore.csproj b/src/WingetRepoBrowserCore/WingetRepoBrowserCore.csproj index 1cf7ade..bce5943 100644 --- a/src/WingetRepoBrowserCore/WingetRepoBrowserCore.csproj +++ b/src/WingetRepoBrowserCore/WingetRepoBrowserCore.csproj @@ -7,7 +7,7 @@ - + diff --git a/src/WingetRepoBrowserCoreTests/DownloadTests.cs b/src/WingetRepoBrowserCoreTests/DownloadTests.cs new file mode 100644 index 0000000..fb19246 --- /dev/null +++ b/src/WingetRepoBrowserCoreTests/DownloadTests.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading.Tasks; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace WingetRepoBrowserCoreTests +{ + [TestClass] + + public class DownloadTests + { + [TestMethod] + + public void TestWebRequest() + { + // https://stackoverflow.com/questions/68529103/why-does-webrequest-fail-with-404/68529810?noredirect=1#comment121112114_68529810 + + //string url = "https://dl.pstmn.io/download/version/8.9.0/win64"; // works + //string url = "https://dl.pstmn.io/download/version/8.9.0/win32"; //404 here, but works in FF or Chrome + string url = "https://download.filezilla-project.org/client/FileZilla_3.55.0_win32-setup.exe"; // 403 + //string url = "https://sourceforge.net/projects/smplayer/files/SMPlayer/21.1.0/smplayer-21.1.0-win32.exe/download";// do not use UserAgent when trying to get filename + HttpWebRequest req = (HttpWebRequest)WebRequest.Create(url); + req.Method = "HEAD"; + req.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;// for postman!! avoid 404 + req.UserAgent = @"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0"; + WebResponse response = req.GetResponse(); + } + } +} diff --git a/src/WingetRepoBrowserCoreTests/Program.cs b/src/WingetRepoBrowserCoreTests/Program.cs index 48d76b9..c922c63 100644 --- a/src/WingetRepoBrowserCoreTests/Program.cs +++ b/src/WingetRepoBrowserCoreTests/Program.cs @@ -1,4 +1,7 @@ -using WingetRepoBrowserCore; +using System.Collections.Generic; +using System.IO; + +using WingetRepoBrowserCore; using WingetRepoBrowserTests; diff --git a/src/WingetRepoBrowserCoreTests/WingetRepoBrowserCoreTests.csproj b/src/WingetRepoBrowserCoreTests/WingetRepoBrowserCoreTests.csproj index 823ed3b..37595c0 100644 --- a/src/WingetRepoBrowserCoreTests/WingetRepoBrowserCoreTests.csproj +++ b/src/WingetRepoBrowserCoreTests/WingetRepoBrowserCoreTests.csproj @@ -43,6 +43,7 @@ + @@ -61,13 +62,13 @@ 16.10.0 - 2.2.4 + 2.2.5 - 2.2.4 + 2.2.5 - 11.1.1 + 11.2.1 diff --git a/src/WingetRepoBrowserCoreTests/YamlTest.cs b/src/WingetRepoBrowserCoreTests/YamlTest.cs index 35872d9..d54bf7a 100644 --- a/src/WingetRepoBrowserCoreTests/YamlTest.cs +++ b/src/WingetRepoBrowserCoreTests/YamlTest.cs @@ -1,4 +1,5 @@ -using System.IO; +using System.Collections.Generic; +using System.IO; using Microsoft.VisualStudio.TestTools.UnitTesting;