diff --git a/src/Shared.CLI/BaseTool.cs b/src/Shared.CLI/BaseTool.cs index 46b22105..88422cb2 100644 --- a/src/Shared.CLI/BaseTool.cs +++ b/src/Shared.CLI/BaseTool.cs @@ -6,7 +6,7 @@ namespace Microsoft.CST.OpenSource using CommandLine.Text; using Helpers; using Microsoft.CST.OpenSource.Shared; - using OssGadget.CLI.Options; + using OssGadget.Options; using PackageManagers; using System; using System.Collections.Generic; @@ -14,6 +14,7 @@ namespace Microsoft.CST.OpenSource using System.Linq; using System.Reflection; using System.Text.RegularExpressions; + using System.Threading.Tasks; using static Microsoft.CST.OpenSource.Shared.OutputBuilderFactory; public class BaseTool : OssGadgetLib where T: BaseToolOptions @@ -30,6 +31,11 @@ public BaseTool(ProjectManagerFactory projectManagerFactory) : base(projectManag public BaseTool() : base() {} + public async Task RunAsync(T opt) + { + throw new NotImplementedException(); + } + /// /// Formulates the help text for each derived tool /// diff --git a/src/Shared.CLI/Options/BaseToolOptions.cs b/src/Shared.CLI/Options/BaseToolOptions.cs index 42261f75..fe9de91d 100644 --- a/src/Shared.CLI/Options/BaseToolOptions.cs +++ b/src/Shared.CLI/Options/BaseToolOptions.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft Corporation. Licensed under the MIT License. -namespace Microsoft.CST.OpenSource.OssGadget.CLI.Options; +namespace Microsoft.CST.OpenSource.OssGadget.Options; using CommandLine; using CommandLine.Text; @@ -8,34 +8,4 @@ namespace Microsoft.CST.OpenSource.OssGadget.CLI.Options; public class BaseToolOptions { - [Usage()] - public static IEnumerable Examples - { - get - { - return new List() { - new Example("Download the given package", - new DownloadToolOptions { Targets = new List() {"[options]", "package-url..." } })}; - } - } - - [Option('x', "download-directory", Required = false, Default = ".", - HelpText = "the directory to download the package to.")] - public string DownloadDirectory { get; set; } = "."; - - [Option('m', "download-metadata-only", Required = false, Default = false, - HelpText = "download only the package metadata, not the package.")] - public bool DownloadMetadataOnly { get; set; } - - [Option('e', "extract", Required = false, Default = false, - HelpText = "Extract the package contents")] - public bool Extract { get; set; } - - [Value(0, Required = true, - HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze - public IEnumerable? Targets { get; set; } - - [Option('c', "use-cache", Required = false, Default = false, - HelpText = "do not download the package if it is already present in the destination directory.")] - public bool UseCache { get; set; } } diff --git a/src/Shared.CLI/Options/CharacteristicToolOptions.cs b/src/Shared.CLI/Options/CharacteristicToolOptions.cs new file mode 100644 index 00000000..7deb72a6 --- /dev/null +++ b/src/Shared.CLI/Options/CharacteristicToolOptions.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +namespace Microsoft.CST.OpenSource.OssGadget.Options; + +using CodeAnalysis.Sarif; +using CommandLine; +using CommandLine.Text; +using System.Collections.Generic; + +[Verb("characteristic", HelpText = "Run risk calculator tool")] +public class CharacteristicToolOptions : BaseToolOptions +{ + [Usage()] + public static IEnumerable Examples + { + get + { + return new List() { + new Example("Find the characterstics for the given package", + new CharacteristicToolOptions() { Targets = new List() {"[options]", "package-url..." } })}; + } + } + + [Option('r', "custom-rule-directory", Required = false, Default = null, + HelpText = "load rules from the specified directory.")] + public string? CustomRuleDirectory { get; set; } + + [Option("disable-default-rules", Required = false, Default = false, + HelpText = "do not load default, built-in rules.")] + public bool DisableDefaultRules { get; set; } + + [Option('d', "download-directory", Required = false, Default = ".", + HelpText = "the directory to download the package to.")] + public string DownloadDirectory { get; set; } = "."; + + [Option('f', "format", Required = false, Default = "text", + HelpText = "specify the output format(text|sarifv1|sarifv2)")] + public string Format { get; set; } = "text"; + + [Option('o', "output-file", Required = false, Default = "", + HelpText = "send the command output to a file instead of stdout")] + public string OutputFile { get; set; } = ""; + + [Value(0, Required = true, + HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze + public IEnumerable? Targets { get; set; } + + [Option('c', "use-cache", Required = false, Default = false, + HelpText = "do not download the package if it is already present in the destination directory.")] + public bool UseCache { get; set; } + + [Option('x', "exclude", Required = false, + HelpText = "exclude files or paths which match provided glob patterns.")] + public string FilePathExclusions { get; set; } = ""; + + [Option('b', "backtracking", Required = false, HelpText = "Use backtracking regex engine by default.")] + public bool EnableBacktracking { get; set; } = false; + + [Option('s', "single-threaded", Required = false, HelpText = "Use single-threaded analysis")] + public bool SingleThread { get; set; } = false; + + public bool AllowTagsInBuildFiles { get; set; } = true; + + public bool AllowDupTags { get; set; } = false; + + public FailureLevel SarifLevel { get; set; } = FailureLevel.Note; +} \ No newline at end of file diff --git a/src/Shared.CLI/Options/DetectBackdoorToolOptions.cs b/src/Shared.CLI/Options/DetectBackdoorToolOptions.cs new file mode 100644 index 00000000..72ddd607 --- /dev/null +++ b/src/Shared.CLI/Options/DetectBackdoorToolOptions.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +namespace Microsoft.CST.OpenSource.OssGadget.Options; + +using CommandLine; +using CommandLine.Text; +using System.Collections.Generic; + +[Verb("detect-backdoor", HelpText = "Run detect backdoor tool")] +public class DetectBackdoorToolOptions +{ + [Usage()] + public static IEnumerable Examples + { + get + { + return new List() { + new Example("Identify potential malware or backdoors in the given package", + new DetectBackdoorToolOptions { Targets = new List() {"[options]", "package-url..." } })}; + } + } + + [Option('d', "download-directory", Required = false, Default = ".", + HelpText = "the directory to download the package to.")] + public string DownloadDirectory { get; set; } = "."; + + [Option('f', "format", Required = false, Default = "text", + HelpText = "specify the output format(text|sarifv1|sarifv2)")] + public string Format { get; set; } = "text"; + + [Option('o', "output-file", Required = false, Default = "", + HelpText = "send the command output to a file instead of stdout")] + public string OutputFile { get; set; } = ""; + + [Value(0, Required = true, + HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze + public IEnumerable? Targets { get; set; } + + [Option('c', "use-cache", Required = false, Default = false, + HelpText = "do not download the package if it is already present in the destination directory.")] + public bool UseCache { get; set; } + + [Option('b', "backtracking", Required = false, HelpText = "Use backtracking engine by default.")] + public bool EnableBacktracking { get; set; } = false; + + [Option('s', "single-threaded", Required = false, HelpText = "Use single-threaded analysis")] + public bool SingleThread { get; set; } = false; +} \ No newline at end of file diff --git a/src/Shared.CLI/Options/DiffToolOptions.cs b/src/Shared.CLI/Options/DiffToolOptions.cs new file mode 100644 index 00000000..eea8961f --- /dev/null +++ b/src/Shared.CLI/Options/DiffToolOptions.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +namespace Microsoft.CST.OpenSource.OssGadget.Options; + +using CommandLine; +using CommandLine.Text; +using System; +using System.Collections.Generic; + +[Verb("diff", HelpText = "Run diff tool")] +public class DiffToolOptions : BaseToolOptions +{ + [Usage()] + public static IEnumerable Examples + { + get + { + return new List() { + new Example("Diff the given packages", + new DiffToolOptions { Targets = new List() {"[options]", "package-url package-url-2" } })}; + } + } + + [Option('d', "download-directory", Required = false, Default = null, + HelpText = "the directory to download the packages to.")] + public string? DownloadDirectory { get; set; } = null; + + [Option('c', "use-cache", Required = false, Default = false, + HelpText = "Do not download the package if it is already present in the destination directory and do not delete the package after processing.")] + public bool UseCache { get; set; } + + [Option('w', "crawl-archives", Required = false, Default = true, + HelpText = "Crawl into archives found in packages.")] + public bool CrawlArchives { get; set; } + + [Option('B', "context-before", Required = false, Default = 0, + HelpText = "Number of previous lines to give as context.")] + public int Before { get; set; } = 0; + + [Option('A', "context-after", Required = false, Default = 0, + HelpText = "Number of subsequent lines to give as context.")] + public int After { get; set; } = 0; + + [Option('C', "context", Required = false, Default = 0, + HelpText = "Number of lines to give as context. Overwrites Before and After options. -1 to print all.")] + public int Context { get; set; } = 0; + + [Option('a', "added-only", Required = false, Default = false, + HelpText = "Only show added lines (and requested context).")] + public bool AddedOnly { get; set; } = false; + + [Option('r', "removed-only", Required = false, Default = false, + HelpText = "Only show removed lines (and requested context).")] + public bool RemovedOnly { get; set; } = false; + + [Option('f', "format", Required = false, Default = "text", + HelpText = "Choose output format. (text|sarifv1|sarifv2)")] + public string Format { get; set; } = "text"; + + [Option('o', "output-location", Required = false, Default = null, + HelpText = "Output location. Don't specify for console output.")] + public string? OutputLocation { get; set; } = null; + + [Value(0, Required = true, + HelpText = "Exactly two Filenames or PackgeURL specifiers to analyze.", Hidden = true)] // capture all targets to analyze + public IEnumerable Targets { get; set; } = Array.Empty(); +} \ No newline at end of file diff --git a/src/Shared.CLI/Options/DownloadToolOptions.cs b/src/Shared.CLI/Options/DownloadToolOptions.cs index a8654efa..d04d9c9d 100644 --- a/src/Shared.CLI/Options/DownloadToolOptions.cs +++ b/src/Shared.CLI/Options/DownloadToolOptions.cs @@ -1,6 +1,5 @@ // Copyright (c) Microsoft Corporation. Licensed under the MIT License. - -namespace Microsoft.CST.OpenSource.OssGadget.CLI.Options; +namespace Microsoft.CST.OpenSource.OssGadget.Options; using CommandLine; using CommandLine.Text; diff --git a/src/Shared.CLI/Options/FindDomainSquatsToolOptions.cs b/src/Shared.CLI/Options/FindDomainSquatsToolOptions.cs new file mode 100644 index 00000000..a1639dd2 --- /dev/null +++ b/src/Shared.CLI/Options/FindDomainSquatsToolOptions.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +namespace Microsoft.CST.OpenSource.OssGadget.Options; + +using CommandLine; +using CommandLine.Text; +using System.Collections.Generic; + +[Verb("find-domain-squats", HelpText = "Run find-domain-squats tool")] +public class FindDomainSquatsToolOptions : BaseToolOptions +{ + [Usage()] + public static IEnumerable Examples + { + get + { + return new List() { + new Example("Find Squat Candidates for the Given Packages", + new FindDomainSquatsToolOptions { Targets = new List() {"[options]", "domains" } })}; + } + } + + [Option('o', "output-file", Required = false, Default = "", + HelpText = "send the command output to a file instead of stdout")] + public string OutputFile { get; set; } = ""; + + [Option('f', "format", Required = false, Default = "text", + HelpText = "specify the output format(text|sarifv1|sarifv2)")] + public string Format { get; set; } = "text"; + + [Option('q', "quiet", Required = false, Default = false, + HelpText = "Suppress console output.")] + public bool Quiet { get; set; } = false; + + [Option('s', "sleep-delay", Required = false, Default = 0, HelpText = "Number of ms to sleep between checks.")] + public int SleepDelay { get; set; } + + [Option('u', "unregistered", Required = false, Default = false, HelpText = "Don't show registered domains.")] + public bool Unregistered { get; set; } + + [Option('r', "registered", Required = false, Default = false, HelpText = "Don't show unregistered domains.")] + public bool Registered { get; set; } + + [Value(0, Required = true, + HelpText = "Domain(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze + public IEnumerable? Targets { get; set; } +} diff --git a/src/Shared.CLI/Options/FindSourceToolOptions.cs b/src/Shared.CLI/Options/FindSourceToolOptions.cs new file mode 100644 index 00000000..ecca5322 --- /dev/null +++ b/src/Shared.CLI/Options/FindSourceToolOptions.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +namespace Microsoft.CST.OpenSource.OssGadget.Options; + +using CommandLine; +using CommandLine.Text; +using System.Collections.Generic; + +[Verb("find-source", HelpText = "Run find-domain-squats tool")] +public class FindSourceToolOptions : BaseToolOptions +{ + [Usage()] + public static IEnumerable Examples + { + get + { + return new List() { + new Example("Find the source code repository for the given package", new FindSourceToolOptions { Targets = new List() {"[options]", "package-url..." } })}; + } + } + + [Option('f', "format", Required = false, Default = "text", + HelpText = "specify the output format(text|sarifv1|sarifv2)")] + public string Format { get; set; } = "text"; + + [Option('o', "output-file", Required = false, Default = "", + HelpText = "send the command output to a file instead of stdout")] + public string OutputFile { get; set; } = ""; + + [Option('S', "single", Required = false, Default = false, + HelpText = "Show only top possibility of the package source repositories. When using text format the *only* output will be the URL or empty string if error or not found.")] + public bool Single { get; set; } + + [Value(0, Required = true, + HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze + public IEnumerable? Targets { get; set; } +} diff --git a/src/Shared.CLI/Options/FindSquatsToolOptions.cs b/src/Shared.CLI/Options/FindSquatsToolOptions.cs new file mode 100644 index 00000000..a5b4fe29 --- /dev/null +++ b/src/Shared.CLI/Options/FindSquatsToolOptions.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +namespace Microsoft.CST.OpenSource.OssGadget.Options; + +using CommandLine; +using CommandLine.Text; +using System.Collections.Generic; + +[Verb("find-squats", HelpText = "Run find-squats tool")] +public class FindSquatsToolOptions : BaseToolOptions +{ + [Usage()] + public static IEnumerable Examples + { + get + { + return new List() { + new Example("Find Squat Candidates for the Given Packages", + new FindSquatsToolOptions { Targets = new List() {"[options]", "package-urls..." } })}; + } + } + + [Option('o', "output-file", Required = false, Default = "", + HelpText = "send the command output to a file instead of stdout")] + public string OutputFile { get; set; } = ""; + + [Option('f', "format", Required = false, Default = "text", + HelpText = "specify the output format(text|sarifv1|sarifv2)")] + public string Format { get; set; } = "text"; + + [Option('q', "quiet", Required = false, Default = false, + HelpText = "Suppress console output.")] + public bool Quiet { get; set; } = false; + + [Option('s', "sleep-delay", Required = false, Default = 0, HelpText = "Number of ms to sleep between checks.")] + public int SleepDelay { get; set; } + + [Value(0, Required = true, + HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze + public IEnumerable? Targets { get; set; } + +} \ No newline at end of file diff --git a/src/Shared.CLI/Options/FreshToolOptions.cs b/src/Shared.CLI/Options/FreshToolOptions.cs new file mode 100644 index 00000000..0af7e3e6 --- /dev/null +++ b/src/Shared.CLI/Options/FreshToolOptions.cs @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +namespace Microsoft.CST.OpenSource.OssGadget.Options; + +using CommandLine; +using CommandLine.Text; +using System.Collections.Generic; + +[Verb("fresh", HelpText = "Run fresh tool")] +public class FreshToolOptions : BaseToolOptions +{ + [Usage()] + public static IEnumerable Examples + { + get + { + return new List() { + new Example("Find the source code repository for the given package", new FreshToolOptions { Targets = new List() {"[options]", "package-url..." } })}; + } + } + + [Option('f', "format", Required = false, Default = "text", + HelpText = "specify the output format(text|sarifv1|sarifv2)")] + public string Format { get; set; } = "text"; + + [Option('o', "output-file", Required = false, Default = "", + HelpText = "send the command output to a file instead of stdout")] + public string OutputFile { get; set; } = ""; + + [Option('m', "max-age-maintained", Required = false, Default = 30 * 18, + HelpText = "maximum age of versions for still-maintained projects, 0 to disable")] + public int MaxAgeMaintained { get; set; } + + [Option('u', "max-age-unmaintained", Required = false, Default = 30 * 48, + HelpText = "maximum age of versions for unmaintained projects, 0 to disable")] + public int MaxAgeUnmaintained { get; set; } + + [Option('v', "max-out-of-date-versions", Required = false, Default = 6, + HelpText = "maximum number of versions out of date, 0 to disable")] + public int MaxOutOfDateVersions { get; set; } + + [Option('r', "filter", Required = false, Default = null, + HelpText = "filter versions by regular expression")] + public string? Filter { get; set; } + + [Value(0, Required = true, + HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze + public IEnumerable? Targets { get; set; } +} \ No newline at end of file diff --git a/src/Shared.CLI/Options/HealthToolOptions.cs b/src/Shared.CLI/Options/HealthToolOptions.cs new file mode 100644 index 00000000..ee3284d0 --- /dev/null +++ b/src/Shared.CLI/Options/HealthToolOptions.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +namespace Microsoft.CST.OpenSource.OssGadget.Options; + +using CommandLine; +using CommandLine.Text; +using System.Collections.Generic; + +[Verb("health", HelpText = "Run oss-health tool")] +public class HealthToolOptions : BaseToolOptions +{ + [Usage()] + public static IEnumerable Examples + { + get + { + return new List() { + new Example("Find the source code repository for the given package", + new HealthToolOptions() { Targets = new List() {"[options]", "package-url..." } })}; + } + } + + [Option('f', "format", Required = false, Default = "text", + HelpText = "specify the output format(text|sarifv1|sarifv2)")] + public string? Format { get; set; } + + [Option('o', "output-file", Required = false, Default = "", + HelpText = "send the command output to a file instead of stdout")] + public string OutputFile { get; set; } = ""; + + [Value(0, Required = true, + HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze + public IEnumerable? Targets { get; set; } +} \ No newline at end of file diff --git a/src/Shared.CLI/Options/ReproducibleToolOptions.cs b/src/Shared.CLI/Options/ReproducibleToolOptions.cs new file mode 100644 index 00000000..bd4156df --- /dev/null +++ b/src/Shared.CLI/Options/ReproducibleToolOptions.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +namespace Microsoft.CST.OpenSource.OssGadget.Options; + +using CommandLine; +using CommandLine.Text; +using System.Collections.Generic; + +[Verb("reproducible", HelpText = "Run reproducible tool")] +public class ReproducibleToolOptions : BaseToolOptions +{ + [Usage()] + public static IEnumerable Examples + { + get + { + return new List() { + new Example("Estimate semantic equivalency of the given package and source code", new ReproducibleToolOptions { Targets = new List() {"[options]", "package-url..." } }) + }; + } + } + + [Value(0, Required = true, HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze + public IEnumerable? Targets { get; set; } + + [Option('a', "all-strategies", Required = false, Default = false, + HelpText = "Execute all strategies, even after a successful one is identified.")] + public bool AllStrategies { get; set; } + + [Option("specific-strategies", Required = false, + HelpText = "Execute specific strategies, comma-separated.")] + public string? SpecificStrategies { get; set; } + + [Option('s', "source-ref", Required = false, Default = "", + HelpText = "If a source version cannot be identified, use the specified git reference (tag, commit, etc.).")] + public string OverrideSourceReference { get; set; } = ""; + + [Option("diff-technique", Required = false, Default = DiffTechnique.Normalized, HelpText = "Configure diff technique.")] + public DiffTechnique DiffTechnique { get; set; } = DiffTechnique.Normalized; + + [Option('o', "output-file", Required = false, Default = "", HelpText = "Send the command output to a file instead of standard output")] + public string OutputFile { get; set; } = ""; + + [Option('d', "show-differences", Required = false, Default = false, + HelpText = "Output the differences between the package and the reference content.")] + public bool ShowDifferences { get; set; } + + [Option("show-all-differences", Required = false, Default = false, + HelpText = "Show all differences (default: capped at 20), implies --show-differences")] + public bool ShowAllDifferences { get; set; } + + [Option('l', "leave-intermediate", Required = false, Default = false, + HelpText = "Do not clean up intermediate files (useful for debugging).")] + public bool LeaveIntermediateFiles { get; set; } +} \ No newline at end of file diff --git a/src/Shared.CLI/Options/RiskCalculatorToolOptions.cs b/src/Shared.CLI/Options/RiskCalculatorToolOptions.cs new file mode 100644 index 00000000..7c738d5e --- /dev/null +++ b/src/Shared.CLI/Options/RiskCalculatorToolOptions.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +namespace Microsoft.CST.OpenSource.OssGadget.Options; + +using CommandLine; +using CommandLine.Text; +using System.Collections.Generic; + +[Verb("risk", HelpText = "Run risk calculator tool")] +public class RiskCalculatorToolOptions : BaseToolOptions +{ + [Usage()] + public static IEnumerable Examples + { + get + { + return new List() { + new Example("Calculate a risk metric for the given package", + new RiskCalculatorToolOptions { Targets = new List() {"[options]", "package-url..." } })}; + } + } + + [Option('d', "download-directory", Required = false, Default = null, + HelpText = "the directory to download the package to.")] + public string DownloadDirectory { get; set; } = "."; + + [Option('r', "external-risk", Required = false, Default = 0, + HelpText = "include additional risk in final calculation.")] + public int ExternalRisk { get; set; } + + [Option('f', "format", Required = false, Default = "text", + HelpText = "specify the output format(text|sarifv1|sarifv2)")] + public string Format { get; set; } = "text"; + + [Option('o', "output-file", Required = false, Default = "", + HelpText = "send the command output to a file instead of stdout")] + public string OutputFile { get; set; } = ""; + + [Option('n', "no-health", Required = false, Default = false, + HelpText = "do not check project health")] + public bool NoHealth { get; set; } + + [Option(Default = false, HelpText = "Verbose output")] + public bool Verbose { get; set; } + + [Value(0, Required = true, + HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze + public IEnumerable? Targets { get; set; } + + [Option('c', "use-cache", Required = false, Default = false, + HelpText = "do not download the package if it is already present in the destination directory.")] + public bool UseCache { get; set; } +} \ No newline at end of file diff --git a/src/Shared.CLI/Shared.CLI.csproj b/src/Shared.CLI/Shared.CLI.csproj index 54d0e1f9..9517345f 100644 --- a/src/Shared.CLI/Shared.CLI.csproj +++ b/src/Shared.CLI/Shared.CLI.csproj @@ -35,6 +35,9 @@ + + Always + @@ -42,17 +45,22 @@ + + + + + @@ -61,10 +69,38 @@ + + + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + + Always + + \ No newline at end of file diff --git a/src/Shared.CLI/Tools/CharacteristicTool.cs b/src/Shared.CLI/Tools/CharacteristicTool.cs new file mode 100644 index 00000000..4914cf06 --- /dev/null +++ b/src/Shared.CLI/Tools/CharacteristicTool.cs @@ -0,0 +1,333 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using CommandLine; +using CommandLine.Text; +using Microsoft.ApplicationInspector.Commands; +using Microsoft.ApplicationInspector.RulesEngine; +using Microsoft.CodeAnalysis.Sarif; +using Microsoft.CST.OpenSource.Shared; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using static Microsoft.CST.OpenSource.Shared.OutputBuilderFactory; +using SarifResult = Microsoft.CodeAnalysis.Sarif.Result; + +namespace Microsoft.CST.OpenSource.OssGadget.Tools +{ + using Microsoft.Extensions.Options; + using OssGadget.Options; + using PackageManagers; + using PackageUrl; + + public class CharacteristicTool : BaseTool + { + public CharacteristicTool(ProjectManagerFactory projectManagerFactory) : base(projectManagerFactory) + { + } + + public CharacteristicTool() : this(new ProjectManagerFactory()) + { + } + + public async Task AnalyzeFile(CharacteristicToolOptions options, string file) + { + Logger.Trace("AnalyzeFile({0})", file); + return await AnalyzeDirectory(options, file); + } + + /// + /// Analyzes a directory of files. + /// + /// directory to analyze. + /// List of tags identified + public async Task AnalyzeDirectory(CharacteristicToolOptions options, string directory) + { + Logger.Trace("AnalyzeDirectory({0})", directory); + + AnalyzeResult? analysisResult = null; + + // Call Application Inspector using the NuGet package + AnalyzeOptions? analyzeOptions = new AnalyzeOptions() + { + SourcePath = new[] { directory }, + IgnoreDefaultRules = options.DisableDefaultRules, + CustomRulesPath = options.CustomRuleDirectory, + ConfidenceFilters = new [] { Confidence.High | Confidence.Medium | Confidence.Low }, + ScanUnknownTypes = true, + AllowAllTagsInBuildFiles = options.AllowTagsInBuildFiles, + SingleThread = options.SingleThread, + FilePathExclusions = options.FilePathExclusions?.Split(',') ?? Array.Empty(), + EnableNonBacktrackingRegex = !options.EnableBacktracking + }; + + try + { + AnalyzeCommand? analyzeCommand = new AnalyzeCommand(analyzeOptions); + analysisResult = analyzeCommand.GetResult(); + Logger.Debug("Operation Complete: {0} files analyzed.", analysisResult?.Metadata?.TotalFiles); + } + catch (Exception ex) + { + Logger.Warn("Error analyzing {0}: {1}", directory, ex.Message); + } + + return analysisResult; + } + + /// + /// Analyze a package by downloading it first. + /// + /// The package-url of the package to analyze. + /// List of tags identified + public async Task> AnalyzePackage(CharacteristicToolOptions options, PackageURL purl, + string? targetDirectoryName, + bool doCaching = false) + { + Logger.Trace("AnalyzePackage({0})", purl.ToString()); + + Dictionary? analysisResults = new Dictionary(); + + PackageDownloader? packageDownloader = new PackageDownloader(purl, ProjectManagerFactory, targetDirectoryName, doCaching); + // ensure that the cache directory has the required package, download it otherwise + List? directoryNames = await packageDownloader.DownloadPackageLocalCopy(purl, + false, + true); + if (directoryNames.Count > 0) + { + foreach (string? directoryName in directoryNames) + { + AnalyzeResult? singleResult = await AnalyzeDirectory(options, directoryName); + analysisResults[directoryName] = singleResult; + } + } + else + { + Logger.Warn("Error downloading {0}.", purl.ToString()); + } + packageDownloader.ClearPackageLocalCopyIfNoCaching(); + return analysisResults; + } + + /// + /// Build and return a list of Sarif Result list from the find characterstics results + /// + /// + /// + /// + private static List GetSarifResults(PackageURL purl, Dictionary analysisResult, CharacteristicToolOptions opts) + { + List sarifResults = new List(); + + if (analysisResult.HasAtLeastOneNonNullValue()) + { + foreach (string? key in analysisResult.Keys) + { + MetaData? metadata = analysisResult?[key]?.Metadata; + + foreach (MatchRecord? result in metadata?.Matches ?? new List()) + { + SarifResult? individualResult = new SarifResult() + { + Message = new Message() + { + Text = result.RuleDescription, + Id = result.RuleId + }, + Kind = ResultKind.Informational, + Level = opts.SarifLevel, + Locations = SarifOutputBuilder.BuildPurlLocation(purl), + Rule = new ReportingDescriptorReference() { Id = result.RuleId }, + }; + + individualResult.SetProperty("Severity", result.Severity); + individualResult.SetProperty("Confidence", result.Confidence); + + individualResult.Locations.Add(new CodeAnalysis.Sarif.Location() + { + PhysicalLocation = new PhysicalLocation() + { + Address = new Address() { FullyQualifiedName = result.FileName }, + Region = new Region() + { + StartLine = result.StartLocationLine, + EndLine = result.EndLocationLine, + StartColumn = result.StartLocationColumn, + EndColumn = result.EndLocationColumn, + SourceLanguage = result.Language, + Snippet = new ArtifactContent() + { + Text = result.Excerpt, + Rendered = new MultiformatMessageString(result.Excerpt, $"`{result.Excerpt}`", null) + } + } + } + }); + + sarifResults.Add(individualResult); + } + } + } + return sarifResults; + } + + /// + /// Convert charactersticTool results into text format + /// + /// + /// + private static List GetTextResults(PackageURL purl, Dictionary analysisResult) + { + List stringOutput = new List(); + + stringOutput.Add(purl.ToString()); + + if (analysisResult.HasAtLeastOneNonNullValue()) + { + foreach (string? key in analysisResult.Keys) + { + MetaData? metadata = analysisResult?[key]?.Metadata; + + stringOutput.Add(string.Format("Programming Language(s): {0}", + string.Join(", ", metadata?.Languages?.Keys ?? new List()))); + + stringOutput.Add("Unique Tags (Confidence): "); + bool hasTags = false; + Dictionary>? dict = new Dictionary>(); + foreach ((string[]? tags, Confidence confidence) in metadata?.Matches?.Where(x => x is not null).Select(x => (x.Tags, x.Confidence)) ?? Array.Empty<(string[], Confidence)>()) + { + foreach (string? tag in tags) + { + if (dict.ContainsKey(tag)) + { + dict[tag].Add(confidence); + } + else + { + dict[tag] = new List() { confidence }; + } + } + } + + foreach ((string? k, List? v) in dict) + { + hasTags = true; + Confidence confidence = v.Max(); + if (confidence > 0) + { + stringOutput.Add($" * {k} ({v.Max()})"); + } + else + { + stringOutput.Add($" * {k}"); + } + } + if (!hasTags) + { + stringOutput.Add("No tags were found."); + } + } + } + return stringOutput; + } + + /// + /// Convert charactersticTool results into output format + /// + /// + /// + /// + private void AppendOutput(IOutputBuilder outputBuilder, PackageURL purl, Dictionary analysisResults, CharacteristicToolOptions opts) + { + switch (currentOutputFormat) + { + case OutputFormat.text: + default: + outputBuilder.AppendOutput(GetTextResults(purl, analysisResults)); + break; + + case OutputFormat.sarifv1: + case OutputFormat.sarifv2: + outputBuilder.AppendOutput(GetSarifResults(purl, analysisResults,opts)); + break; + } + } + + public async Task RunAsync(CharacteristicToolOptions options) + { + _ = await LegacyRunAsync(options); + } + + public async Task>> LegacyRunAsync(CharacteristicToolOptions options) + { + // select output destination and format + SelectOutput(options.OutputFile); + IOutputBuilder outputBuilder = SelectFormat(options.Format); + + List>? finalResults = new List>(); + + if (options.Targets is IList targetList && targetList.Count > 0) + { + foreach (string? target in targetList) + { + try + { + if (target.StartsWith("pkg:")) + { + PackageURL? purl = new PackageURL(target); + string downloadDirectory = options.DownloadDirectory == "." ? System.IO.Directory.GetCurrentDirectory() : options.DownloadDirectory; + Dictionary? analysisResult = await AnalyzePackage(options, purl, + downloadDirectory, + options.UseCache == true); + + AppendOutput(outputBuilder, purl, analysisResult, options); + finalResults.Add(analysisResult); + } + else if (System.IO.Directory.Exists(target)) + { + AnalyzeResult? analysisResult = await AnalyzeDirectory(options, target); + if (analysisResult != null) + { + Dictionary? analysisResults = new Dictionary() + { + { target, analysisResult } + }; + PackageURL? purl = new PackageURL("generic", target); + AppendOutput(outputBuilder, purl, analysisResults, options); + } + finalResults.Add(new Dictionary() { { target, analysisResult } }); + + } + else if (File.Exists(target)) + { + AnalyzeResult? analysisResult = await AnalyzeFile(options, target); + if (analysisResult != null) + { + Dictionary? analysisResults = new Dictionary() + { + { target, analysisResult } + }; + PackageURL? purl = new PackageURL("generic", target); + AppendOutput(outputBuilder, purl, analysisResults, options); + } + finalResults.Add(new Dictionary() { { target, analysisResult } }); + } + else + { + Logger.Warn("Package or file identifier was invalid."); + } + } + catch (Exception ex) + { + Logger.Warn(ex, "Error processing {0}: {1}", target, ex.Message); + } + } + outputBuilder.PrintOutput(); + } + + RestoreOutput(); + return finalResults; + } + } +} diff --git a/src/Shared.CLI/Tools/DiffTool/Diff.cs b/src/Shared.CLI/Tools/DiffTool/Diff.cs new file mode 100644 index 00000000..e1a7b182 --- /dev/null +++ b/src/Shared.CLI/Tools/DiffTool/Diff.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using System; +using System.Collections.Generic; + +namespace Microsoft.CST.OpenSource.DiffTool +{ + public class Diff + { + public enum LineType + { + None, + Added, + Removed, + Context + } + public int startLine1 { get; set; } = -1; + public int endLine1 { get { return startLine1 == -1 ? -1 : Math.Max(startLine1, startLine1 + text1.Count - 1); } } + public int startLine2 { get; set; } = -1; + public int endLine2 { get { return startLine2 == -1 ? -1 : Math.Max(startLine2, startLine2 + text2.Count - 1); } } + public List beforeContext { get; private set; } = new List(); + public List text1 { get; private set; } = new List(); + public List text2 { get; private set; } = new List(); + public List afterContext { get; private set; } = new List(); + public LineType lastLineType { get; private set; } = LineType.None; + + public void AddBeforeContext(string context) + { + beforeContext.Add(context); + lastLineType = LineType.Context; + } + public void AddAfterContext(string context) + { + afterContext.Add(context); + lastLineType = LineType.Context; + } + public void AddText1(string content) + { + text1.Add(content); + lastLineType = LineType.Removed; + } + public void AddText2(string content) + { + text2.Add(content); + lastLineType = LineType.Added; + } + } +} diff --git a/src/Shared.CLI/Tools/DiffTool/DiffTool.cs b/src/Shared.CLI/Tools/DiffTool/DiffTool.cs new file mode 100644 index 00000000..325073af --- /dev/null +++ b/src/Shared.CLI/Tools/DiffTool/DiffTool.cs @@ -0,0 +1,429 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Drawing; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using CommandLine; +using CommandLine.Text; +using DiffPlex.DiffBuilder; +using DiffPlex.DiffBuilder.Model; +using Microsoft.CodeAnalysis.Sarif; +using Microsoft.CST.OpenSource.Shared; +using Microsoft.CST.RecursiveExtractor; +using Pastel; +using SarifResult = Microsoft.CodeAnalysis.Sarif.Result; + +namespace Microsoft.CST.OpenSource.DiffTool +{ + using Contracts; + using Microsoft.CST.OpenSource.Helpers; + using Microsoft.CST.OpenSource.PackageManagers; + using Microsoft.Extensions.Options; + using OssGadget.Options; + using PackageUrl; + + class DiffTool : BaseTool + { + public DiffTool(ProjectManagerFactory projectManagerFactory) : base(projectManagerFactory) + { + } + + public DiffTool() : this (new ProjectManagerFactory()) { } + + public async Task DiffProjects(DiffToolOptions options) + { + Extractor? extractor = new Extractor(); + IOutputBuilder? outputBuilder = OutputBuilderFactory.CreateOutputBuilder(options.Format); + if (outputBuilder is null) + { + Logger.Error($"Format {options.Format} is not supported."); + throw new ArgumentOutOfRangeException("options.Format", $"Format {options.Format} is not supported."); + } + + // Map relative location in package to actual location on disk + ConcurrentDictionary files = new ConcurrentDictionary(); + IEnumerable locations = Array.Empty(); + IEnumerable locations2 = Array.Empty(); + + try + { + PackageURL purl1 = new PackageURL(options.Targets.First()); + IBaseProjectManager? manager = ProjectManagerFactory.CreateProjectManager(purl1, options.DownloadDirectory ?? Path.GetTempPath()); + + if (manager is not null) + { + locations = await manager.DownloadVersionAsync(purl1, true, options.UseCache); + } + } + catch (Exception) + { + string? tmpDir = Path.GetTempFileName(); + File.Delete(tmpDir); + try + { + extractor.ExtractToDirectory(tmpDir, options.Targets.First()); + locations = new string[] { tmpDir }; + } + catch (Exception e) + { + Logger.Error($"{e.Message}:{e.StackTrace}"); + Environment.Exit(-1); + } + } + + foreach (string? directory in locations) + { + foreach (string? file in System.IO.Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories)) + { + files[string.Join(Path.DirectorySeparatorChar, file[directory.Length..].Split(Path.DirectorySeparatorChar)[2..])] = (file, string.Empty); + } + } + + try + { + PackageURL purl2 = new PackageURL(options.Targets.Last()); + IBaseProjectManager? manager2 = ProjectManagerFactory.CreateProjectManager(purl2, options.DownloadDirectory ?? Path.GetTempPath()); + + if (manager2 is not null) + { + locations2 = await manager2.DownloadVersionAsync(purl2, true, options.UseCache); + } + } + catch (Exception) + { + string? tmpDir = Path.GetTempFileName(); + File.Delete(tmpDir); + try + { + extractor.ExtractToDirectory(tmpDir, options.Targets.Last()); + locations2 = new string[] { tmpDir }; + } + catch (Exception e) + { + Logger.Error($"{e.Message}:{e.StackTrace}"); + Environment.Exit(-1); + } + } + foreach (string? directory in locations2) + { + foreach (string? file in System.IO.Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories)) + { + string? key = string.Join(Path.DirectorySeparatorChar, file[directory.Length..].Split(Path.DirectorySeparatorChar)[2..]); + + if (files.ContainsKey(key)) + { + (string, string) existing = files[key]; + existing.Item2 = file; + files[key] = existing; + } + else + { + files[key] = (string.Empty, file); + } + } + } + + Parallel.ForEach(files, filePair => + { + ConcurrentBag? diffObjs = new ConcurrentBag(); + + if (options.CrawlArchives) + { + using Stream fs1 = !string.IsNullOrEmpty(filePair.Value.Item1) ? File.OpenRead(filePair.Value.Item1) : new MemoryStream(); + using Stream fs2 = !string.IsNullOrEmpty(filePair.Value.Item2) ? File.OpenRead(filePair.Value.Item2) : new MemoryStream(); + IEnumerable? entries1 = extractor.Extract(new FileEntry(filePair.Key, fs1), new ExtractorOptions() { Parallel = false, MemoryStreamCutoff = 1 }); + IEnumerable? entries2 = extractor.Extract(new FileEntry(filePair.Key, fs2), new ExtractorOptions() { Parallel = false, MemoryStreamCutoff = 1 }); + ConcurrentDictionary? entryPairs = new ConcurrentDictionary(); + foreach (FileEntry? entry in entries1) + { + entryPairs[entry.FullPath] = (entry, null); + } + foreach (FileEntry? entry in entries2) + { + if (entryPairs.ContainsKey(entry.FullPath)) + { + entryPairs[entry.FullPath] = (entryPairs[entry.FullPath].Item1, entry); + } + else + { + entryPairs[entry.FullPath] = (null, entry); + } + } + + foreach (KeyValuePair entryPair in entryPairs) + { + string? text1 = string.Empty; + string? text2 = string.Empty; + if (entryPair.Value.Item1 is not null) + { + using StreamReader? sr = new StreamReader(entryPair.Value.Item1.Content); + text1 = sr.ReadToEnd(); + } + if (entryPair.Value.Item2 is not null) + { + using StreamReader? sr = new StreamReader(entryPair.Value.Item2.Content); + text2 = sr.ReadToEnd(); + } + WriteFileIssues(entryPair.Key, text1, text2); + } + } + else + { + string? file1 = string.Empty; + string? file2 = string.Empty; + if (!string.IsNullOrEmpty(filePair.Value.Item1)) + { + file1 = File.ReadAllText(filePair.Value.Item1); + } + if (!string.IsNullOrEmpty(filePair.Value.Item2)) + { + file2 = File.ReadAllText(filePair.Value.Item2); + } + WriteFileIssues(filePair.Key, file1, file2); + } + + // If we are writing text write the file name + if (options.Format == "text") + { + if (options.Context > 0 || options.After > 0 || options.Before > 0) + { + outputBuilder.AppendOutput(new string[] { $"*** {filePair.Key}", $"--- {filePair.Key}", "***************" }); + } + else + { + outputBuilder.AppendOutput(new string[] { filePair.Key }); + } + } + List? diffObjList = diffObjs.ToList(); + + // Arrange the diffs in line order + diffObjList.Sort((x, y) => x.startLine1.CompareTo(y.startLine1)); + + foreach (Diff? diff in diffObjList) + { + StringBuilder? sb = new StringBuilder(); + // Write Context Format + if (options.Context > 0 || options.After > 0 || options.Before > 0) + { + sb.AppendLine($"*** {diff.startLine1 - diff.beforeContext.Count},{diff.endLine1 + diff.afterContext.Count} ****"); + + diff.beforeContext.ForEach(x => sb.AppendLine(options.DownloadDirectory is not null ? $" {x}" : $" {x}".Pastel(Color.Gray))); + diff.text1.ForEach(x => sb.AppendLine(options.DownloadDirectory is not null ? $"- {x}" : $"- {x}".Pastel(Color.Red))); + diff.afterContext.ForEach(x => sb.AppendLine(options.DownloadDirectory is not null ? $" {x}" : $" {x}".Pastel(Color.Gray))); + + if (diff.startLine2 > -1) + { + sb.AppendLine($"--- {diff.startLine2 - diff.beforeContext.Count},{diff.endLine2 + diff.afterContext.Count} ----"); + + diff.beforeContext.ForEach(x => sb.AppendLine(options.DownloadDirectory is not null ? $" {x}" : $" {x}".Pastel(Color.Gray))); + diff.text2.ForEach(x => sb.AppendLine(options.DownloadDirectory is not null ? $"+ {x}" : $"+ {x}".Pastel(Color.Green))); + diff.afterContext.ForEach(x => sb.AppendLine(options.DownloadDirectory is not null ? $" {x}" : $" {x}".Pastel(Color.Gray))); + } + } + // Write diff "Normal Format" + else + { + if (diff.text1.Any() && diff.text2.Any()) + { + sb.Append(Math.Max(diff.startLine1, 0)); + if (diff.endLine1 != diff.startLine1) + { + sb.Append($",{diff.endLine1}"); + } + sb.Append('c'); + sb.Append(Math.Max(diff.startLine2, 0)); + if (diff.endLine2 != diff.startLine2) + { + sb.Append($",{diff.endLine2}"); + } + sb.Append(Environment.NewLine); + diff.text1.ForEach(x => sb.AppendLine(options.DownloadDirectory is not null ? $"< {x}" : $"< {x}".Pastel(Color.Red))); + sb.AppendLine("---"); + diff.text2.ForEach(x => sb.AppendLine(options.DownloadDirectory is not null ? $"> {x}" : $"> {x}".Pastel(Color.Green))); + } + else if (diff.text1.Any()) + { + sb.Append(Math.Max(diff.startLine1, 0)); + if (diff.endLine1 != diff.startLine1) + { + sb.Append($",{diff.endLine1}"); + } + sb.Append('d'); + sb.Append(Math.Max(diff.endLine2, 0)); + sb.Append(Environment.NewLine); + diff.text1.ForEach(x => sb.AppendLine(options.DownloadDirectory is not null ? $"< {x}" : $"< {x}".Pastel(Color.Red))); + } + else if (diff.text2.Any()) + { + sb.Append(Math.Max(diff.startLine1, 0)); + sb.Append('a'); + sb.Append(Math.Max(diff.startLine2, 0)); + if (diff.endLine2 != diff.startLine2) + { + sb.Append($",{diff.endLine2}"); + } + sb.Append(Environment.NewLine); + diff.text2.ForEach(x => sb.AppendLine(options.DownloadDirectory is not null ? $"> {x}" : $"> {x}".Pastel(Color.Green))); + } + } + + switch (outputBuilder) + { + case StringOutputBuilder stringOutputBuilder: + stringOutputBuilder.AppendOutput(new string[] { sb.ToString().TrimEnd() }); + break; + case SarifOutputBuilder sarifOutputBuilder: + SarifResult? sr = new SarifResult + { + Locations = new Location[] { new Location() { LogicalLocation = new LogicalLocation() { FullyQualifiedName = filePair.Key } } }, + AnalysisTarget = new ArtifactLocation() { Uri = new Uri(options.Targets.First()) }, + Message = new Message() { Text = sb.ToString() } + }; + sarifOutputBuilder.AppendOutput(new SarifResult[] { sr }); + break; + } + } + + void WriteFileIssues(string path, string file1, string file2) + { + DiffPaneModel? diff = InlineDiffBuilder.Diff(file1, file2); + List beforeBuffer = new List(); + + int afterCount = 0; + int lineNumber1 = 0; + int lineNumber2 = 0; + Diff? diffObj = new Diff(); + + foreach (DiffPiece? line in diff.Lines) + { + switch (line.Type) + { + case ChangeType.Inserted: + lineNumber2++; + + if (diffObj.lastLineType == Diff.LineType.Context || (diffObj.endLine2 != -1 && lineNumber2 - diffObj.endLine2 > 1 && diffObj.lastLineType != Diff.LineType.Added)) + { + diffObjs.Add(diffObj); + diffObj = new Diff() { startLine1 = lineNumber1 }; + } + + if (diffObj.startLine2 == -1) + { + diffObj.startLine2 = lineNumber2; + } + + if (beforeBuffer.Any()) + { + beforeBuffer.ForEach(x => diffObj.AddBeforeContext(x)); + beforeBuffer.Clear(); + } + + afterCount = options.After; + diffObj.AddText2(line.Text); + + break; + case ChangeType.Deleted: + lineNumber1++; + + if (diffObj.lastLineType == Diff.LineType.Context || (diffObj.endLine1 != -1 && lineNumber1 - diffObj.endLine1 > 1 && diffObj.lastLineType != Diff.LineType.Removed)) + { + diffObjs.Add(diffObj); + diffObj = new Diff() { startLine2 = lineNumber2 }; + } + + if (diffObj.startLine1 == -1) + { + diffObj.startLine1 = lineNumber1; + } + + if (beforeBuffer.Any()) + { + beforeBuffer.ForEach(x => diffObj.AddBeforeContext(x)); + beforeBuffer.Clear(); + } + + afterCount = options.After; + diffObj.AddText1(line.Text); + + break; + default: + lineNumber1++; + lineNumber2++; + + if (options.Context == -1) + { + diffObj.AddAfterContext(line.Text); + beforeBuffer.Add(line.Text); + } + else if (afterCount-- > 0) + { + diffObj.AddAfterContext(line.Text); + if (afterCount == 0) + { + diffObjs.Add(diffObj); + diffObj = new Diff(); + } + } + else if (options.Before > 0) + { + beforeBuffer.Add(line.Text); + while (options.Before < beforeBuffer.Count) + { + beforeBuffer.RemoveAt(0); + } + } + break; + } + } + + if (diffObj.startLine1 != -1 || diffObj.startLine2 != -1) + { + diffObjs.Add(diffObj); + } + } + }); + + + if (!options.UseCache) + { + foreach (string? directory in locations.Concat(locations2)) + { + FileSystemHelper.RetryDeleteDirectory(directory); + } + } + + return outputBuilder; + } + + public async Task RunAsync(DiffToolOptions options) + { + if (options.Targets.Count() != 2) + { + Logger.Error("Must provide exactly two packages to diff."); + return; + } + + if (options.Context > 0) + { + options.Before = options.Context; + options.After = options.Context; + } + + IOutputBuilder? result = await DiffProjects(options); + + if (options.OutputLocation is null) + { + result.PrintOutput(); + } + else + { + result.WriteOutput(options.OutputLocation); + } + } + } +} diff --git a/src/Shared.CLI/Tools/DownloadTool.cs b/src/Shared.CLI/Tools/DownloadTool.cs index 2e81e282..de98851f 100644 --- a/src/Shared.CLI/Tools/DownloadTool.cs +++ b/src/Shared.CLI/Tools/DownloadTool.cs @@ -11,7 +11,9 @@ namespace Microsoft.CST.OpenSource.OssGadget.CLI.Tools; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; -using Microsoft.CST.OpenSource.OssGadget.CLI.Options; +using Microsoft.CST.OpenSource.OssGadget.Options; +using Options; + public class DownloadTool : BaseTool { private readonly ProjectManagerFactory _projectManagerFactory; diff --git a/src/Shared.CLI/Tools/FindDomainSquatsTool.cs b/src/Shared.CLI/Tools/FindDomainSquatsTool.cs new file mode 100644 index 00000000..51971705 --- /dev/null +++ b/src/Shared.CLI/Tools/FindDomainSquatsTool.cs @@ -0,0 +1,227 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +namespace Microsoft.CST.OpenSource.DomainSquats +{ + using CommandLine; + using CommandLine.Text; + using Microsoft.CodeAnalysis.Sarif; + using Microsoft.CST.OpenSource.Shared; + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading.Tasks; + using SarifResult = Microsoft.CodeAnalysis.Sarif.Result; + using System.IO; + using System.Threading; + using Whois; + using System.Text.RegularExpressions; + using Microsoft.CST.OpenSource.FindSquats.Mutators; + using OssGadget.Options; + using PackageManagers; + using System.Net.Http; + + public class FindDomainSquatsTool : BaseTool + { + internal static IList BaseMutators { get; } = new List() + { + new AsciiHomoglyphMutator(), + new BitFlipMutator(), + new CloseLettersMutator(), + new DoubleHitMutator(), + new DuplicatorMutator(), + new PrefixMutator(), + new RemovedCharacterMutator(), + new RemoveSeparatedSectionMutator(), + new SeparatorChangedMutator(), + new SeparatorRemovedMutator(), + new SubstitutionMutator(), + new SuffixMutator(), + new SwapOrderOfLettersMutator(), + new UnicodeHomoglyphMutator(), + new VowelSwapMutator() + }; + + public FindDomainSquatsTool(IHttpClientFactory httpClientFactory) : base(new ProjectManagerFactory(httpClientFactory)) + { + } + + public FindDomainSquatsTool() : this(new DefaultHttpClientFactory()) { } + + private readonly Regex ValidDomainRegex = new("^[0-9a-z]+[0-9a-z\\-]*[0-9a-z]+$", RegexOptions.Compiled); + + public async Task<(string output, int registeredSquats, int unregisteredSquats)> RunAsync(FindDomainSquatsToolOptions FindDomainSquatsToolOptions) + { + IOutputBuilder outputBuilder = SelectFormat(FindDomainSquatsToolOptions.Format); + List<(string, KeyValuePair>)> registeredSquats = new(); + List<(string, KeyValuePair>)> unregisteredSquats = new(); + List<(string, KeyValuePair>)> failedSquats = new(); + WhoisLookup whois = new(); + foreach (string? target in FindDomainSquatsToolOptions.Targets ?? Array.Empty()) + { + string[] splits = target.Split('.'); + string domain = splits[0]; + List potentials = new(); + foreach (IMutator mutator in BaseMutators) + { + foreach (Mutation mutation in mutator.Generate(splits[0])) + { + potentials.Add(mutation); + } + } + + foreach(KeyValuePair> potential in potentials.GroupBy(x => x.Mutated).ToDictionary(x => x.Key, x => x.ToList())) + { + await CheckPotential(potential); + } + + async Task CheckPotential(KeyValuePair> potential, int retries = 0) + { + // Not a valid domain + if (!ValidDomainRegex.IsMatch(potential.Key)) + { + return; + } + if (Uri.IsWellFormedUriString(potential.Key, UriKind.Relative)) + { + string url = string.Join('.',potential.Key, string.Join('.',splits[1..])); + + try + { + Thread.Sleep(FindDomainSquatsToolOptions.SleepDelay); + WhoisResponse response = await whois.LookupAsync(url); + if (response.Status == WhoisStatus.Found) + { + registeredSquats.Add((url, potential)); + } + else if (response.Status == WhoisStatus.NotFound) + { + unregisteredSquats.Add((url, potential)); + } + else + { + failedSquats.Add((url, potential)); + } + } + catch (Exception e) + { + Logger.Debug(e, $"{e.Message}:{e.StackTrace}"); + + if (retries++ < 5) + { + Thread.Sleep(1000); + await CheckPotential(potential, retries); + } + else + { + failedSquats.Add((url, potential)); + } + } + } + } + } + if (string.IsNullOrEmpty(FindDomainSquatsToolOptions.OutputFile)) + { + FindDomainSquatsToolOptions.OutputFile = $"oss-find-domain-squat.{FindDomainSquatsToolOptions.Format}"; + } + if (!FindDomainSquatsToolOptions.Unregistered) + { + if (registeredSquats.Any()) + { + Logger.Warn($"Found {registeredSquats.Count} registered potential squats."); + } + foreach ((string, KeyValuePair>) potential in registeredSquats) + { + string output = $"Registered: {potential.Item1} (rules: {string.Join(',', potential.Item2.Value)})"; + if (!FindDomainSquatsToolOptions.Quiet) + { + Logger.Info(output); + } + switch (outputBuilder) + { + case StringOutputBuilder s: + s.AppendOutput(new string[] { output }); + break; + case SarifOutputBuilder sarif: + SarifResult sarifResult = new() + { + Message = new Message() + { + Text = $"Potential Squat candidate { potential.Item1 } is Registered.", + Id = "oss-find-domain-squats" + }, + Kind = ResultKind.Review, + Level = FailureLevel.None, + }; + sarifResult.Tags.Add("Registered"); + foreach (Mutation? tag in potential.Item2.Value) + { + sarifResult.Tags.Add(tag.Reason); + sarifResult.Tags.Add(tag.Mutator.ToString()); + } + sarif.AppendOutput(new SarifResult[] { sarifResult }); + break; + } + } + } + if (!FindDomainSquatsToolOptions.Registered) + { + if (unregisteredSquats.Any()) + { + Logger.Warn($"Found {unregisteredSquats.Count} unregistered potential squats."); + foreach ((string, KeyValuePair>) potential in unregisteredSquats) + { + string output = $"Unregistered: {potential.Item1} (rules: {string.Join(',', potential.Item2.Value)})"; + if (!FindDomainSquatsToolOptions.Quiet) + { + Logger.Info(output); + } + switch (outputBuilder) + { + case StringOutputBuilder s: + s.AppendOutput(new string[] { output }); + break; + case SarifOutputBuilder sarif: + SarifResult sarifResult = new() + { + Message = new Message() + { + Text = $"Potential Squat candidate { potential.Item1 } is Unregistered.", + Id = "oss-find-domain-squats" + }, + Kind = ResultKind.Review, + Level = FailureLevel.None, + }; + sarifResult.Tags.Add("Unregistered"); + foreach (Mutation? tag in potential.Item2.Value) + { + sarifResult.Tags.Add(tag.Reason); + sarifResult.Tags.Add(tag.Mutator.ToString()); + } + sarif.AppendOutput(new SarifResult[] { sarifResult }); + break; + } + } + } + } + if (failedSquats.Any()) + { + Logger.Error($"{failedSquats.Count} potential squats hit an exception when querying. Try increasing the sleep setting and trying again or check these manually."); + if (!FindDomainSquatsToolOptions.Quiet) + { + foreach ((string, KeyValuePair>) fail in failedSquats) + { + Logger.Info($"Failed: {fail.Item1} (rules: {string.Join(',', fail.Item2.Value)})"); + } + } + + } + + + using StreamWriter fw = new(FindDomainSquatsToolOptions.OutputFile); + string outString = outputBuilder.GetOutput(); + fw.WriteLine(outString); + fw.Close(); + return (outString, registeredSquats.Count, unregisteredSquats.Count); + } + } +} diff --git a/src/Shared.CLI/Tools/FindSourceTool.cs b/src/Shared.CLI/Tools/FindSourceTool.cs new file mode 100644 index 00000000..aea9293b --- /dev/null +++ b/src/Shared.CLI/Tools/FindSourceTool.cs @@ -0,0 +1,226 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using CommandLine; +using CommandLine.Text; +using Microsoft.CodeAnalysis.Sarif; +using Microsoft.CST.OpenSource.Shared; +using NLog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using static Microsoft.CST.OpenSource.Shared.OutputBuilderFactory; + +namespace Microsoft.CST.OpenSource +{ + using OssGadget.Options; + using PackageManagers; + using PackageUrl; + + public class FindSourceTool : BaseTool + { + private static readonly HashSet IGNORE_PURLS = new() + { + "pkg:github/metacpan/metacpan-web" + }; + + public FindSourceTool(ProjectManagerFactory projectManagerFactory) : base(projectManagerFactory) + { + } + + public FindSourceTool() : this(new ProjectManagerFactory()) + { + } + + public async Task> FindSourceAsync(PackageURL purl) + { + Logger.Trace("FindSourceAsync({0})", purl); + + Dictionary? repositoryMap = new Dictionary(); + + if (purl == null) + { + Logger.Warn("FindSourceAsync was passed an invalid purl."); + return repositoryMap; + } + + PackageURL? purlNoVersion = new PackageURL(purl.Type, purl.Namespace, purl.Name, + null, purl.Qualifiers, purl.Subpath); + Logger.Debug("Searching for source code for {0}", purlNoVersion.ToString()); + + try + { + RepoSearch repoSearcher = new RepoSearch(ProjectManagerFactory); + Dictionary? repos = await (repoSearcher.ResolvePackageLibraryAsync(purl) ?? + Task.FromResult(new Dictionary())); + + foreach (string? ignorePurl in IGNORE_PURLS) + { + repos.Remove(new PackageURL(ignorePurl)); + } + + if (repos.Any()) + { + foreach (PackageURL? key in repos.Keys) + { + repositoryMap[key] = repos[key]; + } + Logger.Debug("Identified {0} repositories.", repos.Count); + } + else + { + Logger.Warn("No repositories found for package {0}", purl); + } + } + catch (Exception ex) + { + Logger.Warn(ex, "Error identifying source repository for {0}: {1}", purl, ex.Message); + } + + return repositoryMap; + } + + + /// + /// Build and return a list of Sarif Result list from the find source results + /// + /// + /// + /// + private static List GetSarifResults(PackageURL purl, List> results) + { + List sarifResults = new List(); + foreach (KeyValuePair result in results) + { + double confidence = result.Value * 100.0; + Result sarifResult = new Result() + { + Message = new Message() + { + Text = $"https://github.com/{result.Key.Namespace}/{result.Key.Name}" + }, + Kind = ResultKind.Informational, + Level = FailureLevel.None, + Rank = confidence, + Locations = SarifOutputBuilder.BuildPurlLocation(purl) + }; + + sarifResults.Add(sarifResult); + } + return sarifResults; + } + + /// + /// Convert findSourceTool results into text format + /// + /// + /// + private static List GetTextResults(List> results) + { + List stringOutput = new List(); + foreach (KeyValuePair result in results) + { + double confidence = result.Value * 100.0; + stringOutput.Add( + $"{confidence:0.0}%\thttps://github.com/{result.Key.Namespace}/{result.Key.Name} ({result.Key})"); + } + return stringOutput; + } + + static async Task Main(string[] args) + { + FindSourceTool findSourceTool = new FindSourceTool(); + await findSourceTool.ParseOptions(args).WithParsedAsync(findSourceTool.RunAsync); + } + + /// + /// Convert findSourceTool results into output format + /// + /// + /// + /// + private void AppendOutput(IOutputBuilder outputBuilder, PackageURL purl, List> results) + { + switch (currentOutputFormat) + { + case OutputFormat.text: + default: + outputBuilder.AppendOutput(GetTextResults(results)); + break; + + case OutputFormat.sarifv1: + case OutputFormat.sarifv2: + outputBuilder.AppendOutput(GetSarifResults(purl, results)); + break; + } + } + + /// + /// Convert findSourceTool results into output format + /// + /// + /// + /// + private void AppendSingleOutput(IOutputBuilder outputBuilder, PackageURL purl, KeyValuePair result) + { + switch (currentOutputFormat) + { + case OutputFormat.text: + default: + outputBuilder.AppendOutput(new string[] { $"https://github.com/{result.Key.Namespace}/{result.Key.Name}" }); + break; + + case OutputFormat.sarifv1: + case OutputFormat.sarifv2: + outputBuilder.AppendOutput(GetSarifResults(purl, new List>() { result })); + break; + } + } + + private async Task RunAsync(FindSourceToolOptions findSourceToolOptions) + { + // Save the console logger to restore it later if we are in single mode + NLog.Targets.Target? oldConfig = LogManager.Configuration.FindTargetByName("consoleLog"); + if (findSourceToolOptions.Single) + { + // Suppress console logging for single mode + LogManager.Configuration.RemoveTarget("consoleLog"); + } + // select output destination and format + SelectOutput(findSourceToolOptions.OutputFile); + IOutputBuilder outputBuilder = SelectFormat(findSourceToolOptions.Format); + if (findSourceToolOptions.Targets is IList targetList && targetList.Count > 0) + { + foreach (string? target in targetList) + { + try + { + PackageURL? purl = new PackageURL(target); + Dictionary dictionary = await FindSourceAsync(purl); + List>? results = dictionary.ToList(); + results.Sort((b, a) => a.Value.CompareTo(b.Value)); + if (findSourceToolOptions.Single) + { + AppendSingleOutput(outputBuilder, purl, results[0]); + } + else + { + AppendOutput(outputBuilder, purl, results); + } + } + catch (Exception ex) + { + Logger.Warn("Error processing {0}: {1}", target, ex.Message); + } + } + outputBuilder.PrintOutput(); + } + RestoreOutput(); + // Restore console logging if we were in single mode + if (findSourceToolOptions.Single) + { + LogManager.Configuration.AddTarget(oldConfig); + } + } + } +} \ No newline at end of file diff --git a/src/Shared.CLI/Tools/FreshTool.cs b/src/Shared.CLI/Tools/FreshTool.cs new file mode 100644 index 00000000..c881b839 --- /dev/null +++ b/src/Shared.CLI/Tools/FreshTool.cs @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using CommandLine; +using CommandLine.Text; +using Microsoft.CodeAnalysis.Sarif; +using Microsoft.CST.OpenSource.Shared; +using NLog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using SemanticVersioning; +using static Microsoft.CST.OpenSource.Shared.OutputBuilderFactory; + +namespace Microsoft.CST.OpenSource +{ + using AngleSharp; + using OssGadget.Options; + using PackageManagers; + using PackageUrl; + using System.Text.Json; + using System.Text.RegularExpressions; + + public class FreshTool : OSSGadget + { + public FreshTool(ProjectManagerFactory projectManagerFactory) : base(projectManagerFactory) + { + } + + public FreshTool() : this(new ProjectManagerFactory()) + { + } + + static async Task Main(string[] args) + { + FreshTool freshTool = new FreshTool(); + await freshTool.ParseOptions(args).WithParsedAsync(freshTool.RunAsync); + } + + private async Task RunAsync(FreshToolOptions freshToolOptions) + { + // select output destination and format + SelectOutput(freshToolOptions.OutputFile); + IOutputBuilder outputBuilder = SelectFormat(freshToolOptions.Format); + + int maintainedThresholdDays = freshToolOptions.MaxAgeMaintained > 0 ? freshToolOptions.MaxAgeMaintained : int.MaxValue; + int nonMaintainedThresholdDays = freshToolOptions.MaxAgeUnmaintained > 0 ? freshToolOptions.MaxAgeUnmaintained : int.MaxValue; + int maintainedThresholdVersions = freshToolOptions.MaxOutOfDateVersions; + + string? versionFilter = freshToolOptions.Filter; + + int safeDays = 90; + DateTime NOW = DateTime.Now; + + maintainedThresholdDays = Math.Max(maintainedThresholdDays, safeDays); + nonMaintainedThresholdDays = Math.Max(nonMaintainedThresholdDays, safeDays); + + if (freshToolOptions.Targets is IList targetList && targetList.Count > 0) + { + foreach (string? target in targetList) + { + try + { + PackageURL? purl = new PackageURL(target); + BaseMetadataSource metadataSource = new LibrariesIoMetadataSource(); + Logger.Info("Collecting metadata for {0}", purl); + JsonDocument? metadata = await metadataSource.GetMetadataForPackageUrlAsync(purl, true); + if (metadata != null) + { + JsonElement root = metadata.RootElement; + + string? latestRelease = root.GetProperty("latest_release_number").GetString(); + DateTime latestReleasePublishedAt = root.GetProperty("latest_release_published_at").GetDateTime(); + bool stillMaintained = (NOW - latestReleasePublishedAt).TotalDays < maintainedThresholdDays; + + // Extract versions + IEnumerable versions = root.GetProperty("versions").EnumerateArray(); + + // Filter if needed + if (versionFilter != null) + { + Regex versionFilterRegex = new Regex(versionFilter, RegexOptions.Compiled); + versions = versions.Where(elt => { + string? _version = elt.GetProperty("number").GetString(); + if (_version != null) + { + return versionFilterRegex.IsMatch(_version); + } + return true; + }); + } + // Order by semantic version + versions = versions.OrderBy(elt => { + try + { + string? _v = elt.GetProperty("number").GetString(); + if (_v == null) + { + _v = "0.0.0"; + } else if (_v.Count(ch => ch == '.') == 1) + { + _v = _v + ".0"; + } + return new SemanticVersioning.Version(_v, true); + } + catch(Exception) + { + return new SemanticVersioning.Version("0.0.0"); + } + }); + + int versionIndex = 0; + foreach (JsonElement version in versions) + { + ++versionIndex; + string? versionName = version.GetProperty("number").GetString(); + DateTime publishedAt = version.GetProperty("published_at").GetDateTime(); + string? resultMessage = null; + + if (stillMaintained) + { + if ((NOW - publishedAt).TotalDays > maintainedThresholdDays) + { + resultMessage = $"This version {versionName} was published more than {maintainedThresholdDays} days ago."; + } + + if (maintainedThresholdVersions > 0 && + versionIndex < (versions.Count() - maintainedThresholdVersions)) + { + if ((NOW - publishedAt).TotalDays > safeDays) + { + if (resultMessage != null ) + { + resultMessage += $" In addition, this version was more than {maintainedThresholdVersions} versions out of date."; + } + else + { + resultMessage = $"This version {versionName} was more than {maintainedThresholdVersions} versions out of date."; + } + } + } + } + else + { + if ((NOW - publishedAt).TotalDays > nonMaintainedThresholdDays) + { + resultMessage = $"This version {versionName} was published more than {nonMaintainedThresholdDays} days ago."; + } + } + + // Write output + if (resultMessage != null) + { + Console.WriteLine(resultMessage); + } + else + { + Console.WriteLine($"This version {versionName} is current."); + } + + } + } + } + catch (Exception ex) + { + Logger.Warn("Error processing {0}: {1}", target, ex.Message); + } + } + } + } + } +} \ No newline at end of file diff --git a/src/oss-health/BaseHealthAlgorithm.cs b/src/Shared.CLI/Tools/HealthTool/BaseHealthAlgorithm.cs similarity index 94% rename from src/oss-health/BaseHealthAlgorithm.cs rename to src/Shared.CLI/Tools/HealthTool/BaseHealthAlgorithm.cs index 765c12b3..f49ba09e 100644 --- a/src/oss-health/BaseHealthAlgorithm.cs +++ b/src/Shared.CLI/Tools/HealthTool/BaseHealthAlgorithm.cs @@ -3,7 +3,7 @@ using Microsoft.CST.OpenSource.Helpers; using System.Threading.Tasks; -namespace Microsoft.CST.OpenSource.Health +namespace Microsoft.CST.OpenSource.OssGadget.Tools.HealthTool { /// /// Abstract base class for health algorithms diff --git a/src/oss-health/GitHubHealthAlgorithm.cs b/src/Shared.CLI/Tools/HealthTool/GitHubHealthAlgorithm.cs similarity index 99% rename from src/oss-health/GitHubHealthAlgorithm.cs rename to src/Shared.CLI/Tools/HealthTool/GitHubHealthAlgorithm.cs index b1901528..3415088a 100644 --- a/src/oss-health/GitHubHealthAlgorithm.cs +++ b/src/Shared.CLI/Tools/HealthTool/GitHubHealthAlgorithm.cs @@ -5,7 +5,7 @@ using System.Linq; using System.Threading.Tasks; -namespace Microsoft.CST.OpenSource.Health +namespace Microsoft.CST.OpenSource.OssGadget.Tools.HealthTool { using PackageUrl; diff --git a/src/oss-health/HealthMetrics.cs b/src/Shared.CLI/Tools/HealthTool/HealthMetrics.cs similarity index 98% rename from src/oss-health/HealthMetrics.cs rename to src/Shared.CLI/Tools/HealthTool/HealthMetrics.cs index 696280cf..563d1c08 100644 --- a/src/oss-health/HealthMetrics.cs +++ b/src/Shared.CLI/Tools/HealthTool/HealthMetrics.cs @@ -11,7 +11,7 @@ using System.Text; using System.Text.RegularExpressions; -namespace Microsoft.CST.OpenSource.Health +namespace Microsoft.CST.OpenSource.OssGadget.Tools.HealthTool { using Contracts; using PackageUrl; diff --git a/src/Shared.CLI/Tools/HealthTool/HealthTool.cs b/src/Shared.CLI/Tools/HealthTool/HealthTool.cs new file mode 100644 index 00000000..38f43145 --- /dev/null +++ b/src/Shared.CLI/Tools/HealthTool/HealthTool.cs @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using Microsoft.CST.OpenSource.Shared; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using static Microsoft.CST.OpenSource.Shared.OutputBuilderFactory; + +namespace Microsoft.CST.OpenSource.OssGadget.Tools.HealthTool +{ + using Contracts; + using Helpers; + using Microsoft.CST.OpenSource.PackageManagers; + using Options; + using PackageUrl; + + public class HealthTool : BaseTool + { + + public HealthTool(ProjectManagerFactory projectManagerFactory) : base(projectManagerFactory) + { + } + + public HealthTool() : this(new ProjectManagerFactory()) + { + } + public async Task CheckHealth(PackageURL purl) + { + IBaseProjectManager? packageManager = ProjectManagerFactory.CreateProjectManager(purl); + + if (packageManager != null) + { + string? content = await packageManager.GetMetadataAsync(purl); + if (!string.IsNullOrWhiteSpace(content)) + { + RepoSearch repoSearcher = new RepoSearch(ProjectManagerFactory); + foreach ((PackageURL githubPurl, double _) in await repoSearcher.ResolvePackageLibraryAsync(purl)) + { + try + { + GitHubHealthAlgorithm? healthAlgorithm = new GitHubHealthAlgorithm(githubPurl); + HealthMetrics? health = await healthAlgorithm.GetHealth(); + return health; + } + catch (Exception ex) + { + Logger.Warn(ex, "Unable to calculate health for {0}: {1}", githubPurl, ex.Message); + } + } + } + else + { + Logger.Warn("No metadata found for {0}", purl.ToString()); + } + } + else + { + throw new ArgumentException($"Invalid Package URL type: {purl.Type}"); + } + return null; + } + + private void AppendOutput(IOutputBuilder outputBuilder, PackageURL purl, HealthMetrics healthMetrics) + { + switch (currentOutputFormat) + { + case OutputFormat.text: + default: + outputBuilder.AppendOutput(new List() { + $"Health for {purl} (via {purl})", + healthMetrics.ToString() + }); + break; + + case OutputFormat.sarifv1: + case OutputFormat.sarifv2: + outputBuilder.AppendOutput(healthMetrics.toSarif()); + break; + } + } + + + public async Task RunAsync(HealthToolOptions options) + { + // select output destination and format + SelectOutput(options.OutputFile); + IOutputBuilder outputBuilder = SelectFormat(options.Format ?? OutputFormat.text.ToString()); + if (options.Targets is IList targetList && targetList.Count > 0) + { + foreach (string target in targetList) + { + try + { + // PackageURL requires the @ in a namespace declaration to be escaped + // We find if the namespace contains an @ in the namespace + // And replace it with %40 + string escapedNameSpaceTarget = CliHelpers.EscapeAtSymbolInNameSpace(target); + PackageURL? purl = new PackageURL(escapedNameSpaceTarget); + HealthMetrics? healthMetrics = CheckHealth(purl).Result; + if (healthMetrics == null) + { + Logger.Debug($"Cannot compute Health for {purl}"); + } + else + { + AppendOutput(outputBuilder, purl, healthMetrics); + } + } + catch (Exception ex) + { + Logger.Warn("Error processing {0}: {1}", target, ex.Message); + } + } + outputBuilder.PrintOutput(); + } + RestoreOutput(); + } + } +} \ No newline at end of file diff --git a/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/cargo/autobuild.sh b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/cargo/autobuild.sh new file mode 100644 index 00000000..cece79ea --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/cargo/autobuild.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +PREBUILD_SCRIPT="$1" +BUILD_SCRIPT="$2" +POSTBUILD_SCRIPT="$3" + +if [ ! -z "$PREBUILD_SCRIPT" -a -f "/build-helpers/$PREBUILD_SCRIPT" ]; then + echo "Executing pre-build script: [$PREBUILD_SCRIPT]" + source "/build-helpers/$PREBUILD_SCRIPT" +else + echo "No custom pre-build script found." +fi + +if [ ! -z "$BUILD_SCRIPT" -a -f "/build-helpers/$BUILD_SCRIPT" ]; then + echo "Executing build script: [$BUILD_SCRIPT]" + source "/build-helpers/$BUILD_SCRIPT" +else + echo "No custom build script found. Using auto-builder." + mkdir -p /build-output/work + cargo package --color never -v --target-dir /build-output/workmc + ls -lR /build-output +fi + +if [ ! -z "$POSTBUILD_SCRIPT" -a -f "/build-helpers/$POSTBUILD_SCRIPT" ]; then + echo "Executing post-build script: [$POSTBUILD_SCRIPT]" + source "/build-helpers/$POSTBUILD_SCRIPT" +else + + mv /build-output/work/package/*.crate /build-output/output.archive +fi + +echo "Autobuild complete." diff --git a/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/cpan/autobuild.sh b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/cpan/autobuild.sh new file mode 100644 index 00000000..3da5072c --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/cpan/autobuild.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +PREBUILD_SCRIPT="$1" +BUILD_SCRIPT="$2" +POSTBUILD_SCRIPT="$3" + +if [ ! -z "$PREBUILD_SCRIPT" -a -f "/build-helpers/$PREBUILD_SCRIPT" ]; then + echo "Executing pre-build script: [$PREBUILD_SCRIPT]" + source "/build-helpers/$PREBUILD_SCRIPT" +else + echo "No custom pre-build script found." +fi + +if [ ! -z "$BUILD_SCRIPT" -a -f "/build-helpers/$BUILD_SCRIPT" ]; then + pwd + echo "Executing build script: [$BUILD_SCRIPT]" + source "/build-helpers/$BUILD_SCRIPT" +else + echo "No custom build script found. Using auto-builder." + + echo "Executing cpan build scripts" + [ -f Build.PL ] && perl Build.PL + [ -f Makefile.PL ] && perl Makefile.PL + make manifest + make + make dist +fi + +if [ ! -z "$POSTBUILD_SCRIPT" -a -f "/build-helpers/$POSTBUILD_SCRIPT" ]; then + echo "Executing post-build script: [$POSTBUILD_SCRIPT]" + source "/build-helpers/$POSTBUILD_SCRIPT" +else + cp *.tar.gz /build-output/output.archive +fi + +echo "Autobuild complete." diff --git a/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/gem/autobuild.sh b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/gem/autobuild.sh new file mode 100644 index 00000000..c11e5d95 --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/gem/autobuild.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +PREBUILD_SCRIPT="$1" +BUILD_SCRIPT="$2" +POSTBUILD_SCRIPT="$3" + +if [ ! -z "$PREBUILD_SCRIPT" -a -f "/build-helpers/$PREBUILD_SCRIPT" ]; then + echo "Executing pre-build script: [$PREBUILD_SCRIPT]" + source "/build-helpers/$PREBUILD_SCRIPT" +else + echo "No custom pre-build script found." +fi + +if [ ! -z "$BUILD_SCRIPT" -a -f "/build-helpers/$BUILD_SCRIPT" ]; then + pwd + echo "Executing build script: [$BUILD_SCRIPT]" + source "/build-helpers/$BUILD_SCRIPT" +else + echo "No custom build script found. Using auto-builder." + + echo "Executing gem build scripts" + gem build --output /build-output/output.archive +fi + +if [ ! -z "$POSTBUILD_SCRIPT" -a -f "/build-helpers/$POSTBUILD_SCRIPT" ]; then + echo "Executing post-build script: [$POSTBUILD_SCRIPT]" + source "/build-helpers/$POSTBUILD_SCRIPT" +fi + +echo "Autobuild complete." diff --git a/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/npm/@objectisundefined/typo.build b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/npm/@objectisundefined/typo.build new file mode 100644 index 00000000..095fbfdf --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/npm/@objectisundefined/typo.build @@ -0,0 +1,4 @@ +#!/bin/bash + +yarn +yarn run build \ No newline at end of file diff --git a/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/npm/autobuild.sh b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/npm/autobuild.sh new file mode 100644 index 00000000..7c2a4a84 --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/npm/autobuild.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +PREBUILD_SCRIPT="$1" +BUILD_SCRIPT="$2" +POSTBUILD_SCRIPT="$3" + +if [ ! -z "$PREBUILD_SCRIPT" -a -f "/build-helpers/$PREBUILD_SCRIPT" ]; then + echo "Executing pre-build script: [$PREBUILD_SCRIPT]" + source "/build-helpers/$PREBUILD_SCRIPT" +else + echo "No custom pre-build script found." +fi + +if [ ! -z "$BUILD_SCRIPT" -a -f "/build-helpers/$BUILD_SCRIPT" ]; then + echo "Executing build script: [$BUILD_SCRIPT]" + source "/build-helpers/$BUILD_SCRIPT" +else + echo "No custom build script found. Using auto-builder." + echo "Executing 'npm install'" + npm install + + echo "Executing npm scripts" + # Note, we expect most of these to fail gracefully + npm run preprepare + npm run prepare + npm run postprepare + + npm run prepack + npm run pack + npm run postpack + + npm run build + npm pack + npm run prepublish +fi + +if [ ! -z "$POSTBUILD_SCRIPT" -a -f "/build-helpers/$POSTBUILD_SCRIPT" ]; then + echo "Executing post-build script: [$POSTBUILD_SCRIPT]" + source "/build-helpers/$POSTBUILD_SCRIPT" +else + echo "No custom post-build script found. Using default packer." + echo "Running 'npm pack'" + npm pack --json > /build-output/output.json + cp *.tgz /build-output/output.archive +fi + +echo "Autobuild complete." diff --git a/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/npm/bluebird.build b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/npm/bluebird.build new file mode 100644 index 00000000..e7dd82f2 --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/npm/bluebird.build @@ -0,0 +1,4 @@ +#!/bin/bash + +npm install +npm run prepublish release \ No newline at end of file diff --git a/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/npm/bluebird.prebuild b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/npm/bluebird.prebuild new file mode 100644 index 00000000..9cd2cdc1 --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/npm/bluebird.prebuild @@ -0,0 +1,4 @@ +#!/bin/bash + +# Requires Yarn to be installed +npm -g i yarn diff --git a/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/pypi/autobuild.sh b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/pypi/autobuild.sh new file mode 100644 index 00000000..1f89e0bb --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/BuildHelperScripts/pypi/autobuild.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +PREBUILD_SCRIPT="$1" +BUILD_SCRIPT="$2" +POSTBUILD_SCRIPT="$3" + +if [ ! -z "$PREBUILD_SCRIPT" -a -f "/build-helpers/$PREBUILD_SCRIPT" ]; then + echo "Executing pre-build script: [$PREBUILD_SCRIPT]" + source "/build-helpers/$PREBUILD_SCRIPT" +else + echo "No custom pre-build script found." +fi + +if [ ! -z "$BUILD_SCRIPT" -a -f "/build-helpers/$BUILD_SCRIPT" ]; then + pwd + echo "Executing build script: [$BUILD_SCRIPT]" + source "/build-helpers/$BUILD_SCRIPT" +else + echo "No custom build script found. Using auto-builder." + + echo "Executing Python build scripts" + python -m pip install --upgrade build + python -mbuild +fi + +if [ ! -z "$POSTBUILD_SCRIPT" -a -f "/build-helpers/$POSTBUILD_SCRIPT" ]; then + echo "Executing post-build script: [$POSTBUILD_SCRIPT]" + source "/build-helpers/$POSTBUILD_SCRIPT" +else + tar cvfz /build-output/output.archive dist/* +fi + +echo "Autobuild complete." \ No newline at end of file diff --git a/src/Shared.CLI/Tools/ReproducibleTool/Helpers/DataObjects.cs b/src/Shared.CLI/Tools/ReproducibleTool/Helpers/DataObjects.cs new file mode 100644 index 00000000..4fe20079 --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/Helpers/DataObjects.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using System.Collections.Generic; + +namespace Microsoft.CST.OpenSource.Reproducibility +{ + public class ReproducibleToolResult + { + public string? PackageUrl { get; set; } + + public bool IsReproducible + { + get + { + if (Results == null) + { + return false; + } + + foreach (StrategyResult? result in Results) + { + if (result.IsSuccess && !result.IsError) + { + return true; + } + } + return false; + } + } + + public List? Results { get; set; } + } +} \ No newline at end of file diff --git a/src/Shared.CLI/Tools/ReproducibleTool/Helpers/IgnoreFilter.cs b/src/Shared.CLI/Tools/ReproducibleTool/Helpers/IgnoreFilter.cs new file mode 100644 index 00000000..2fa0c522 --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/Helpers/IgnoreFilter.cs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; + +namespace Microsoft.CST.OpenSource.Reproducibility +{ + using PackageUrl; + + internal class IgnoreFilter + { + private static readonly List FilterText; + + protected static readonly NLog.Logger Logger = NLog.LogManager.GetCurrentClassLogger(); + + /// + /// Initialiizes the ignore filter using the embedded resource. + /// + static IgnoreFilter() + { + Assembly? assembly = Assembly.GetExecutingAssembly(); + string? resourceName = assembly.GetManifestResourceNames().Single(str => str.EndsWith("PackageIgnoreList.txt")); + using Stream? stream = assembly.GetManifestResourceStream(resourceName); + + FilterText = new List(); + + if (stream != null) + { + using StreamReader reader = new StreamReader(stream); + string? line; + while ((line = reader?.ReadLine()) != null) + { + line = line.Trim(); + if (!line.Contains(":") || line.StartsWith("#") || string.IsNullOrWhiteSpace(line)) + { + continue; // Not a valid pattern + } + Logger.Trace("Adding {0} to filter.", line); + FilterText.Add(line); + } + } + else + { + Logger.Warn("Unable to find PackageIgnoreList.txt."); + } + } + + /// + /// Checks to see if a given file should be ignored when making a comparison. + /// + /// + /// + /// + /// + internal static bool IsIgnored(PackageURL? packageUrl, string strategyName, string filePath) + { + bool shouldIgnore = false; + filePath = filePath.Replace("\\", "/").Trim(); + foreach (string? filter in FilterText) + { + string[]? parts = filter.Split(':', 3); + if (parts.Length != 3) + { + continue; // Invalid line + } + string? _packageManager = parts[0].Trim(); + string? _strategy = parts[1].Trim(); + string? _regex = parts[2].Trim(); + + if (string.Equals(_packageManager, "*", StringComparison.InvariantCultureIgnoreCase) || + string.Equals(_packageManager, packageUrl?.Type ?? "*", StringComparison.InvariantCultureIgnoreCase)) + { + if (string.Equals(_strategy, "*", StringComparison.InvariantCultureIgnoreCase) || + string.Equals(strategyName, _strategy, StringComparison.InvariantCultureIgnoreCase)) + { + if (Regex.IsMatch(filePath, _regex, RegexOptions.IgnoreCase)) + { + shouldIgnore = true; + break; + } + } + } + } + Logger.Trace("IsIgnored({0}, {1}, {2} => {3}", packageUrl, strategyName, filePath, shouldIgnore); + return shouldIgnore; + } + } +} \ No newline at end of file diff --git a/src/Shared.CLI/Tools/ReproducibleTool/Helpers/OssReproducibleHelpers.cs b/src/Shared.CLI/Tools/ReproducibleTool/Helpers/OssReproducibleHelpers.cs new file mode 100644 index 00000000..5256f05c --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/Helpers/OssReproducibleHelpers.cs @@ -0,0 +1,342 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using DiffPlex.DiffBuilder; +using DiffPlex.DiffBuilder.Model; +using Microsoft.CST.OpenSource.Helpers; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; + +namespace Microsoft.CST.OpenSource.Reproducibility +{ + public enum DirectoryDifferenceOperation + { + Added, + Removed, + Equals, + Modified + } + + public class DirectoryDifference + { + public string Filename { get; set; } = ""; + public DirectoryDifferenceOperation Operation { get; set; } + public string? ComparisonFile { get; set; } + public IEnumerable? Difference { get; set; } + //public SideBySideDiffModel? Differences { get; set; } + } + + public class OssReproducibleHelpers + { + protected static readonly NLog.Logger Logger = NLog.LogManager.GetCurrentClassLogger(); + + static OssReproducibleHelpers() + { + } + + internal static void DirectoryCopy(string sourceDirName, string destDirName, bool copySubDirs = true) + { + // Get the subdirectories for the specified directory. + DirectoryInfo dir = new DirectoryInfo(sourceDirName); + + if (!dir.Exists) + { + throw new DirectoryNotFoundException( + "Source directory does not exist or could not be found: " + + sourceDirName); + } + + DirectoryInfo[] dirs = dir.GetDirectories(); + + // If the destination directory doesn't exist, create it. + Directory.CreateDirectory(destDirName); + + // Get the files in the directory and copy them to the new location. + FileInfo[] files = dir.GetFiles(); + foreach (FileInfo file in files) + { + string tempPath = Path.Combine(destDirName, file.Name); + file.CopyTo(tempPath, false); + } + + // If copying subdirectories, copy them and their contents to new location. + if (copySubDirs) + { + foreach (DirectoryInfo subdir in dirs) + { + string tempPath = Path.Combine(destDirName, subdir.Name); + DirectoryCopy(subdir.FullName, tempPath, copySubDirs); + } + } + } + + /// + /// Identifies all elements in leftDirectory that either don't exist in rightDirectory or + /// exist with different content. This function is "smart" in that it is resilient to + /// changes in directory names, meaning: leftDirectory, file = /foo/bar/quux/baz.txt + /// rightDirectory, file = /bing/quux/baz.txt These would be correctly classified as the + /// same file. If there was another file in rightDirectory: rightDirectory, file = + /// /qwerty/bing/quux/baz.txt Then that one would match better, since it has a longer suffix + /// in common with the leftDirectory file. + /// + /// Typically the existing package + /// Source repo, or built package, etc. + /// + public static IEnumerable DirectoryDifference(string leftDirectory, string rightDirectory, DiffTechnique diffTechnique) + { + List? results = new List(); + + // left = built package, right = source repo + IEnumerable? leftFiles = Directory.EnumerateFiles(leftDirectory, "*", SearchOption.AllDirectories); + IEnumerable? rightFiles = Directory.EnumerateFiles(rightDirectory, "*", SearchOption.AllDirectories); + + foreach (string? leftFile in leftFiles) + { + IEnumerable? closestMatches = GetClosestFileMatch(leftFile, rightFiles); + string? closestMatch = closestMatches.FirstOrDefault(); + if (closestMatch == null) + { + results.Add(new DirectoryDifference() + { + Filename = leftFile[leftDirectory.Length..].Replace("\\", "/"), + ComparisonFile = null, + Operation = DirectoryDifferenceOperation.Added + }); + } + else + { + string? filenameContent = File.ReadAllText(leftFile); + string? closestMatchContent = File.ReadAllText(closestMatch); + if (!string.Equals(filenameContent, closestMatchContent)) + { + if (diffTechnique == DiffTechnique.Normalized) + { + filenameContent = NormalizeContent(leftFile); + closestMatchContent = NormalizeContent(closestMatch); + } + } + + DiffPaneModel? diff = InlineDiffBuilder.Diff(filenameContent, closestMatchContent, ignoreWhiteSpace: true, ignoreCase: false); + if (diff.HasDifferences) + { + results.Add(new DirectoryDifference() + { + Filename = leftFile[leftDirectory.Length..].Replace("\\", "/"), + ComparisonFile = closestMatch[rightDirectory.Length..].Replace("\\", "/"), + Operation = DirectoryDifferenceOperation.Modified, + Difference = diff.Lines + }); + } + } + } + return results; + } + + public static bool RunCommand(string workingDirectory, string filename, IEnumerable args, out string? stdout, out string? stderr) + { + Logger.Debug("RunCommand({0}, {1})", filename, string.Join(';', args)); + + ProcessStartInfo? startInfo = new ProcessStartInfo() + { + CreateNoWindow = false, + UseShellExecute = false, + FileName = filename, + RedirectStandardOutput = true, + RedirectStandardError = true, + WorkingDirectory = workingDirectory + }; + foreach (string? arg in args) + { + startInfo.ArgumentList.Add(arg); + } + + Stopwatch? timer = new Stopwatch(); + timer.Start(); + + using Process? process = Process.Start(startInfo); + if (process == null) + { + stdout = null; + stderr = null; + return false; + } + StringBuilder? sbStdout = new StringBuilder(); + StringBuilder? sbStderr = new StringBuilder(); + object? sbStdoutLock = new Object(); + object? sbStderrLock = new Object(); + + process.OutputDataReceived += (sender, args) => + { + if (args.Data != null) + { + Logger.Trace("OUT: {0}", args.Data); + lock (sbStdoutLock) + { + sbStdout.AppendLine(args.Data); + } + } + }; + process.ErrorDataReceived += (sender, args) => + { + if (args.Data != null) + { + Logger.Trace("ERR: {0}", args.Data); + lock (sbStderrLock) + { + sbStderr.AppendLine(args.Data); + } + } + }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + // Apply a timeout + int timeout = 1000 * 60 * 15; // 15 minute default timeout + string? envTimeout = Environment.GetEnvironmentVariable("OSS_REPRODUCIBLE_COMMAND_TIMEOUT"); + if (envTimeout != null) + { + if (int.TryParse(envTimeout, out int envTimeoutInt)) + { + timeout = envTimeoutInt; + } + } + process.WaitForExit(timeout); + + lock (sbStderrLock) + { + stderr = sbStderr.ToString(); + } + lock (sbStdoutLock) + { + stdout = sbStdout.ToString(); + } + + timer.Stop(); + Logger.Debug("Elapsed time: {0}s", timer.Elapsed.TotalSeconds); + Logger.Debug("Exit Code: {0}", process.ExitCode); + + return process.ExitCode == 0; + } + + /// + /// Attempts to "normalize" source code content by beautifying it. In some cases, this can + /// remove trivial differences. Uses the NPM 'prettier' module within a docker container. + /// + /// File to normalize + /// Normalized content, or the raw file content. + public static string NormalizeContent(string filename) + { + if (filename.EndsWith(".js") || filename.EndsWith(".ts")) + { + Logger.Debug("Normalizing {0}", filename); + string? tempDirectoryName = Guid.NewGuid().ToString(); + if (!Directory.Exists(tempDirectoryName)) + { + Directory.CreateDirectory(tempDirectoryName); + } + string? extension = Path.GetExtension(filename); + string? tempFile = Path.ChangeExtension(Path.Join(tempDirectoryName, "temp"), extension); + byte[]? bytes = File.ReadAllBytes(filename); + File.WriteAllBytes(tempFile, bytes); + + bool runResult = RunCommand(tempDirectoryName, "docker", new[] { + "run", + "--rm", + "--memory=1g", + "--cpus=1.0", + "--volume", $"{Path.GetFullPath(tempDirectoryName)}:/repo", + "--workdir=/repo", + "tmknom/prettier", + Path.ChangeExtension("/repo/temp", extension) + }, out string? stdout, out string? stderr); + + FileSystemHelper.RetryDeleteDirectory(tempDirectoryName); + if (stdout != null) + { + return stdout; + } + else + { + return File.ReadAllText(filename); + } + } + else + { + return File.ReadAllText(filename); + } + } + + /// + /// Identifes the closest filename match to the target filename. + /// + /// + /// + /// + public static IEnumerable GetClosestFileMatch(string target, IEnumerable filenames) + { + target = target.Replace("\\", "/").Trim().Trim(' ', '/'); + filenames = filenames.Select(f => f.Replace("\\", "/").Trim().TrimEnd('/')); + + string? candidate = ""; + + HashSet? bestCandidates = new HashSet(); + int bestCandidateScore = 0; + + int targetNumDirs = target.Count(ch => ch == '/') + 1; + + foreach (string? part in target.Split('/').Reverse()) + { + candidate = Path.Join(part, candidate).Replace("\\", "/"); + foreach (string? filename in filenames) + { + if (filename.EndsWith(candidate, StringComparison.InvariantCultureIgnoreCase)) + { + int candidateScore = candidate.Count(ch => ch == '/'); + if (candidateScore > bestCandidateScore) + { + bestCandidateScore = candidateScore; + bestCandidates.Clear(); + bestCandidates.Add(filename); + } + else if (candidateScore == bestCandidateScore) + { + bestCandidates.Add(filename); + } + } + } + } + List? resultList = bestCandidates.ToList(); + resultList.Sort((a, b) => 1 - a.Length.CompareTo(b.Length)); + return resultList; + } + + internal static string? GetFirstNonSingularDirectory(string? directory) + { + if (directory == null || !Directory.Exists(directory)) + { + return null; + } + IEnumerable? entries = Directory.EnumerateFileSystemEntries(directory, "*", SearchOption.TopDirectoryOnly); + if (entries.Count() == 1) + { + string? firstEntry = entries.First(); + if (Directory.Exists(firstEntry)) + { + return GetFirstNonSingularDirectory(firstEntry); + } + else + { + return directory; + } + } + else + { + return directory; + } + } + } +} \ No newline at end of file diff --git a/src/Shared.CLI/Tools/ReproducibleTool/ReproducibleTool.cs b/src/Shared.CLI/Tools/ReproducibleTool/ReproducibleTool.cs new file mode 100644 index 00000000..49fb737e --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/ReproducibleTool.cs @@ -0,0 +1,493 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using CommandLine; +using CommandLine.Text; +using DiffPlex.DiffBuilder.Model; +using Microsoft.CST.OpenSource.Reproducibility; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Threading.Tasks; +using static Crayon.Output; + +namespace Microsoft.CST.OpenSource +{ + using Microsoft.CST.OpenSource.Helpers; + using OssGadget.Options; + using PackageManagers; + using PackageUrl; + + public enum DiffTechnique + { + Strict, + Normalized + }; + + public class ReproducibleTool : BaseTool + { + public ReproducibleTool(ProjectManagerFactory projectManagerFactory) : base(projectManagerFactory) + { + } + + public ReproducibleTool() : this(new ProjectManagerFactory()) + { + } + + /// + /// Algorithm: 0.0 = Worst, 1.0 = Best 1.0 => Bit for bit archive match. @TODO Refactor + /// this into the individual strategy objects. + /// + /// + /// + public KeyValuePair GetReproducibilityScore(ReproducibleToolResult fullResult) + { + if (fullResult.Results == null) + { + return KeyValuePair.Create(0.0, "No semantic equivalency results were created."); + } + + KeyValuePair bestScore = KeyValuePair.Create(0.0, "No strategies were able to successfully derive the package from the source code."); + + foreach (StrategyResult? result in fullResult.Results) + { + string? filesString = result.NumIgnoredFiles == 1 ? "file" : "files"; + + if (string.Equals(result.StrategyName, "PackageMatchesSourceStrategy")) + { + if (result.IsSuccess && !result.IsError) + { + if (result.NumIgnoredFiles == 0) + { + if (bestScore.Key < 0.80) + { + bestScore = KeyValuePair.Create(0.80, $"Package contents match the source repository contents, file-by-file, with no ignored {filesString}."); + } + } + else + { + if (bestScore.Key < 0.70) + { + bestScore = KeyValuePair.Create(0.70, $"Package contents match the source repository contents, file-by-file, with {result.NumIgnoredFiles} ignored {filesString}."); + } + } + } + } + else if (string.Equals(result.StrategyName, "PackageContainedInSourceStrategy")) + { + if (result.IsSuccess && !result.IsError) + { + if (result.NumIgnoredFiles == 0) + { + if (bestScore.Key < 0.75) + { + bestScore = KeyValuePair.Create(0.75, $"Package is a subset of the source repository contents, with no ignored {filesString}."); + } + } + else + { + if (bestScore.Key < 0.65) + { + bestScore = KeyValuePair.Create(0.65, $"Package is a subset of the source repository contents, with {result.NumIgnoredFiles} ignored {filesString}."); + } + } + } + } + else if (string.Equals(result.StrategyName, "AutoBuildProducesSamePackage")) + { + if (result.IsSuccess && !result.IsError) + { + if (result.NumIgnoredFiles == 0) + { + if (bestScore.Key < 0.90) + { + bestScore = KeyValuePair.Create(0.90, $"Package was re-built from source, with no ignored {filesString}."); + } + } + else + { + if (bestScore.Key < 0.65) + { + bestScore = KeyValuePair.Create(0.65, $"Package was re-built from source, with {result.NumIgnoredFiles} ignored {filesString}."); + } + } + } + } + else if (string.Equals(result.StrategyName, "OryxBuildStrategy")) + { + if (result.IsSuccess && !result.IsError) + { + if (result.NumIgnoredFiles == 0) + { + if (bestScore.Key < 0.90) + { + bestScore = KeyValuePair.Create(0.90, $"Package was re-built from source, with no ignored {filesString}."); + } + } + else + { + if (bestScore.Key < 0.65) + { + bestScore = KeyValuePair.Create(0.65, $"Package was re-built from source, with {result.NumIgnoredFiles} ignored {filesString}."); + } + } + } + } + } + return bestScore; + } + + private async Task RunAsync(ReproducibleToolOptions reproducibleToolOptions) + { + if (reproducibleToolOptions.ShowAllDifferences) + { + reproducibleToolOptions.ShowDifferences = true; + } + + // Validate strategies (before we do any other processing + IEnumerable? runSpecificStrategies = null; + if (reproducibleToolOptions.SpecificStrategies != null) + { + IEnumerable? requestedStrategies = reproducibleToolOptions.SpecificStrategies.Split(',').Select(s => s.Trim().ToLowerInvariant()).Distinct(); + runSpecificStrategies = typeof(BaseStrategy).Assembly.GetTypes() + .Where(t => t.IsSubclassOf(typeof(BaseStrategy))) + .Where(t => requestedStrategies.Contains(t.Name.ToLowerInvariant())); + Logger.Debug("Specific strategies requested: {0}", string.Join(", ", runSpecificStrategies.Select(t => t.Name))); + + if (requestedStrategies.Count() != runSpecificStrategies.Count()) + { + Logger.Debug("Invalid strategies."); + Console.WriteLine("Invalid strategy, available options are:"); + IEnumerable? allStrategies = typeof(BaseStrategy).Assembly.GetTypes().Where(t => t.IsSubclassOf(typeof(BaseStrategy))); + foreach (Type? s in allStrategies) + { + Console.WriteLine($" * {s.Name}"); + } + Console.WriteLine("Example: oss-reproducible --specific-strategies AutoBuildProducesSamePackage,PackageMatchesSourceStrategy pkg:npm/left-pad@1.3.0"); + return; + } + } + + // Expand targets + List? targets = new List(); + foreach (string? target in reproducibleToolOptions.Targets ?? Array.Empty()) + { + PackageURL? purl = new PackageURL(target); + PackageDownloader? downloader = new PackageDownloader(purl, ProjectManagerFactory, "temp"); + foreach (PackageURL? version in downloader.PackageVersions) + { + targets.Add(version.ToString()); + } + } + List? finalResults = new List(); + + foreach (string? target in targets) + { + try + { + Console.WriteLine($"Analyzing {target}..."); + Logger.Debug("Processing: {0}", target); + + PackageURL? purl = new PackageURL(target); + if (purl.Version == null) + { + Logger.Error("Package is missing a version, which is required for this tool."); + continue; + } + + string? tempDirectoryName = Path.Join(Path.GetTempPath(), Guid.NewGuid().ToString().Substring(0, 8)); + FileSystemHelper.RetryDeleteDirectory(tempDirectoryName); + // Download the package + Console.WriteLine("Downloading package..."); + PackageDownloader? packageDownloader = new PackageDownloader(purl, ProjectManagerFactory, Path.Join(tempDirectoryName, "package")); + List? downloadResults = await packageDownloader.DownloadPackageLocalCopy(purl, false, true); + + if (!downloadResults.Any()) + { + Logger.Debug("Unable to download package."); + } + + // Locate the source + Console.WriteLine("Locating source..."); + FindSourceTool? findSourceTool = new FindSourceTool(ProjectManagerFactory); + Dictionary? sourceMap = await findSourceTool.FindSourceAsync(purl); + if (sourceMap.Any()) + { + List>? sourceMapList = sourceMap.ToList(); + sourceMapList.Sort((a, b) => a.Value.CompareTo(b.Value)); + PackageURL? bestSourcePurl = sourceMapList.Last().Key; + if (string.IsNullOrEmpty(bestSourcePurl.Version)) + { + // Tie back the original version to the new PackageURL + bestSourcePurl = new PackageURL(bestSourcePurl.Type, bestSourcePurl.Namespace, bestSourcePurl.Name, + purl.Version, bestSourcePurl.Qualifiers, bestSourcePurl.Subpath); + } + Logger.Debug("Identified best source code repository: {0}", bestSourcePurl); + + // Download the source + Console.WriteLine("Downloading source..."); + foreach (string? reference in new[] { bestSourcePurl.Version, reproducibleToolOptions.OverrideSourceReference, "master", "main" }) + { + if (string.IsNullOrWhiteSpace(reference)) + { + continue; + } + Logger.Debug("Trying to download package, version/reference [{0}].", reference); + PackageURL? purlRef = new PackageURL(bestSourcePurl.Type, bestSourcePurl.Namespace, bestSourcePurl.Name, reference, bestSourcePurl.Qualifiers, bestSourcePurl.Subpath); + packageDownloader = new PackageDownloader(purlRef, ProjectManagerFactory, Path.Join(tempDirectoryName, "src")); + downloadResults = await packageDownloader.DownloadPackageLocalCopy(purlRef, false, true); + if (downloadResults.Any()) + { + break; + } + } + if (!downloadResults.Any()) + { + Logger.Debug("Unable to download source."); + } + } + else + { + Logger.Debug("Unable to locate source repository."); + } + + // Execute all available strategies + StrategyOptions? strategyOptions = new StrategyOptions() + { + PackageDirectory = Path.Join(tempDirectoryName, "package"), + SourceDirectory = Path.Join(tempDirectoryName, "src"), + PackageUrl = purl, + TemporaryDirectory = Path.GetFullPath(tempDirectoryName), + DiffTechnique = reproducibleToolOptions.DiffTechnique + }; + + // First, check to see how many strategies apply + IEnumerable? strategies = runSpecificStrategies; + if (strategies == null || !strategies.Any()) + { + strategies = BaseStrategy.GetStrategies(strategyOptions) ?? Array.Empty(); + } + + int numStrategiesApplies = 0; + foreach (Type? strategy in strategies) + { + ConstructorInfo? ctor = strategy.GetConstructor(new Type[] { typeof(StrategyOptions) }); + if (ctor != null) + { + BaseStrategy? strategyObject = (BaseStrategy)(ctor.Invoke(new object?[] { strategyOptions })); + if (strategyObject.StrategyApplies()) + { + numStrategiesApplies++; + } + } + } + + Console.Write($"Out of {Yellow(strategies.Count().ToString())} potential strategies, {Yellow(numStrategiesApplies.ToString())} apply. "); + if (reproducibleToolOptions.AllStrategies) + { + Console.WriteLine("Analysis will continue even after a successful strategy is found."); + } + else + { + Console.WriteLine("Analysis will stop after the first successful strategy is found."); + } + List strategyResults = new List(); + + Console.WriteLine($"\n{Blue("Results: ")}"); + + bool hasSuccessfulStrategy = false; + + foreach (Type? strategy in strategies) + { + ConstructorInfo? ctor = strategy.GetConstructor(new Type[] { typeof(StrategyOptions) }); + if (ctor != null) + { + // Create a temporary directory, copy the contents from source/package + // so that this strategy can modify the contents without affecting other strategies. + StrategyOptions? tempStrategyOptions = new StrategyOptions + { + PackageDirectory = Path.Join(strategyOptions.TemporaryDirectory, strategy.Name, "package"), + SourceDirectory = Path.Join(strategyOptions.TemporaryDirectory, strategy.Name, "src"), + TemporaryDirectory = Path.Join(strategyOptions.TemporaryDirectory, strategy.Name), + PackageUrl = strategyOptions.PackageUrl, + IncludeDiffoscope = reproducibleToolOptions.ShowDifferences + }; + + try + { + OssReproducibleHelpers.DirectoryCopy(strategyOptions.PackageDirectory, tempStrategyOptions.PackageDirectory); + OssReproducibleHelpers.DirectoryCopy(strategyOptions.SourceDirectory, tempStrategyOptions.SourceDirectory); + } + catch (Exception ex) + { + Logger.Debug(ex, "Error copying directory for strategy. Aborting execution."); + } + + System.IO.Directory.CreateDirectory(tempStrategyOptions.PackageDirectory); + System.IO.Directory.CreateDirectory(tempStrategyOptions.SourceDirectory); + + try + { + BaseStrategy? strategyObject = (BaseStrategy)(ctor.Invoke(new object?[] { tempStrategyOptions })); + StrategyResult? strategyResult = strategyObject.Execute(); + + if (strategyResult != null) + { + strategyResults.Add(strategyResult); + } + + if (strategyResult != null) + { + if (strategyResult.IsSuccess) + { + Console.WriteLine($" [{Bold().Yellow("PASS")}] {Yellow(strategy.Name)}"); + hasSuccessfulStrategy = true; + } + else + { + Console.WriteLine($" [{Red("FAIL")}] {Red(strategy.Name)}"); + } + + if (reproducibleToolOptions.ShowDifferences) + { + foreach (StrategyResultMessage? resultMessage in strategyResult.Messages) + { + if (resultMessage.Filename != null && resultMessage.CompareFilename != null) + { + Console.WriteLine($" {Bright.Black("(")}{Blue("P ")}{Bright.Black(")")} {resultMessage.Filename}"); + Console.WriteLine($" {Bright.Black("(")}{Blue(" S")}{Bright.Black(")")} {resultMessage.CompareFilename}"); + } + else if (resultMessage.Filename != null) + { + Console.WriteLine($" {Bright.Black("(")}{Blue("P+")}{Bright.Black(")")} {resultMessage.Filename}"); + } + else if (resultMessage.CompareFilename != null) + { + Console.WriteLine($" {Bright.Black("(")}{Blue("S+")}{Bright.Black(")")} {resultMessage.CompareFilename}"); + } + + IEnumerable? differences = resultMessage.Differences ?? Array.Empty(); + + int maxShowDifferences = 20; + int numShowDifferences = 0; + + foreach (DiffPiece? diff in differences) + { + if (!reproducibleToolOptions.ShowAllDifferences && numShowDifferences > maxShowDifferences) + { + Console.WriteLine(Background.Blue(Bold().White("NOTE: Additional differences exist but are not shown. Pass --show-all-differences to view them all."))); + break; + } + + switch (diff.Type) + { + case ChangeType.Inserted: + Console.WriteLine($"{Bright.Black(diff.Position + ")")}\t{Red("+")} {Blue(diff.Text)}"); + ++numShowDifferences; + break; + + case ChangeType.Deleted: + Console.WriteLine($"\t{Green("-")} {Green(diff.Text)}"); + ++numShowDifferences; + break; + + default: + break; + } + } + if (numShowDifferences > 0) + { + Console.WriteLine(); + } + } + + string? diffoscopeFile = Guid.NewGuid().ToString() + ".html"; + File.WriteAllText(diffoscopeFile, strategyResult.Diffoscope); + Console.WriteLine($" Diffoscope results written to {diffoscopeFile}."); + } + } + else + { + Console.WriteLine(Green($" [-] {strategy.Name}")); + } + } + catch (Exception ex) + { + Logger.Warn(ex, "Error processing {0}: {1}", strategy, ex.Message); + Logger.Debug(ex.StackTrace); + } + } + + if (hasSuccessfulStrategy && !reproducibleToolOptions.AllStrategies) + { + break; // We don't need to continue + } + } + + ReproducibleToolResult? reproducibilityToolResult = new ReproducibleToolResult + { + PackageUrl = purl.ToString(), + Results = strategyResults + }; + + finalResults.Add(reproducibilityToolResult); + + (double score, string scoreText) = GetReproducibilityScore(reproducibilityToolResult); + Console.WriteLine($"\n{Blue("Summary:")}"); + string? scoreDisplay = $"{(score * 100.0):0.#}"; + if (reproducibilityToolResult.IsReproducible) + { + Console.WriteLine($" [{Yellow(scoreDisplay + "%")}] {Yellow(scoreText)}"); + } + else + { + Console.WriteLine($" [{Red(scoreDisplay + "%")}] {Red(scoreText)}"); + } + + if (reproducibleToolOptions.LeaveIntermediateFiles) + { + Console.WriteLine(); + Console.WriteLine($"Intermediate files are located in [{tempDirectoryName}]."); + } + else + { + FileSystemHelper.RetryDeleteDirectory(tempDirectoryName); + } + } + catch (Exception ex) + { + Logger.Warn(ex, "Error processing {0}: {1}", target, ex.Message); + Logger.Debug(ex.StackTrace); + } + } + + if (finalResults.Any()) + { + // Write the output somewhere + string? jsonResults = Newtonsoft.Json.JsonConvert.SerializeObject(finalResults, Newtonsoft.Json.Formatting.Indented); + if (!string.IsNullOrWhiteSpace(reproducibleToolOptions.OutputFile) && !string.Equals(reproducibleToolOptions.OutputFile, "-", StringComparison.InvariantCultureIgnoreCase)) + { + try + { + File.WriteAllText(reproducibleToolOptions.OutputFile, jsonResults); + Console.WriteLine($"Detailed results are available in {reproducibleToolOptions.OutputFile}."); + } + catch (Exception ex) + { + Logger.Warn(ex, "Unable to write to {0}. Writing to console instead.", reproducibleToolOptions.OutputFile); + Console.WriteLine(jsonResults); + } + } + else if (string.Equals(reproducibleToolOptions.OutputFile, "-", StringComparison.InvariantCultureIgnoreCase)) + { + Console.WriteLine(jsonResults); + } + } + else + { + Logger.Debug("No results were produced."); + } + } + } +} diff --git a/src/Shared.CLI/Tools/ReproducibleTool/Strategies/AutoBuildProducesSamePackage.cs b/src/Shared.CLI/Tools/ReproducibleTool/Strategies/AutoBuildProducesSamePackage.cs new file mode 100644 index 00000000..f2a4f4fd --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/Strategies/AutoBuildProducesSamePackage.cs @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using Microsoft.CST.RecursiveExtractor; +using NLog; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Microsoft.CST.OpenSource.Reproducibility +{ + using PackageUrl; + + /// + /// This strategy identifies packages as reproducible if running `npm pack` on the source + /// repository produces a file that matches the content of the package downloaded from the registry. + /// + internal class AutoBuildProducesSamePackage : BaseStrategy + { + private static readonly Dictionary DOCKER_CONTAINERS = new() + { + { "npm", "node:latest" }, + { "gem", "ruby:latest" }, + { "cpan", "perl:latest" }, + { "pypi", "python:latest" }, + { "cargo", "rust:latest" } + }; + + public override StrategyPriority PRIORITY => StrategyPriority.Medium; + + public AutoBuildProducesSamePackage(StrategyOptions options) : base(options) + { + } + + /// + /// This strategy applies when the source and package directories exist, as well as if an + /// autobuilder script is available. + /// + /// + public override bool StrategyApplies() + { + if (!GenericStrategyApplies(new[] { Options.SourceDirectory, Options.PackageDirectory })) + { + return false; + } + + if (!File.Exists(Path.Join("BuildHelperScripts", Options.PackageUrl?.Type, "autobuild.sh"))) + { + Logger.Debug("Strategy {0} does not apply because no autobuilder script could be found.", GetType().Name); + return false; + } + + if (GetPathToCommand(new[] { "docker" }) == null) + { + Logger.Debug("Strategy {0} cannot be used, as Docker does not appear to be installed.", GetType().Name); + return false; + } + + if (!DOCKER_CONTAINERS.TryGetValue(Options.PackageUrl?.Type!, out _)) + { + Logger.Debug("Strategy {0} does not apply because no docker container is known for type: {0}", Options.PackageUrl?.Type); + return false; + } + return true; + } + + public override StrategyResult? Execute() + { + Logger.Debug("Executing {0} reproducibility strategy.", GetType().Name); + if (!StrategyApplies()) + { + Logger.Debug("Strategy does not apply, so cannot execute."); + return null; + } + + string? workingDirectory = OssReproducibleHelpers.GetFirstNonSingularDirectory(Options.SourceDirectory); + if (workingDirectory == null) + { + Logger.Warn("Unable to find correct source directory to run `npm pack` from. Unable to continue."); + return null; + } + string? outputDirectory = Path.Join(Options.TemporaryDirectory, "build-output"); + + StrategyResult? strategyResult = new StrategyResult() + { + Strategy = GetType() + }; + + string? autoBuilderScript = Path.Join("/build-helpers", Options.PackageUrl?.Type, "autobuild.sh").Replace("\\", "/"); + string? customPrebuild = GetCustomScript(Options.PackageUrl!, "prebuild")?.Replace("BuildHelperScripts/", "") ?? ""; + string? customBuild = GetCustomScript(Options.PackageUrl!, "build")?.Replace("BuildHelperScripts/", "") ?? ""; + string? customPostBuild = GetCustomScript(Options.PackageUrl!, "postbuild")?.Replace("BuildHelperScripts/", "") ?? ""; + if (!DOCKER_CONTAINERS.TryGetValue(Options.PackageUrl!.Type!, out string? dockerContainerName)) + { + Logger.Debug("No docker container is known for type: {0}", Options.PackageUrl.Type); + return null; + } + + bool runResult = OssReproducibleHelpers.RunCommand(workingDirectory, "docker", new[] { + "run", + "--rm", + "--memory=4g", + "--cpus=0.5", + "--volume", $"{Path.GetFullPath(workingDirectory)}:/repo", + "--volume", $"{Path.GetFullPath(outputDirectory)}:/build-output", + "--volume", $"{Path.GetFullPath("BuildHelperScripts")}:/build-helpers", + "--workdir=/repo", + dockerContainerName, + "bash", + autoBuilderScript, + customPrebuild, + customBuild, + customPostBuild + }, out string? stdout, out string? stderr); + if (runResult) + { + string? packedFilenamePath = Path.Join(outputDirectory, "output.archive"); + if (!File.Exists(packedFilenamePath)) + { + Logger.Warn("Unable to find AutoBuilder archive."); + strategyResult.IsError = true; + strategyResult.Summary = "The AutoBuilder did not produce an output archive."; + return strategyResult; + } + + Extractor? extractor = new Extractor(); + string? packedDirectory = Path.Join(Options.TemporaryDirectory, "src_packed"); + extractor.ExtractToDirectoryAsync(packedDirectory, packedFilenamePath).Wait(); + + if (Options.IncludeDiffoscope) + { + string? diffoscopeTempDir = Path.Join(Options.TemporaryDirectory, "diffoscope"); + string? diffoscopeResults = GenerateDiffoscope(diffoscopeTempDir, packedDirectory, Options.PackageDirectory!); + strategyResult.Diffoscope = diffoscopeResults; + } + + IEnumerable? diffResults = OssReproducibleHelpers.DirectoryDifference(Options.PackageDirectory!, packedDirectory, Options.DiffTechnique); + int diffResultsOriginalCount = diffResults.Count(); + diffResults = diffResults.Where(d => !IgnoreFilter.IsIgnored(Options.PackageUrl, GetType().Name, d.Filename)); + strategyResult.NumIgnoredFiles += (diffResultsOriginalCount - diffResults.Count()); + strategyResult.AddDifferencesToStrategyResult(diffResults); + } + else + { + strategyResult.IsError = true; + strategyResult.Summary = "The AutoBuilder did not complete successfully."; + } + + return strategyResult; + } + + internal static string? GetCustomScript(PackageURL packageUrl, string scriptType) + { + if (packageUrl == null || string.IsNullOrWhiteSpace(scriptType)) + { + return null; + } + string? targetWithVersion; + string? targetWithoutVersion; + + if (!string.IsNullOrEmpty(packageUrl.Namespace)) + { + targetWithVersion = Path.Join("BuildHelperScripts", packageUrl.Type, packageUrl.Namespace, packageUrl.Name + "@" + packageUrl.Version + $".{scriptType}"); + targetWithoutVersion = Path.Join("BuildHelperScripts", packageUrl.Type, packageUrl.Namespace, packageUrl.Name + $".{scriptType}"); + } + else + { + targetWithVersion = Path.Join("BuildHelperScripts", packageUrl.Type, packageUrl.Name + "@" + packageUrl.Version + $".{scriptType}"); + targetWithoutVersion = Path.Join("BuildHelperScripts", packageUrl.Type, packageUrl.Name + $".{scriptType}"); + } + Func normalize = (s => + { + return s.Replace("%40", "@").Replace("%2F", "/").Replace("%2f", "/"); + }); + + targetWithVersion = normalize(targetWithVersion); + targetWithoutVersion = normalize(targetWithoutVersion); + + if (File.Exists(targetWithVersion)) + { + return targetWithVersion.Replace("\\", "/"); + } + if (File.Exists(targetWithoutVersion)) + { + return targetWithoutVersion.Replace("\\", "/"); + } + return null; + } + } +} \ No newline at end of file diff --git a/src/Shared.CLI/Tools/ReproducibleTool/Strategies/BaseStrategy.cs b/src/Shared.CLI/Tools/ReproducibleTool/Strategies/BaseStrategy.cs new file mode 100644 index 00000000..a34c24c0 --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/Strategies/BaseStrategy.cs @@ -0,0 +1,299 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using DiffPlex.DiffBuilder.Model; +using SharpCompress.Archives; +using SharpCompress.Archives.Zip; +using SharpCompress.Common; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text.Json.Serialization; + +namespace Microsoft.CST.OpenSource.Reproducibility +{ + using PackageUrl; + + public enum StrategyPriority + { + None = 0, + Low = 10, + Medium = 20, + High = 30 + } + + public class StrategyOptions + { + public PackageURL? PackageUrl { get; set; } + public string? SourceDirectory { get; set; } + public string? PackageDirectory { get; set; } + public string? TemporaryDirectory { get; set; } + public DiffTechnique DiffTechnique { get; set; } = DiffTechnique.Normalized; + public bool IncludeDiffoscope { get; set; } = false; + } + + public class StrategyResultMessage + { + public StrategyResultMessage() + { + Text = ""; + Filename = ""; + } + + public string Text { get; set; } + public string? Filename { get; set; } + public string? CompareFilename { get; set; } + public IEnumerable? Differences { get; set; } + } + + public class StrategyResult + { + public StrategyResult() + { + Messages = new HashSet(); + } + + [JsonIgnore] + public Type? Strategy { get; set; } + + public string? StrategyName { get => Strategy?.Name; } + public string? Summary { get; set; } + public HashSet Messages; + public bool IsSuccess { get; set; } = false; + public bool IsError { get; set; } = false; + public int NumIgnoredFiles { get; set; } = 0; + public string? Diffoscope { get; set; } + + /// + /// Adds a collection of DirectoryDiffererence objects to this strategy result. + /// + /// + /// Differences between two sets of files (filename / comparefilename) + /// + /// + /// + public bool AddDifferencesToStrategyResult(IEnumerable directoryDifferences, bool reverseDirection = false) + { + if (!directoryDifferences.Any()) + { + Summary = "Successfully reproduced package."; + IsSuccess = true; + return true; + //Logger.Debug("Strategy succeeded. The results match the package contents."); + } + else + { + Summary = "Strategy failed. The results do not match the package contents."; + IsSuccess = false; + + foreach (DirectoryDifference? dirDiff in directoryDifferences) + { + StrategyResultMessage? message = new StrategyResultMessage() + { + Filename = reverseDirection ? dirDiff.ComparisonFile : dirDiff.Filename, + CompareFilename = reverseDirection ? dirDiff.Filename : dirDiff.ComparisonFile, + Differences = dirDiff.Difference, + }; + switch (dirDiff.Operation) + { + case DirectoryDifferenceOperation.Added: + message.Text = "File added"; break; + case DirectoryDifferenceOperation.Modified: + message.Text = "File modified"; break; + case DirectoryDifferenceOperation.Removed: + message.Text = "File removed"; break; + default: + break; + } + Messages.Add(message); + } + return false; + } + } + } + + public abstract class BaseStrategy + { + protected StrategyOptions Options; + + public virtual StrategyPriority PRIORITY => StrategyPriority.None; + + /// + /// Logger for each of the subclasses + /// + protected static readonly NLog.Logger Logger = NLog.LogManager.GetCurrentClassLogger(); + + public abstract bool StrategyApplies(); + + public abstract StrategyResult? Execute(); + + public BaseStrategy(StrategyOptions options) + { + Options = options; + } + + /// + /// Checks the directories passed in, ensuring they aren't null, exist, and aren't empty. + /// + /// Directories to check + /// True if the satisfy the above conditions, else false. + public bool GenericStrategyApplies(IEnumerable directories) + { + if (directories == null) + { + Logger.Debug("Strategy {0} does not apply as no directories checked.", GetType().Name); + return false; + } + + bool result = true; + foreach (string? directory in directories) + { + if (directory == null) + { + Logger.Debug("Strategy {0} does not apply as no directories checked.", GetType().Name); + result = false; + } + else if (Directory.Exists(directory) && !Directory.EnumerateFileSystemEntries(directory).Any()) + { + Logger.Debug("Strategy {0} does not apply as {1} was empty.", GetType().Name, directory); + result = false; + } + } + + return result; + } + + /// + /// Locates all strategies (meaning, classes derived from BaseStrategy). + /// + /// + public static IEnumerable? GetStrategies(StrategyOptions strategyOptions) + { + List? strategies = typeof(BaseStrategy).Assembly.GetTypes().Where(t => t.IsSubclassOf(typeof(BaseStrategy))).ToList(); + + strategies.Sort((a, b) => + { + if (a == b) + { + return 0; + } + System.Reflection.ConstructorInfo? aCtor = a.GetConstructor(new Type[] { typeof(StrategyOptions) }); + BaseStrategy? aObj = aCtor?.Invoke(new object?[] { strategyOptions }) as BaseStrategy; + + System.Reflection.ConstructorInfo? bCtor = b.GetConstructor(new Type[] { typeof(StrategyOptions) }); + BaseStrategy? bObj = bCtor?.Invoke(new object?[] { strategyOptions }) as BaseStrategy; + + if (aObj == null && bObj != null) return -1; + if (aObj != null && bObj == null) return 1; + if (aObj != null && bObj != null) + { + return aObj.PRIORITY.CompareTo(bObj.PRIORITY); + } + return 0; + }); + strategies.Reverse(); // We want high priority to go first + + return strategies; + } + + protected static string? GetPathToCommand(IEnumerable commands) + { + foreach (string? command in commands) + { + string[]? pathParts = null; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + pathParts = Environment.GetEnvironmentVariable("PATH")?.Split(';'); + } + else + { + pathParts = Environment.GetEnvironmentVariable("PATH")?.Split(':'); + } + + foreach (string? pathPart in pathParts ?? Array.Empty()) + { + string? target = Path.Combine(pathPart, command); + if (File.Exists(target)) + { + return target; + } + } + } + return null; + } + + /// + /// This is a failure-resistent version of .CreateFromDirectory or .AddDirectory, both of + /// which fail under various conditions. This function continues on (emitting a message to + /// the logger) and returns false on any error. + /// + /// Directory to zip + /// File to write to. + /// true iff no errors occur + protected static bool CreateZipFromDirectory(string directoryName, string archiveName) + { + bool result = true; + // Note that we're not using something like .CreateFromDirectory, or .AddDirectory, + // since both of these had problems with permissions. Instead, we'll try to add each + // file separately, and continue on any failures. + using (ZipArchive? archive = ZipArchive.Create()) + { + using (archive.PauseEntryRebuilding()) + { + foreach (string? path in Directory.EnumerateFiles(directoryName, "*", SearchOption.AllDirectories)) + { + try + { + FileInfo? fileInfo = new FileInfo(path); + archive.AddEntry(path[directoryName.Length..], fileInfo.OpenRead(), true, fileInfo.Length, fileInfo.LastWriteTime); + } + catch (Exception ex) + { + Logger.Debug("Unable to add {0} to archive: {1}", path, ex.Message); + result = false; + } + } + archive.SaveTo(archiveName, CompressionType.Deflate); + } + } + return result; + } + + internal static string? GenerateDiffoscope(string workingDirectory, string leftDirectory, string rightDirectory) + { + Logger.Debug("Running Diffoscope on ({0}, {1})", leftDirectory, rightDirectory); + Directory.CreateDirectory(workingDirectory); + + bool runResult = OssReproducibleHelpers.RunCommand(workingDirectory, "docker", new[] { + "run", + "--rm", + "--memory=1g", + "--cpus=0.5", + "--volume", $"{Path.GetFullPath(leftDirectory)}:/work/left:ro", + "--volume", $"{Path.GetFullPath(rightDirectory)}:/work/right:ro", + "--volume", $"{Path.GetFullPath(workingDirectory)}:/work/output", + "--workdir=/work", + "registry.salsa.debian.org/reproducible-builds/diffoscope", + "--html", + "/work/output/results.html", + "/work/left", + "/work/right" + }, out string? stdout, out string? stderr); + + string? resultsFile = Path.Join(workingDirectory, "results.html"); + if (File.Exists(resultsFile)) + { + Logger.Debug("Diffoscope run successful."); + string? results = File.ReadAllText(resultsFile); + return results; + } + else + { + Logger.Debug("Diffoscope result file was empty."); + return null; + } + } + } +} \ No newline at end of file diff --git a/src/Shared.CLI/Tools/ReproducibleTool/Strategies/OryxBuildStrategy.cs b/src/Shared.CLI/Tools/ReproducibleTool/Strategies/OryxBuildStrategy.cs new file mode 100644 index 00000000..aeef7be4 --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/Strategies/OryxBuildStrategy.cs @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using NLog; +using System.IO; +using System.Linq; + +namespace Microsoft.CST.OpenSource.Reproducibility +{ + /// + /// This strategy uses the Microsoft Oryx (github.com/Microsoft/oryx) Docker image to attempt to + /// build the source repository. The priority high as this project attempts to create a runnable + /// build, meaning, it will bring in ancillary packages that aren't included in the actual + /// package itself. + /// + internal class OryxBuildStrategy : BaseStrategy + { + public override StrategyPriority PRIORITY => StrategyPriority.Low; + + public OryxBuildStrategy(StrategyOptions options) : base(options) + { + } + + /// + /// Determines whether this strategy applies to the given package/source. For this strategy, + /// we'll let Oryx do whatever it can. + /// + /// + public override bool StrategyApplies() + { + if (!GenericStrategyApplies(new[] { Options.SourceDirectory, Options.PackageDirectory })) + { + return false; + } + + if (GetPathToCommand(new[] { "docker" }) == null) + { + Logger.Debug("Strategy {0} cannot be used, as Docker does not appear to be installed.", GetType().Name); + return false; + } + return true; + } + + public override StrategyResult? Execute() + { + Logger.Debug("Executing {0} reproducibility strategy.", GetType().Name); + if (!StrategyApplies()) + { + Logger.Debug("Strategy does not apply, so cannot execute."); + return null; + } + + string? workingDirectory = OssReproducibleHelpers.GetFirstNonSingularDirectory(Options.SourceDirectory); + if (workingDirectory == null) + { + Logger.Warn("Unable to find correct source directory to run Oryx against. Unable to continue."); + return null; + } + + string? outputDirectory = Path.Join(Options.TemporaryDirectory, "build"); + Directory.CreateDirectory(outputDirectory); + string? tempBuildArchiveDirectory = Path.Join(Options.TemporaryDirectory, "archive"); + Directory.CreateDirectory(tempBuildArchiveDirectory); + + bool runResult = OssReproducibleHelpers.RunCommand(workingDirectory, "docker", new[] { + "run", + "--rm", + "--volume", $"{Path.GetFullPath(workingDirectory)}:/repo", + "--volume", $"{Path.GetFullPath(outputDirectory)}:/build-output", + "mcr.microsoft.com/oryx/build:latest", + "oryx", + "build", + "/repo", + "--output", "/build-output" + }, out string? stdout, out string? stderr); + + StrategyResult? strategyResult = new StrategyResult() + { + Strategy = GetType() + }; + + if (runResult) + { + if (Directory.GetFiles(outputDirectory, "*", SearchOption.AllDirectories).Any()) + { + if (Options.IncludeDiffoscope) + { + string? diffoscopeTempDir = Path.Join(Options.TemporaryDirectory, "diffoscope"); + string? diffoscopeResults = GenerateDiffoscope(diffoscopeTempDir, outputDirectory, Options.PackageDirectory!); + strategyResult.Diffoscope = diffoscopeResults; + } + + System.Collections.Generic.IEnumerable? diffResults = OssReproducibleHelpers.DirectoryDifference(Options.PackageDirectory!, outputDirectory, Options.DiffTechnique); + int diffResultsOriginalCount = diffResults.Count(); + diffResults = diffResults.Where(d => !IgnoreFilter.IsIgnored(Options.PackageUrl, GetType().Name, d.Filename)); + strategyResult.NumIgnoredFiles += (diffResultsOriginalCount - diffResults.Count()); + strategyResult.AddDifferencesToStrategyResult(diffResults); + } + else + { + strategyResult.IsError = true; + strategyResult.Summary = "The OryxBuildStrategy did not complete successfully (no files produced)."; + } + } + else + { + strategyResult.IsError = true; + strategyResult.Summary = "The OryxBuildStrategy did not complete successfully (container execution failed)."; + } + + return strategyResult; + } + } +} \ No newline at end of file diff --git a/src/Shared.CLI/Tools/ReproducibleTool/Strategies/PackageContainedInSourceStrategy.cs b/src/Shared.CLI/Tools/ReproducibleTool/Strategies/PackageContainedInSourceStrategy.cs new file mode 100644 index 00000000..cc4db80e --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/Strategies/PackageContainedInSourceStrategy.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using NLog; +using System.IO; +using System.Linq; + +namespace Microsoft.CST.OpenSource.Reproducibility +{ + internal class PackageContainedInSourceStrategy : BaseStrategy + { + public override StrategyPriority PRIORITY => StrategyPriority.Medium; + + public PackageContainedInSourceStrategy(StrategyOptions options) : base(options) + { + } + + public override bool StrategyApplies() + { + return GenericStrategyApplies(new[] { Options.SourceDirectory, Options.PackageDirectory }); + } + + public override StrategyResult? Execute() + { + Logger.Debug("Executing {0} reproducibility strategy.", GetType().Name); + if (!StrategyApplies()) + { + Logger.Debug("Strategy does not apply, so cannot execute."); + return null; + } + + StrategyResult? strategyResult = new StrategyResult() + { + Strategy = GetType() + }; + + if (Options.IncludeDiffoscope) + { + string? diffoscopeTempDir = Path.Join(Options.TemporaryDirectory, "diffoscope"); + string? diffoscopeResults = GenerateDiffoscope(diffoscopeTempDir, Options.SourceDirectory!, Options.PackageDirectory!); + strategyResult.Diffoscope = diffoscopeResults; + } + + System.Collections.Generic.IEnumerable? diffResults = OssReproducibleHelpers.DirectoryDifference(Options.PackageDirectory!, Options.SourceDirectory!, Options.DiffTechnique); + int diffResultsOriginalCount = diffResults.Count(); + diffResults = diffResults.Where(d => !IgnoreFilter.IsIgnored(Options.PackageUrl, GetType().Name, d.Filename)); + strategyResult.NumIgnoredFiles += (diffResultsOriginalCount - diffResults.Count()); + strategyResult.AddDifferencesToStrategyResult(diffResults); + + return strategyResult; + } + } +} \ No newline at end of file diff --git a/src/Shared.CLI/Tools/ReproducibleTool/Strategies/PackageIgnoreList.txt b/src/Shared.CLI/Tools/ReproducibleTool/Strategies/PackageIgnoreList.txt new file mode 100644 index 00000000..11ccdd4d --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/Strategies/PackageIgnoreList.txt @@ -0,0 +1,36 @@ +# +# This file contains ignore patterns for different strategies +# The format of each line is: +# A:B:C +# where: +# A => the Package URL type (e.g. npm, pypi), or '*', meaning all types +# B => the strategy name (just the class name, like 'AutoBuildProducesSamePackage') +# C => the regex that matches the files to ignore +# +# +# All files +*:*:^.*/\.git/.*$ +*:*:^.*\.txt$ +*:*:^.*\.md$ +*:*:^.*\.rst$ +*:*:^.*changelog(\.[a-z]+)?$ + +# NPM +# Oryx performs a full build, which puts a lot of content in node_modules, which isn't useful. +npm:OryxBuildStrategy:^.*/node_modules/.*$ + +# RubyGems +gem:*:^.*metadata\.gz/metadata$ +gem:*:^.*checksums\.yaml\.gz/checksums\.yaml$ + +# PyPI +pypi:*:(^|.*/)METADATA$ +pypi:*:(^|.*/)RECORD$ +pypi:*:(^|.*/)WHEEL$ +pypi:*:(^|.*/)PKG-INFO$ +pypi:*:(^|.*/)LICENSE$ + +# CPAN +cpan:*:(^|.*/)META\.json$ +cpan:*:(^|.*/)META\.yml$ +cpan:*:(^|.*/)MANIFEST$ diff --git a/src/Shared.CLI/Tools/ReproducibleTool/Strategies/PackageMatchesSourceStrategy.cs b/src/Shared.CLI/Tools/ReproducibleTool/Strategies/PackageMatchesSourceStrategy.cs new file mode 100644 index 00000000..8295dfa0 --- /dev/null +++ b/src/Shared.CLI/Tools/ReproducibleTool/Strategies/PackageMatchesSourceStrategy.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using NLog; +using System.IO; +using System.Linq; + +namespace Microsoft.CST.OpenSource.Reproducibility +{ + /// + /// This strategy checks to see if the package content exactly matches the source code repository. + /// + internal class PackageMatchesSourceStrategy : BaseStrategy + { + public override StrategyPriority PRIORITY => StrategyPriority.High; + + public PackageMatchesSourceStrategy(StrategyOptions options) : base(options) + { + } + + public override bool StrategyApplies() + { + return GenericStrategyApplies(new[] { Options.SourceDirectory, Options.PackageDirectory }); + } + + public override StrategyResult? Execute() + { + Logger.Debug("Executing {0} reproducibility strategy.", GetType().Name); + if (!StrategyApplies()) + { + Logger.Debug("Strategy does not apply, so cannot execute."); + return null; + } + + StrategyResult? strategyResult = new StrategyResult() + { + Strategy = GetType() + }; + + if (Options.IncludeDiffoscope) + { + string? diffoscopeTempDir = Path.Join(Options.TemporaryDirectory, "diffoscope"); + string? diffoscopeResults = GenerateDiffoscope(diffoscopeTempDir, Options.SourceDirectory!, Options.PackageDirectory!); + strategyResult.Diffoscope = diffoscopeResults; + } + + System.Collections.Generic.IEnumerable? diffResults = OssReproducibleHelpers.DirectoryDifference(Options.PackageDirectory!, Options.SourceDirectory!, Options.DiffTechnique); + int originalDiffResultsLength = diffResults.Count(); + diffResults = diffResults.Where(d => !IgnoreFilter.IsIgnored(Options.PackageUrl, GetType().Name, d.Filename)); + strategyResult.NumIgnoredFiles += (originalDiffResultsLength - diffResults.Count()); + strategyResult.AddDifferencesToStrategyResult(diffResults); + + diffResults = OssReproducibleHelpers.DirectoryDifference(Options.SourceDirectory!, Options.PackageDirectory!, Options.DiffTechnique); + originalDiffResultsLength = diffResults.Count(); + diffResults = diffResults.Where(d => !IgnoreFilter.IsIgnored(Options.PackageUrl, GetType().Name, d.Filename)); + strategyResult.NumIgnoredFiles += (originalDiffResultsLength - diffResults.Count()); + strategyResult.AddDifferencesToStrategyResult(diffResults, reverseDirection: true); + + return strategyResult; + } + } +} \ No newline at end of file diff --git a/src/Shared.CLI/Tools/RiskCalculatorTool.cs b/src/Shared.CLI/Tools/RiskCalculatorTool.cs new file mode 100644 index 00000000..5ae36628 --- /dev/null +++ b/src/Shared.CLI/Tools/RiskCalculatorTool.cs @@ -0,0 +1,217 @@ +// Copyright (c) Microsoft Corporation. Licensed under the MIT License. + +using NLog; +using CommandLine; +using CommandLine.Text; +using Microsoft.ApplicationInspector.Commands; +using Microsoft.CodeAnalysis.Sarif; +using Microsoft.CST.OpenSource.Shared; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using static Microsoft.CST.OpenSource.Shared.OutputBuilderFactory; +using SarifResult = Microsoft.CodeAnalysis.Sarif.Result; +using Microsoft.ApplicationInspector.RulesEngine; + +namespace Microsoft.CST.OpenSource +{ + using Microsoft.Extensions.Options; + using OssGadget.Options; + using OssGadget.Tools; + using OssGadget.Tools.HealthTool; + using PackageManagers; + using PackageUrl; + + public class RiskCalculatorTool : BaseTool + { + public RiskCalculatorTool(ProjectManagerFactory projectManagerFactory) : base(projectManagerFactory) + { + } + + public RiskCalculatorTool() : this(new ProjectManagerFactory()) + { + } + + /// + /// Calculates project risk based on health and characteristics. + /// + /// Package URL to load + /// Target directory to download content to (default: temporary location) + /// Cache the project for later processing (default: false) + /// + public async Task CalculateRisk(PackageURL purl, string? targetDirectory, bool doCaching = false, bool checkHealth = true) + { + Logger.Trace("CalculateRisk({0})", purl.ToString()); + + CharacteristicTool? characteristicTool = new CharacteristicTool(ProjectManagerFactory); + CharacteristicToolOptions? cOptions = new CharacteristicToolOptions(); + Dictionary? characteristics = characteristicTool.AnalyzePackage(cOptions, purl, targetDirectory, doCaching).Result; + double aggregateRisk = 0.0; + + if (checkHealth) + { + HealthTool? healthTool = new HealthTool(ProjectManagerFactory); + HealthMetrics? healthMetrics = await healthTool.CheckHealth(purl); + if (healthMetrics == null) + { + Logger.Warn("Unable to determine health metrics, will use a default of 0."); + healthMetrics = new HealthMetrics(purl) + { + SecurityIssueHealth = 0, + CommitHealth = 0, + ContributorHealth = 0, + IssueHealth = 0, + ProjectSizeHealth = 0, + PullRequestHealth = 0, + RecentActivityHealth = 0, + ReleaseHealth = 0 + }; + } + Logger.Trace("Health Metrics:\n{}", healthMetrics); + + // Risk calculation algorithm: Weight each of the health scores. + aggregateRisk = 1.0 - ( + 5.0 * healthMetrics.SecurityIssueHealth / 100.0 + + 1.0 * healthMetrics.CommitHealth / 100.0 + + 3.0 * healthMetrics.IssueHealth / 100.0 + + 2.0 * healthMetrics.PullRequestHealth / 100.0 + + 0.25 * healthMetrics.RecentActivityHealth / 100.0 + + 1.0 * healthMetrics.ContributorHealth / 100.0 + ) / 12.25; + Logger.Trace("Aggregate Health Risk: {}", aggregateRisk); + } + + string[]? highRiskTags = new string[] { "Cryptography.", "Authentication.", "Authorization.", "Data.Deserialization." }; + Dictionary? highRiskTagsSeen = new Dictionary(); + + foreach (AnalyzeResult? analyzeResult in characteristics.Values) + { + foreach (MatchRecord? match in analyzeResult?.Metadata?.Matches ?? new List()) + { + foreach (string? tag in match.Tags ?? Array.Empty()) + { + foreach (string? highRiskTag in highRiskTags) + { + if (tag.StartsWith(highRiskTag)) + { + if (!highRiskTagsSeen.ContainsKey(highRiskTag)) + { + highRiskTagsSeen[highRiskTag] = 0; + } + highRiskTagsSeen[highRiskTag]++; + } + } + } + } + } + if (Logger.IsTraceEnabled) + { + Logger.Trace("Found {} high-risk tags over {} categories.", highRiskTagsSeen.Values.Sum(), highRiskTagsSeen.Keys.Count()); + } + + double highRiskTagRisk = ( + 0.4 * highRiskTagsSeen.Keys.Count() + + 0.6 * Math.Min(highRiskTagsSeen.Values.Sum(), 5) + ); + highRiskTagRisk = highRiskTagRisk > 1.0 ? 1.0 : highRiskTagRisk; + + aggregateRisk = ( + 0.7 * aggregateRisk + + 0.7 * highRiskTagRisk + ); + aggregateRisk = aggregateRisk > 1.0 ? 1.0 : aggregateRisk; + + Logger.Trace("Final Risk: {}", aggregateRisk); + + return aggregateRisk; + } + + /// + /// Build and return a list of Sarif Result list from the find characterstics results + /// + /// + /// + /// + private static List GetSarifResults(PackageURL purl, double riskLevel) + { + List sarifResults = new List(); + SarifResult sarifResult = new SarifResult() + { + Kind = ResultKind.Informational, + Level = FailureLevel.None, + Locations = SarifOutputBuilder.BuildPurlLocation(purl), + Rank = riskLevel + }; + + sarifResults.Add(sarifResult); + return sarifResults; + } + + /// + /// Convert charactersticTool results into output format + /// + /// + /// + /// + private void AppendOutput(IOutputBuilder outputBuilder, PackageURL purl, double riskLevel) + { + switch (currentOutputFormat) + { + case OutputFormat.sarifv1: + case OutputFormat.sarifv2: + outputBuilder.AppendOutput(GetSarifResults(purl, riskLevel)); + break; + + case OutputFormat.text: + default: + string? riskDescription = "low"; + if (riskLevel > 0.50) riskDescription = "medium"; + if (riskLevel > 0.80) riskDescription = "high"; + if (riskLevel > 0.90) riskDescription = "very high"; + + outputBuilder.AppendOutput(new string[] { $"Risk Level: {riskLevel:N2} ({riskDescription})" }); + break; + } + } + + public async Task RunAsync(RiskCalculatorToolOptions options) + { + // select output destination and format + SelectOutput(options.OutputFile); + IOutputBuilder outputBuilder = SelectFormat(options.Format); + + // Support for --verbose + if (options.Verbose) + { + NLog.Config.LoggingRule? consoleLog = LogManager.Configuration.FindRuleByName("consoleLog"); + consoleLog.SetLoggingLevels(LogLevel.Trace, LogLevel.Fatal); + } + + if (options.Targets is IList targetList && targetList.Count > 0) + { + foreach (string? target in targetList) + { + try + { + bool useCache = options.UseCache == true; + PackageURL? purl = new PackageURL(target); + string downloadDirectory = options.DownloadDirectory == "." ? System.IO.Directory.GetCurrentDirectory() : options.DownloadDirectory; + double riskLevel = CalculateRisk(purl, + downloadDirectory, + useCache, + !options.NoHealth).Result; + AppendOutput(outputBuilder, purl, riskLevel); + } + catch (Exception ex) + { + Logger.Warn("Error processing {0}: {1}", target, ex.Message); + } + outputBuilder.PrintOutput(); + } + + RestoreOutput(); + } + } + } +} \ No newline at end of file diff --git a/src/oss-find-source/FindSourceTool.cs b/src/oss-find-source/FindSourceTool.cs index d487630a..5f38ad8d 100644 --- a/src/oss-find-source/FindSourceTool.cs +++ b/src/oss-find-source/FindSourceTool.cs @@ -79,34 +79,6 @@ public async Task> FindSourceAsync(PackageURL pur return repositoryMap; } - public class Options - { - [Usage()] - public static IEnumerable Examples - { - get - { - return new List() { - new Example("Find the source code repository for the given package", new Options { Targets = new List() {"[options]", "package-url..." } })}; - } - } - - [Option('f', "format", Required = false, Default = "text", - HelpText = "specify the output format(text|sarifv1|sarifv2)")] - public string Format { get; set; } = "text"; - - [Option('o', "output-file", Required = false, Default = "", - HelpText = "send the command output to a file instead of stdout")] - public string OutputFile { get; set; } = ""; - - [Option('S', "single", Required = false, Default = false, - HelpText = "Show only top possibility of the package source repositories. When using text format the *only* output will be the URL or empty string if error or not found.")] - public bool Single { get; set; } - - [Value(0, Required = true, - HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze - public IEnumerable? Targets { get; set; } - } /// /// Build and return a list of Sarif Result list from the find source results @@ -157,7 +129,7 @@ private static List GetTextResults(List static async Task Main(string[] args) { FindSourceTool findSourceTool = new FindSourceTool(); - await findSourceTool.ParseOptions(args).WithParsedAsync(findSourceTool.RunAsync); + await findSourceTool.ParseOptions(args).WithParsedAsync(findSourceTool.RunAsync); } /// @@ -204,11 +176,11 @@ private void AppendSingleOutput(IOutputBuilder outputBuilder, PackageURL purl, K } } - private async Task RunAsync(Options options) + private async Task RunAsync(FindSourceToolOptions findSourceToolOptions) { // Save the console logger to restore it later if we are in single mode NLog.Targets.Target? oldConfig = LogManager.Configuration.FindTargetByName("consoleLog"); - if (!options.Single) + if (!findSourceToolOptions.Single) { ShowToolBanner(); } @@ -218,9 +190,9 @@ private async Task RunAsync(Options options) LogManager.Configuration.RemoveTarget("consoleLog"); } // select output destination and format - SelectOutput(options.OutputFile); - IOutputBuilder outputBuilder = SelectFormat(options.Format); - if (options.Targets is IList targetList && targetList.Count > 0) + SelectOutput(findSourceToolOptions.OutputFile); + IOutputBuilder outputBuilder = SelectFormat(findSourceToolOptions.Format); + if (findSourceToolOptions.Targets is IList targetList && targetList.Count > 0) { foreach (string? target in targetList) { @@ -230,7 +202,7 @@ private async Task RunAsync(Options options) Dictionary dictionary = await FindSourceAsync(purl); List>? results = dictionary.ToList(); results.Sort((b, a) => a.Value.CompareTo(b.Value)); - if (options.Single) + if (findSourceToolOptions.Single) { AppendSingleOutput(outputBuilder, purl, results[0]); } @@ -248,7 +220,7 @@ private async Task RunAsync(Options options) } RestoreOutput(); // Restore console logging if we were in single mode - if (options.Single) + if (findSourceToolOptions.Single) { LogManager.Configuration.AddTarget(oldConfig); } diff --git a/src/oss-gadget-cli/Program.cs b/src/oss-gadget-cli/Program.cs index d2d4420b..a3abdb1e 100644 --- a/src/oss-gadget-cli/Program.cs +++ b/src/oss-gadget-cli/Program.cs @@ -2,8 +2,10 @@ using CommandLine; using Microsoft.CST.OpenSource; -using Microsoft.CST.OpenSource.OssGadget.CLI.Options; +using Microsoft.CST.OpenSource.OssGadget.Options; using Microsoft.CST.OpenSource.OssGadget.CLI.Tools; +using Microsoft.CST.OpenSource.OssGadget.Tools; +using Microsoft.CST.OpenSource.OssGadget.Tools.HealthTool; using System.Reflection; class OssGadgetCli : OSSGadget @@ -29,6 +31,9 @@ private async Task Run(object obj) _returnCode = obj switch { DownloadToolOptions d => await new DownloadTool(ProjectManagerFactory).RunAsync(d), + HealthToolOptions healthToolOptions => await new HealthTool(ProjectManagerFactory).RunAsync(healthToolOptions), + RiskCalculatorToolOptions riskCalculatorToolOptions => await new RiskCalculatorTool(ProjectManagerFactory).RunAsync(riskCalculatorToolOptions), + CharacteristicToolOptions characteristicToolOptions => await new CharacteristicTool(ProjectManagerFactory).RunAsync(characteristicToolOptions), _ => ErrorCode.Ok }; }