Refactor More Tools

As of this commit:

These still need a larger scope refactor due to a non standard options or calling pattern:

Defog
Detect Backdoor
Detect Crypto
Find Squats
Metadata

Also, even those implemented mostly need to return a proper error code isntead of void Task.

src/Shared.CLI/BaseTool.cs

@@ -6,14 +6,15 @@ namespace Microsoft.CST.OpenSource
     using CommandLine.Text;
     using Helpers;
     using Microsoft.CST.OpenSource.Shared;
-    using OssGadget.CLI.Options;
+    using OssGadget.Options;
     using PackageManagers;
     using System;
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
     using System.Reflection;
     using System.Text.RegularExpressions;
+    using System.Threading.Tasks;
     using static Microsoft.CST.OpenSource.Shared.OutputBuilderFactory;
 
     public class BaseTool<T> : OssGadgetLib where T: BaseToolOptions
@@ -30,6 +31,11 @@ public BaseTool(ProjectManagerFactory projectManagerFactory) : base(projectManag
         public BaseTool() : base()
         {}
 
+        public async Task RunAsync(T opt)
+        {
+            throw new NotImplementedException();
+        }
+
         /// <summary>
         /// Formulates the help text for each derived tool
         /// </summary>

src/Shared.CLI/Options/BaseToolOptions.cs

@@ -1,41 +1,11 @@
 // Copyright (c) Microsoft Corporation. Licensed under the MIT License.
 
-namespace Microsoft.CST.OpenSource.OssGadget.CLI.Options;
+namespace Microsoft.CST.OpenSource.OssGadget.Options;
 
 using CommandLine;
 using CommandLine.Text;
 using System.Collections.Generic;
 
 public class BaseToolOptions
 {
-    [Usage()]
-    public static IEnumerable<Example> Examples
-    {
-        get
-        {
-            return new List<Example>() {
-                new Example("Download the given package",
-                    new DownloadToolOptions { Targets = new List<string>() {"[options]", "package-url..." } })};
-        }
-    }
-
-    [Option('x', "download-directory", Required = false, Default = ".",
-        HelpText = "the directory to download the package to.")]
-    public string DownloadDirectory { get; set; } = ".";
-
-    [Option('m', "download-metadata-only", Required = false, Default = false,
-        HelpText = "download only the package metadata, not the package.")]
-    public bool DownloadMetadataOnly { get; set; }
-
-    [Option('e', "extract", Required = false, Default = false,
-        HelpText = "Extract the package contents")]
-    public bool Extract { get; set; }
-
-    [Value(0, Required = true,
-        HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze
-    public IEnumerable<string>? Targets { get; set; }
-
-    [Option('c', "use-cache", Required = false, Default = false,
-        HelpText = "do not download the package if it is already present in the destination directory.")]
-    public bool UseCache { get; set; }
 }

src/Shared.CLI/Options/CharacteristicToolOptions.cs

@@ -0,0 +1,67 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+namespace Microsoft.CST.OpenSource.OssGadget.Options;
+
+using CodeAnalysis.Sarif;
+using CommandLine;
+using CommandLine.Text;
+using System.Collections.Generic;
+
+[Verb("characteristic", HelpText = "Run risk calculator tool")]
+public class CharacteristicToolOptions : BaseToolOptions
+{
+        [Usage()]
+        public static IEnumerable<Example> Examples
+        {
+            get
+            {
+                return new List<Example>() {
+                    new Example("Find the characterstics for the given package",
+                    new CharacteristicToolOptions() { Targets = new List<string>() {"[options]", "package-url..." } })};
+            }
+        }
+
+        [Option('r', "custom-rule-directory", Required = false, Default = null,
+            HelpText = "load rules from the specified directory.")]
+        public string? CustomRuleDirectory { get; set; }
+
+        [Option("disable-default-rules", Required = false, Default = false,
+            HelpText = "do not load default, built-in rules.")]
+        public bool DisableDefaultRules { get; set; }
+
+        [Option('d', "download-directory", Required = false, Default = ".",
+            HelpText = "the directory to download the package to.")]
+        public string DownloadDirectory { get; set; } = ".";
+
+        [Option('f', "format", Required = false, Default = "text",
+            HelpText = "specify the output format(text|sarifv1|sarifv2)")]
+        public string Format { get; set; } = "text";
+
+        [Option('o', "output-file", Required = false, Default = "",
+            HelpText = "send the command output to a file instead of stdout")]
+        public string OutputFile { get; set; } = "";
+
+        [Value(0, Required = true,
+            HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze
+        public IEnumerable<string>? Targets { get; set; }
+
+        [Option('c', "use-cache", Required = false, Default = false,
+            HelpText = "do not download the package if it is already present in the destination directory.")]
+        public bool UseCache { get; set; }
+
+        [Option('x', "exclude", Required = false,
+            HelpText = "exclude files or paths which match provided glob patterns.")]
+        public string FilePathExclusions { get; set; } = "";
+
+        [Option('b', "backtracking", Required = false, HelpText = "Use backtracking regex engine by default.")]
+        public bool EnableBacktracking { get; set; } = false;
+
+        [Option('s', "single-threaded", Required = false, HelpText = "Use single-threaded analysis")]
+        public bool SingleThread { get; set; } = false;
+
+        public bool AllowTagsInBuildFiles { get; set; } = true;
+
+        public bool AllowDupTags { get; set; } = false;
+
+        public FailureLevel SarifLevel { get; set; } = FailureLevel.Note;
+}

src/Shared.CLI/Options/DetectBackdoorToolOptions.cs

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+namespace Microsoft.CST.OpenSource.OssGadget.Options;
+
+using CommandLine;
+using CommandLine.Text;
+using System.Collections.Generic;
+
+[Verb("detect-backdoor", HelpText = "Run detect backdoor tool")]
+public class DetectBackdoorToolOptions
+{
+    [Usage()]
+    public static IEnumerable<Example> Examples
+    {
+        get
+        {
+            return new List<Example>() {
+                new Example("Identify potential malware or backdoors in the given package",
+                    new DetectBackdoorToolOptions { Targets = new List<string>() {"[options]", "package-url..." } })};
+        }
+    }
+
+    [Option('d', "download-directory", Required = false, Default = ".",
+        HelpText = "the directory to download the package to.")]
+    public string DownloadDirectory { get; set; } = ".";
+
+    [Option('f', "format", Required = false, Default = "text",
+        HelpText = "specify the output format(text|sarifv1|sarifv2)")]
+    public string Format { get; set; } = "text";
+
+    [Option('o', "output-file", Required = false, Default = "",
+        HelpText = "send the command output to a file instead of stdout")]
+    public string OutputFile { get; set; } = "";
+
+    [Value(0, Required = true,
+        HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze
+    public IEnumerable<string>? Targets { get; set; }
+
+    [Option('c', "use-cache", Required = false, Default = false,
+        HelpText = "do not download the package if it is already present in the destination directory.")]
+    public bool UseCache { get; set; }
+
+    [Option('b', "backtracking", Required = false, HelpText = "Use backtracking engine by default.")]
+    public bool EnableBacktracking { get; set; } = false;
+
+    [Option('s', "single-threaded", Required = false, HelpText = "Use single-threaded analysis")]
+    public bool SingleThread { get; set; } = false;
+}

src/Shared.CLI/Options/DiffToolOptions.cs

@@ -0,0 +1,67 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+namespace Microsoft.CST.OpenSource.OssGadget.Options;
+
+using CommandLine;
+using CommandLine.Text;
+using System;
+using System.Collections.Generic;
+
+[Verb("diff", HelpText = "Run diff tool")]
+public class DiffToolOptions : BaseToolOptions
+{
+    [Usage()]
+    public static IEnumerable<Example> Examples
+    {
+        get
+        {
+            return new List<Example>() {
+                new Example("Diff the given packages",
+                new DiffToolOptions { Targets = new List<string>() {"[options]", "package-url package-url-2" } })};
+        }
+    }
+
+    [Option('d', "download-directory", Required = false, Default = null,
+                    HelpText = "the directory to download the packages to.")]
+    public string? DownloadDirectory { get; set; } = null;
+
+    [Option('c', "use-cache", Required = false, Default = false,
+        HelpText = "Do not download the package if it is already present in the destination directory and do not delete the package after processing.")]
+    public bool UseCache { get; set; }
+
+    [Option('w', "crawl-archives", Required = false, Default = true,
+        HelpText = "Crawl into archives found in packages.")]
+    public bool CrawlArchives { get; set; }
+
+    [Option('B', "context-before", Required = false, Default = 0,
+        HelpText = "Number of previous lines to give as context.")]
+    public int Before { get; set; } = 0;
+
+    [Option('A', "context-after", Required = false, Default = 0,
+        HelpText = "Number of subsequent lines to give as context.")]
+    public int After { get; set; } = 0;
+
+    [Option('C', "context", Required = false, Default = 0,
+        HelpText = "Number of lines to give as context. Overwrites Before and After options. -1 to print all.")]
+    public int Context { get; set; } = 0;
+
+    [Option('a', "added-only", Required = false, Default = false,
+        HelpText = "Only show added lines (and requested context).")]
+    public bool AddedOnly { get; set; } = false;
+
+    [Option('r', "removed-only", Required = false, Default = false,
+        HelpText = "Only show removed lines (and requested context).")]
+    public bool RemovedOnly { get; set; } = false;
+
+    [Option('f', "format", Required = false, Default = "text",
+        HelpText = "Choose output format. (text|sarifv1|sarifv2)")]
+    public string Format { get; set; } = "text";
+
+    [Option('o', "output-location", Required = false, Default = null,
+        HelpText = "Output location. Don't specify for console output.")]
+    public string? OutputLocation { get; set; } = null;
+
+    [Value(0, Required = true,
+        HelpText = "Exactly two Filenames or PackgeURL specifiers to analyze.", Hidden = true)] // capture all targets to analyze
+    public IEnumerable<string> Targets { get; set; } = Array.Empty<string>();
+}

src/Shared.CLI/Options/DownloadToolOptions.cs

@@ -1,6 +1,5 @@
 // Copyright (c) Microsoft Corporation. Licensed under the MIT License.
-
-namespace Microsoft.CST.OpenSource.OssGadget.CLI.Options;
+namespace Microsoft.CST.OpenSource.OssGadget.Options;
 
 using CommandLine;
 using CommandLine.Text;

src/Shared.CLI/Options/FindDomainSquatsToolOptions.cs

@@ -0,0 +1,47 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+namespace Microsoft.CST.OpenSource.OssGadget.Options;
+
+using CommandLine;
+using CommandLine.Text;
+using System.Collections.Generic;
+
+[Verb("find-domain-squats", HelpText = "Run find-domain-squats tool")]
+public class FindDomainSquatsToolOptions : BaseToolOptions
+{
+    [Usage()]
+    public static IEnumerable<Example> Examples
+    {
+        get
+        {
+            return new List<Example>() {
+                new Example("Find Squat Candidates for the Given Packages",
+                    new FindDomainSquatsToolOptions { Targets = new List<string>() {"[options]", "domains" } })};
+        }
+    }
+
+    [Option('o', "output-file", Required = false, Default = "",
+        HelpText = "send the command output to a file instead of stdout")]
+    public string OutputFile { get; set; } = "";
+
+    [Option('f', "format", Required = false, Default = "text",
+        HelpText = "specify the output format(text|sarifv1|sarifv2)")]
+    public string Format { get; set; } = "text";
+
+    [Option('q', "quiet", Required = false, Default = false,
+        HelpText = "Suppress console output.")]
+    public bool Quiet { get; set; } = false;
+
+    [Option('s', "sleep-delay", Required = false, Default = 0, HelpText = "Number of ms to sleep between checks.")]
+    public int SleepDelay { get; set; }
+
+    [Option('u', "unregistered", Required = false, Default = false, HelpText = "Don't show registered domains.")]
+    public bool Unregistered { get; set; }
+
+    [Option('r', "registered", Required = false, Default = false, HelpText = "Don't show unregistered domains.")]
+    public bool Registered { get; set; }
+
+    [Value(0, Required = true,
+        HelpText = "Domain(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze
+    public IEnumerable<string>? Targets { get; set; }
+}

src/Shared.CLI/Options/FindSourceToolOptions.cs

@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+namespace Microsoft.CST.OpenSource.OssGadget.Options;
+
+using CommandLine;
+using CommandLine.Text;
+using System.Collections.Generic;
+
+[Verb("find-source", HelpText = "Run find-domain-squats tool")]
+public class FindSourceToolOptions : BaseToolOptions
+{
+    [Usage()]
+    public static IEnumerable<Example> Examples
+    {
+        get
+        {
+            return new List<Example>() {
+                new Example("Find the source code repository for the given package", new FindSourceToolOptions { Targets = new List<string>() {"[options]", "package-url..." } })};
+        }
+    }
+
+    [Option('f', "format", Required = false, Default = "text",
+        HelpText = "specify the output format(text|sarifv1|sarifv2)")]
+    public string Format { get; set; } = "text";
+
+    [Option('o', "output-file", Required = false, Default = "",
+        HelpText = "send the command output to a file instead of stdout")]
+    public string OutputFile { get; set; } = "";
+
+    [Option('S', "single", Required = false, Default = false,
+        HelpText = "Show only top possibility of the package source repositories. When using text format the *only* output will be the URL or empty string if error or not found.")]
+    public bool Single { get; set; }
+
+    [Value(0, Required = true,
+        HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze
+    public IEnumerable<string>? Targets { get; set; }
+}

src/Shared.CLI/Options/FindSquatsToolOptions.cs

@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+namespace Microsoft.CST.OpenSource.OssGadget.Options;
+
+using CommandLine;
+using CommandLine.Text;
+using System.Collections.Generic;
+
+[Verb("find-squats", HelpText = "Run find-squats tool")]
+public class FindSquatsToolOptions : BaseToolOptions
+{
+    [Usage()]
+    public static IEnumerable<Example> Examples
+    {
+        get
+        {
+            return new List<Example>() {
+                new Example("Find Squat Candidates for the Given Packages",
+                    new FindSquatsToolOptions { Targets = new List<string>() {"[options]", "package-urls..." } })};
+        }
+    }
+
+    [Option('o', "output-file", Required = false, Default = "",
+        HelpText = "send the command output to a file instead of stdout")]
+    public string OutputFile { get; set; } = "";
+
+    [Option('f', "format", Required = false, Default = "text",
+        HelpText = "specify the output format(text|sarifv1|sarifv2)")]
+    public string Format { get; set; } = "text";
+
+    [Option('q', "quiet", Required = false, Default = false,
+        HelpText = "Suppress console output.")]
+    public bool Quiet { get; set; } = false;
+
+    [Option('s', "sleep-delay", Required = false, Default = 0, HelpText = "Number of ms to sleep between checks.")]
+    public int SleepDelay { get; set; }
+
+    [Value(0, Required = true,
+        HelpText = "PackgeURL(s) specifier to analyze (required, repeats OK)", Hidden = true)] // capture all targets to analyze
+    public IEnumerable<string>? Targets { get; set; }
+
+}