Removed messages until we get a proper verbose mode

app/scan.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
@@ -27,6 +28,7 @@ func Scan(cmd *cobra.Command, args []string) {
 	configFile, _ := cmd.Flags().GetString("config-file")
 	suffix, _ := cmd.Flags().GetString("suffix")
 	prefix, _ := cmd.Flags().GetString("prefix")
+	httpRequestTimeout, _ := cmd.Flags().GetInt("timeout")
 	blockedFlag, _ := cmd.Flags().GetString("block")
 
 	var tmpURL string
@@ -78,54 +80,59 @@ func Scan(cmd *cobra.Command, args []string) {
 	date := currentTime.Format("2006-01-02_15-04-05")
 	out := []data.Output{}
 
+	var wg sync.WaitGroup
+	wg.Add(len(urlList))
+
 	for i := 0; i < len(urlList); i++ {
-		fmt.Print("Testing domain : ")
-		fmt.Println(prefix + urlList[i] + suffix)
-		for index, plugin := range y.Plugins {
-			_ = index
-			tmpURL = prefix + urlList[i] + suffix + fmt.Sprint(plugin.URI)
-			if plugin.QueryString != "" {
-				tmpURL += "?" + plugin.QueryString
-			}
+		go func(domain string) {
+			defer wg.Done()
+			fmt.Print("Testing domain : ")
+			fmt.Println(prefix + domain + suffix)
+			for index, plugin := range y.Plugins {
+				_ = index
+				tmpURL = prefix + domain + suffix + fmt.Sprint(plugin.URI)
+				if plugin.QueryString != "" {
+					tmpURL += "?" + plugin.QueryString
+				}
 
-			// By default we follow HTTP redirects
-			followRedirects := true
-			// But for each plugin we can override and don't follow HTTP redirects
-			if plugin.FollowRedirects != nil && *plugin.FollowRedirects == false {
-				followRedirects = false
-			}
+				// By default we follow HTTP redirects
+				followRedirects := true
+				// But for each plugin we can override and don't follow HTTP redirects
+				if plugin.FollowRedirects != nil && *plugin.FollowRedirects == false {
+					followRedirects = false
+				}
 
-			httpResponse, err := pkg.HTTPGet(insecure, tmpURL, followRedirects)
-			if err != nil {
-				_ = errors.Wrap(err, "Timeout of HTTP Request")
-			}
+				httpResponse, err := pkg.HTTPGet(insecure, tmpURL, followRedirects, httpRequestTimeout)
+				if err != nil {
+					_ = errors.Wrap(err, "Timeout of HTTP Request")
+				}
 
-			if httpResponse != nil {
-				for index, check := range plugin.Checks {
-					_ = index
-					answer := pkg.ResponseAnalysis(httpResponse, check)
-					if answer {
-						hit = true
-						if BlockCI(blockedFlag, *check.Severity) {
-							block = true
+				if httpResponse != nil {
+					for index, check := range plugin.Checks {
+						_ = index
+						answer := pkg.ResponseAnalysis(httpResponse, check)
+						if answer {
+							hit = true
+							if BlockCI(blockedFlag, *check.Severity) {
+								block = true
+							}
+							out = append(out, data.Output{
+								Domain:      domain,
+								PluginName:  check.PluginName,
+								TestedURL:   plugin.URI,
+								Severity:    string(*check.Severity),
+								Remediation: *check.Remediation,
+							})
 						}
-						out = append(out, data.Output{
-							Domain:      urlList[i],
-							PluginName:  check.PluginName,
-							TestedURL:   plugin.URI,
-							Severity:    string(*check.Severity),
-							Remediation: *check.Remediation,
-						})
 					}
 				}
-			} else {
-				fmt.Println("Server refused the connection for URL : " + tmpURL)
-				continue
+				_ = httpResponse.Body.Close()
 			}
-			_ = httpResponse.Body.Close()
-		}
+		}(urlList[i])
 	}
 
+	wg.Wait()
+
 	if hit {
 		pkg.FormatOutputTable(out)
 		if json {

cmd/scan.go

@@ -18,6 +18,7 @@ func init() {
 	scanCmd.Flags().StringP("url-file", "f", "", "path to a specified file containing urls to test")        // --uri-file ou -f
 	scanCmd.Flags().StringP("suffix", "s", "", "Add suffix to urls when flag url-file is specified")        // --suffix ou -s
 	scanCmd.Flags().StringP("prefix", "p", "", "Add prefix to urls when flag url-file is specified")        // --prefix ou -p
+	scanCmd.Flags().Int32P("timeout", "t", 10, "Timeout for the HTTP requests (default: 10s)")              // --timeout ou -t
 	scanCmd.Flags().StringP("block", "b", "", "Block pipeline if severity is over or equal specified flag") // --block ou -b
 	scanCmd.Flags().BoolP("csv", "", false, "output as a csv file")                                         //--csv
 	scanCmd.Flags().BoolP("json", "", false, "output as a json file")                                       //--json

data/format_output.go

@@ -40,13 +40,16 @@ type Check struct {
 
 // Config struct to load the configuration from the YAML file
 type Config struct {
-	Insecure bool `yaml:"insecure"`
-	Plugins  []struct {
-		URI             string  `yaml:"uri"`
-		QueryString     string  `yaml:"query_string"`
-		Checks          []Check `yaml:"checks"`
-		FollowRedirects *bool   `yaml:"follow_redirects"`
-	} `yaml:"plugins"`
+	Insecure bool        `yaml:"insecure"`
+	Plugins  []Signature `yaml:"plugins"`
+}
+
+// Signature struct to load it afterwards
+type Signature struct {
+	URI             string  `yaml:"uri"`
+	QueryString     string  `yaml:"query_string"`
+	Checks          []Check `yaml:"checks"`
+	FollowRedirects *bool   `yaml:"follow_redirects"`
 }
 
 // IsValid will verify that the severityType is part of the enum previously declared

pkg/http.go

@@ -2,20 +2,20 @@ package pkg
 
 import (
 	"crypto/tls"
-	"log"
 	"net/http"
 	"time"
 )
 
 //HTTPGet return http response of http get request
-func HTTPGet(insecure bool, url string, followRedirects bool) (*http.Response, error) {
+func HTTPGet(insecure bool, url string, followRedirects bool, timeout int) (*http.Response, error) {
+
 	tr := &http.Transport{}
 	if insecure {
 		tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
 	}
 	var netClient = &http.Client{
 		Transport: tr,
-		Timeout:   time.Second * 3,
+		Timeout:   time.Second * time.Duration(timeout),
 	}
 
 	// If we don't want to follow HTTP redirects
@@ -27,10 +27,10 @@ func HTTPGet(insecure bool, url string, followRedirects bool) (*http.Response, e
 	}
 
 	resp, err := netClient.Get(url)
-	if err != nil {
-		log.Println(err)
-		log.Println("If error unsupported protocol scheme encountered, try adding flag --prefix with http://, or add prefix directly in url list")
-	}
+
+	// if err != nil {
+	// 	log.Println(err)
+	// }
 
 	return resp, err
 }