Merge pull request #41 from edoardottt/devel

Devel - v1.1.2

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/mattn/go-isatty v0.0.13 // indirect
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
-	github.com/spyse-com/go-spyse v1.2.1
+	github.com/spyse-com/go-spyse v1.2.4
 	github.com/temoto/robotstxt v1.1.2 // indirect
 	golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect
 	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect

go.sum

@@ -34,7 +34,6 @@ github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8Nz
 github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
 github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA=
 github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
@@ -46,6 +45,10 @@ github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxT
 github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU=
 github.com/spyse-com/go-spyse v1.2.1 h1:Za/BnLnXWY/DqZZQm2V7NQ69aJ8FgFA8vBiipf3CHC8=
 github.com/spyse-com/go-spyse v1.2.1/go.mod h1:YzL0kTQIlCVTtP0Bna4I7p/sKF2rgY1cV32dq/L4oIw=
+github.com/spyse-com/go-spyse v1.2.3 h1:0qo0OP5kLv0equyvI7H5pAGuDFTiFON3zXLv4BSw1yY=
+github.com/spyse-com/go-spyse v1.2.3/go.mod h1:MTle/KKITU7B2oSlfpzHZPc2k+WH+n5YATh1eUje7po=
+github.com/spyse-com/go-spyse v1.2.4 h1:QEI+CHM2w4kCIkTihk0TQSIUupWueRhyBWf6L6LcYQY=
+github.com/spyse-com/go-spyse v1.2.4/go.mod h1:MTle/KKITU7B2oSlfpzHZPc2k+WH+n5YATh1eUje7po=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -76,6 +79,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

input/flags.go

@@ -247,7 +247,7 @@ func ReadArgs() Input {
 		examplesCommand.Parse(os.Args[2:])
 	default:
 		output.Intro()
-		flag.PrintDefaults()
+		output.Help()
 		os.Exit(1)
 	}
 

main.go

@@ -50,31 +50,41 @@ func main() {
 func execute(userInput input.Input, subs map[string]output.Asset, dirs map[string]output.Asset, common []int) {
 
 	var mutex = &sync.Mutex{}
+	var commandProvided = false
 
 	// :::::::: REPORT SUBCOMMAND HANDLER ::::::::
 	if userInput.ReportTarget != "" {
+		commandProvided = true
 		ReportSubcommandHandler(userInput, mutex, dirs, subs)
 	}
 
 	// :::::::: DNS SUBCOMMAND HANDLER ::::::::
 	if userInput.DNSTarget != "" {
+		commandProvided = true
 		DNSSubcommandHandler(userInput)
 	}
 
 	// :::::::: SUBDOMAIN SUBCOMMAND HANDLER ::::::::
 	if userInput.SubdomainTarget != "" {
+		commandProvided = true
 		SubdomainSubcommandHandler(userInput, mutex, dirs, subs)
 	}
 
 	// :::::::: DIRECTORIES SUBCOMMAND HANDLER ::::::::
 	if userInput.DirTarget != "" {
+		commandProvided = true
 		DirSubcommandHandler(userInput, mutex, dirs, subs)
 	}
 
 	// :::::::: PORT SUBCOMMAND HANDLER ::::::::
 	if userInput.PortTarget != "" {
+		commandProvided = true
 		PortSubcommandHandler(userInput, common)
 	}
+
+	if !commandProvided {
+		output.Help()
+	}
 }
 
 //ReportSubcommandHandler >

opendb/spyse.go

@@ -32,6 +32,9 @@ import (
 	spyse "github.com/spyse-com/go-spyse/pkg"
 )
 
+const searchMethodResultsLimit = 10000
+const defaultScrollResultsLimit = 20000
+
 //SpyseSubdomains appends to the subdomains in the list
 //thr subdomains found with the Spyse service.
 func SpyseSubdomains(target string, accessToken string) []string {
@@ -45,34 +48,77 @@ func SpyseSubdomains(target string, accessToken string) []string {
 
 	svc := spyse.NewDomainService(client)
 
-	//Dot before the domain name is important because search fetch any domains that end with ".$target"
+	//Dot before the domain name is important because search fetch any domains ending with ".$target"
 	var searchDomain = "." + target
 	var subdomainsSearchParams spyse.QueryBuilder
+	var ctx = context.Background()
 
 	subdomainsSearchParams.AppendParam(spyse.QueryParam{
 		Name:     svc.Params().Name.Name,
 		Operator: svc.Params().Name.Operator.EndsWith,
 		Value:    searchDomain,
 	})
 
-	countResults, err := svc.SearchCount(context.Background(), subdomainsSearchParams.Query)
+	totalResults, err := svc.SearchCount(ctx, subdomainsSearchParams.Query)
 	if err != nil {
 		log.Fatal(err.Error())
 	}
 
-	var limit = 100
-	var offset = 0
-	var searchResults []spyse.Domain
-	var domain spyse.Domain
-	for ; int64(offset) < countResults; offset += limit {
-		//Notice that you can fetch only the first 10000 (can depend on your subscription plan) results using the Search method
-		searchResults, err = svc.Search(context.Background(), subdomainsSearchParams.Query, limit, offset)
-		if err != nil {
-			log.Fatal(err.Error())
+	if totalResults == 0 {
+		return result
+	}
+
+	// The default "Search" method returns only first 10 000 subdomains.
+	// To obtain more than 10 000 subdomains the "Scroll" method should be used.
+	// Note: The "Scroll" method is only available for "PRO" customers, so we need to check
+	// quota.IsScrollSearchEnabled parameter.
+	if totalResults > searchMethodResultsLimit && client.Account().IsScrollSearchEnabled {
+		var scrollID string
+		var scrollResults *spyse.DomainScrollResponse
+
+		for {
+			if scrollResults, err = svc.ScrollSearch(ctx, subdomainsSearchParams.Query, scrollID); err != nil {
+				if len(result) > 0 {
+					spyseErr, ok := err.(*spyse.ErrResponse)
+					if ok && spyseErr.Err.Code == spyse.CodeRequestsLimitReached {
+						break
+					}
+				}
+				log.Fatal(err.Error())
+			}
+			if len(scrollResults.Items) > 0 {
+				scrollID = scrollResults.SearchID
+
+				for _, domain := range scrollResults.Items {
+					result = append(result, domain.Name)
+				}
+				// The default "Scroll" limit, to avoid results that more than 20k
+				// If a limit of the number of requested subdomains will be added this block should be changed
+				if len(result) > defaultScrollResultsLimit {
+					break
+				}
+			}
 		}
-		for _, domain = range searchResults {
-			result = append(result, domain.Name)
+	} else {
+		var limit = 100
+		var searchResults []spyse.Domain
+
+		for offset := 0; int64(offset) < totalResults && int64(offset) < searchMethodResultsLimit; offset += limit {
+			if searchResults, err = svc.Search(ctx, subdomainsSearchParams.Query, limit, offset); err != nil {
+				if len(result) > 0 {
+					spyseErr, ok := err.(*spyse.ErrResponse)
+					if ok && spyseErr.Err.Code == spyse.CodeRequestsLimitReached {
+						break
+					}
+				}
+				log.Fatal(err.Error())
+			}
+
+			for _, domain := range searchResults {
+				result = append(result, domain.Name)
+			}
 		}
 	}
+
 	return result
 }

output/intro.go

@@ -37,7 +37,7 @@ func Intro() {
 	banner2 := "         ___  ___(_) | | __ _\n"
 	banner3 := "        / __|/ __| | | |/ _` |\n"
 	banner4 := "        \\__ \\ (__| | | | (_| |\n"
-	banner5 := "        |___/\\___|_|_|_|\\__,_| v1.1.1\n"
+	banner5 := "        |___/\\___|_|_|_|\\__,_| v1.1.2\n"
 	banner6 := " > github.com/edoardottt/scilla\n"
 	banner7 := " > edoardoottavianelli.it"
 	bannerPart1 := banner1 + banner2 + banner3 + banner4 + banner5

utils/url.go

@@ -78,7 +78,8 @@ func AppendDir(scheme string, domain string, dir string) (string, string) {
 	return scheme + "://" + domain + "/" + dir + "/", scheme + "://" + domain + "/" + dir
 }
 
-//CleanSubdomainsOk >
+//CleanSubdomainsOk takes as input a slice of subdomains and remove
+//from the input slice all the 'wrong' subdomains.
 func CleanSubdomainsOk(target string, input []string) []string {
 	var result []string
 	for _, elem := range input {
@@ -87,7 +88,7 @@ func CleanSubdomainsOk(target string, input []string) []string {
 				splits := strings.Split(elem, "\n")
 				elem = splits[1]
 			}
-			result = append(result, elem)
+			result = append(result, CleanProtocol(elem))
 		}
 	}
 	return result