Devel update

README.md

@@ -270,7 +270,7 @@ If there aren't errors, go ahead :)
 
 **Help me building this!**
 
-Special thanks to: [danielmiessler](https://github.com/danielmiessler), [sonarSearch](https://github.com/cgboal/sonarsearch), [HackerTarget](https://hackertarget.com/), [BufferOverrun](http://dns.bufferover.run/), [Threatcrowd](https://www.threatcrowd.org/), [Crt.sh](https://crt.sh/), [VirusTotal](https://www.virustotal.com/), [tomnomnom](https://github.com/tomnomnom/assetfinder).
+Special thanks to: [danielmiessler](https://github.com/danielmiessler), [HackerTarget](https://hackertarget.com/), [Threatcrowd](https://www.threatcrowd.org/), [Crt.sh](https://crt.sh/), [VirusTotal](https://www.virustotal.com/), [tomnomnom](https://github.com/tomnomnom/assetfinder).
 
 **To do:**
 

go.mod

@@ -3,15 +3,15 @@ module github.com/edoardottt/scilla
 go 1.18
 
 require (
+	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89
-	github.com/fatih/color v1.15.0
+	github.com/fatih/color v1.16.0
 	github.com/gocolly/colly v1.2.0
 	github.com/stretchr/testify v1.8.4
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	github.com/PuerkitoBio/goquery v1.8.1 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/antchfx/htmlquery v1.3.0 // indirect
 	github.com/antchfx/xmlquery v1.3.17 // indirect
@@ -22,12 +22,12 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/temoto/robotstxt v1.1.2 // indirect
 	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect

go.sum

@@ -15,8 +15,8 @@ github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89/go.mod h1:/0
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI=
@@ -34,8 +34,8 @@ github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
-github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA=
@@ -76,8 +76,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=

pkg/opendb/dnsreponoc.go

@@ -0,0 +1,90 @@
+/*
+
+=======================
+Scilla - Information Gathering Tool
+=======================
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see http://www.gnu.org/licenses/.
+
+	@Repository:  https://github.com/edoardottt/scilla
+
+	@Author:      edoardottt, https://www.edoardoottavianelli.it
+
+	@License: https://github.com/edoardottt/scilla/blob/main/LICENSE
+
+*/
+
+package opendb
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/PuerkitoBio/goquery"
+
+	httpUtils "github.com/edoardottt/scilla/internal/http"
+)
+
+func scrape(body io.ReadCloser) []string {
+	var result = make([]string, 0)
+
+	tableIndex := 1  // Looks for the second table from DNSRepoNoc html page
+	columnIndex := 0 // Looks for the first index that contains list of subdomains
+
+	doc, err := goquery.NewDocumentFromReader(body)
+	if err != nil {
+		fmt.Println(err)
+	}
+
+	var row string
+
+	doc.Find(".table-responsive").Each(func(index int, tablehtml *goquery.Selection) {
+		if index == tableIndex {
+			tablehtml.Find("tr").Each(func(indextr int, rowhtml *goquery.Selection) {
+				rowhtml.Find("td").Each(func(indexth int, tablecell *goquery.Selection) {
+					if indexth == columnIndex {
+						row = tablecell.Text()
+					}
+				})
+				result = append(result, row)
+			})
+		}
+	})
+
+	return result
+}
+
+// DNSRepoNoc retrieves from the url below some known subdomains - without API Key.
+func DNSRepoNocSubdomains(domain string, plain bool) []string {
+	if !plain {
+		fmt.Println("Pulling data from Dns Repo Noc")
+	}
+
+	client := http.Client{
+		Timeout: httpUtils.Seconds30,
+	}
+
+	url := "https://dnsrepo.noc.org/?domain=" + domain
+	resp, err := client.Get(url)
+
+	if err != nil {
+		return []string{}
+	}
+	defer resp.Body.Close()
+
+	output := scrape(resp.Body)
+
+	return output
+}

pkg/opendb/subdomaincenter.go

@@ -0,0 +1,75 @@
+/*
+
+=======================
+Scilla - Information Gathering Tool
+=======================
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see http://www.gnu.org/licenses/.
+
+	@Repository:  https://github.com/edoardottt/scilla
+
+	@Author:      edoardottt, https://www.edoardoottavianelli.it
+
+	@License: https://github.com/edoardottt/scilla/blob/main/LICENSE
+
+*/
+
+package opendb
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	httpUtils "github.com/edoardottt/scilla/internal/http"
+)
+
+// SubdomainCenter retrieves from the url below some known subdomains.
+func SubdomainCenterSubdomains(domain string, plain bool) []string {
+	if !plain {
+		fmt.Println("Pulling data from Subdomain Center")
+	}
+
+	client := http.Client{
+		Timeout: httpUtils.Seconds30,
+	}
+
+	result := make([]string, 0)
+	url := "http://api.subdomain.center/?domain=" + domain
+
+	resp, err := client.Get(url)
+	if err != nil {
+		return result
+	}
+	defer resp.Body.Close()
+
+	// read the response body
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return result
+	}
+
+	// Decode the response body as list of string
+	var response []string
+	err = json.Unmarshal(body, &response)
+
+	if err != nil {
+		return result
+	}
+
+	result = append(result, response...)
+
+	return result
+}

pkg/runner/runner.go

@@ -179,6 +179,8 @@ func ReportSubcommandHandler(userInput input.Input, mutex *sync.Mutex,
 		subdomains = opendb.AppendDBSubdomains(anubis, subdomains)
 		threatminer := opendb.ThreatMinerSubdomains(urlUtils.CleanProtocol(target), false)
 		subdomains = opendb.AppendDBSubdomains(threatminer, subdomains)
+		dnsreponoc := opendb.DNSRepoNocSubdomains(urlUtils.CleanProtocol(target), false)
+		subdomains = opendb.AppendDBSubdomains(dnsreponoc, subdomains)
 
 		// Service Not Working
 		// bufferOverrun := opendb.BufferOverrunSubdomains(urlUtils.CleanProtocol(target), false)
@@ -188,6 +190,10 @@ func ReportSubcommandHandler(userInput input.Input, mutex *sync.Mutex,
 		// sonar := opendb.SonarSubdomains(urlUtils.CleanProtocol(target), false)
 		// subdomains = opendb.AppendDBSubdomains(sonar, subdomains)
 
+		// Service not working
+		// subdomaincenter := opendb.SubdomainCenterSubdomains(urlUtils.CleanProtocol(target), false)
+		// subdomains = opendb.AppendDBSubdomains(subdomaincenter, subdomains)
+
 		if userInput.ReportVirusTotal {
 			vtSubs := opendb.VirusTotalSubdomains(urlUtils.CleanProtocol(target), input.GetKey("virustotal"), false)
 			subdomains = opendb.AppendDBSubdomains(vtSubs, subdomains)
@@ -356,6 +362,8 @@ func SubdomainSubcommandHandler(userInput input.Input, mutex *sync.Mutex,
 		subdomains = opendb.AppendDBSubdomains(anubis, subdomains)
 		threatminer := opendb.ThreatMinerSubdomains(urlUtils.CleanProtocol(target), false)
 		subdomains = opendb.AppendDBSubdomains(threatminer, subdomains)
+		dnsreponoc := opendb.DNSRepoNocSubdomains(urlUtils.CleanProtocol(target), false)
+		subdomains = opendb.AppendDBSubdomains(dnsreponoc, subdomains)
 
 		// Service Not Working
 		// bufferOverrun := opendb.BufferOverrunSubdomains(urlUtils.CleanProtocol(target), userInput.SubdomainPlain)
@@ -365,6 +373,10 @@ func SubdomainSubcommandHandler(userInput input.Input, mutex *sync.Mutex,
 		// sonar := opendb.SonarSubdomains(urlUtils.CleanProtocol(target), userInput.SubdomainPlain)
 		// subdomains = opendb.AppendDBSubdomains(sonar, subdomains)
 
+		// Service not working
+		// subdomaincenter := opendb.SubdomainCenterSubdomains(urlUtils.CleanProtocol(target), false)
+		// subdomains = opendb.AppendDBSubdomains(subdomaincenter, subdomains)
+
 		// Service not fully reliable yet
 		// if userInput.SubdomainBuiltWith {
 		// 	builtWithSubs := opendb.BuiltWithSubdomains(urlUtils.CleanProtocol(target), input.GetKey("builtwith"),