diff --git a/README.md b/README.md index c994007..a7c0555 100644 --- a/README.md +++ b/README.md @@ -270,7 +270,7 @@ If there aren't errors, go ahead :) **Help me building this!** -Special thanks to: [danielmiessler](https://github.com/danielmiessler), [sonarSearch](https://github.com/cgboal/sonarsearch), [HackerTarget](https://hackertarget.com/), [BufferOverrun](http://dns.bufferover.run/), [Threatcrowd](https://www.threatcrowd.org/), [Crt.sh](https://crt.sh/), [VirusTotal](https://www.virustotal.com/), [tomnomnom](https://github.com/tomnomnom/assetfinder). +Special thanks to: [danielmiessler](https://github.com/danielmiessler), [HackerTarget](https://hackertarget.com/), [Threatcrowd](https://www.threatcrowd.org/), [Crt.sh](https://crt.sh/), [VirusTotal](https://www.virustotal.com/), [tomnomnom](https://github.com/tomnomnom/assetfinder). **To do:** diff --git a/go.mod b/go.mod index 2ed3494..61753d3 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,15 @@ module github.com/edoardottt/scilla go 1.18 require ( + github.com/PuerkitoBio/goquery v1.8.1 github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 - github.com/fatih/color v1.15.0 + github.com/fatih/color v1.16.0 github.com/gocolly/colly v1.2.0 github.com/stretchr/testify v1.8.4 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect github.com/antchfx/htmlquery v1.3.0 // indirect github.com/antchfx/xmlquery v1.3.17 // indirect @@ -22,12 +22,12 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/temoto/robotstxt v1.1.2 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect + golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.13.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.31.0 // indirect diff --git a/go.sum b/go.sum index 6887183..358931a 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,8 @@ github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89/go.mod h1:/0 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= @@ -34,8 +34,8 @@ github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= @@ -76,8 +76,8 @@ golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= diff --git a/pkg/opendb/dnsreponoc.go b/pkg/opendb/dnsreponoc.go new file mode 100644 index 0000000..d277124 --- /dev/null +++ b/pkg/opendb/dnsreponoc.go @@ -0,0 +1,90 @@ +/* + +======================= +Scilla - Information Gathering Tool +======================= + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see http://www.gnu.org/licenses/. + + @Repository: https://github.com/edoardottt/scilla + + @Author: edoardottt, https://www.edoardoottavianelli.it + + @License: https://github.com/edoardottt/scilla/blob/main/LICENSE + +*/ + +package opendb + +import ( + "fmt" + "io" + "net/http" + + "github.com/PuerkitoBio/goquery" + + httpUtils "github.com/edoardottt/scilla/internal/http" +) + +func scrape(body io.ReadCloser) []string { + var result = make([]string, 0) + + tableIndex := 1 // Looks for the second table from DNSRepoNoc html page + columnIndex := 0 // Looks for the first index that contains list of subdomains + + doc, err := goquery.NewDocumentFromReader(body) + if err != nil { + fmt.Println(err) + } + + var row string + + doc.Find(".table-responsive").Each(func(index int, tablehtml *goquery.Selection) { + if index == tableIndex { + tablehtml.Find("tr").Each(func(indextr int, rowhtml *goquery.Selection) { + rowhtml.Find("td").Each(func(indexth int, tablecell *goquery.Selection) { + if indexth == columnIndex { + row = tablecell.Text() + } + }) + result = append(result, row) + }) + } + }) + + return result +} + +// DNSRepoNoc retrieves from the url below some known subdomains - without API Key. +func DNSRepoNocSubdomains(domain string, plain bool) []string { + if !plain { + fmt.Println("Pulling data from Dns Repo Noc") + } + + client := http.Client{ + Timeout: httpUtils.Seconds30, + } + + url := "https://dnsrepo.noc.org/?domain=" + domain + resp, err := client.Get(url) + + if err != nil { + return []string{} + } + defer resp.Body.Close() + + output := scrape(resp.Body) + + return output +} diff --git a/pkg/opendb/subdomaincenter.go b/pkg/opendb/subdomaincenter.go new file mode 100644 index 0000000..f14a7ec --- /dev/null +++ b/pkg/opendb/subdomaincenter.go @@ -0,0 +1,75 @@ +/* + +======================= +Scilla - Information Gathering Tool +======================= + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see http://www.gnu.org/licenses/. + + @Repository: https://github.com/edoardottt/scilla + + @Author: edoardottt, https://www.edoardoottavianelli.it + + @License: https://github.com/edoardottt/scilla/blob/main/LICENSE + +*/ + +package opendb + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + + httpUtils "github.com/edoardottt/scilla/internal/http" +) + +// SubdomainCenter retrieves from the url below some known subdomains. +func SubdomainCenterSubdomains(domain string, plain bool) []string { + if !plain { + fmt.Println("Pulling data from Subdomain Center") + } + + client := http.Client{ + Timeout: httpUtils.Seconds30, + } + + result := make([]string, 0) + url := "http://api.subdomain.center/?domain=" + domain + + resp, err := client.Get(url) + if err != nil { + return result + } + defer resp.Body.Close() + + // read the response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return result + } + + // Decode the response body as list of string + var response []string + err = json.Unmarshal(body, &response) + + if err != nil { + return result + } + + result = append(result, response...) + + return result +} diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index b9535c6..85f290a 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -179,6 +179,8 @@ func ReportSubcommandHandler(userInput input.Input, mutex *sync.Mutex, subdomains = opendb.AppendDBSubdomains(anubis, subdomains) threatminer := opendb.ThreatMinerSubdomains(urlUtils.CleanProtocol(target), false) subdomains = opendb.AppendDBSubdomains(threatminer, subdomains) + dnsreponoc := opendb.DNSRepoNocSubdomains(urlUtils.CleanProtocol(target), false) + subdomains = opendb.AppendDBSubdomains(dnsreponoc, subdomains) // Service Not Working // bufferOverrun := opendb.BufferOverrunSubdomains(urlUtils.CleanProtocol(target), false) @@ -188,6 +190,10 @@ func ReportSubcommandHandler(userInput input.Input, mutex *sync.Mutex, // sonar := opendb.SonarSubdomains(urlUtils.CleanProtocol(target), false) // subdomains = opendb.AppendDBSubdomains(sonar, subdomains) + // Service not working + // subdomaincenter := opendb.SubdomainCenterSubdomains(urlUtils.CleanProtocol(target), false) + // subdomains = opendb.AppendDBSubdomains(subdomaincenter, subdomains) + if userInput.ReportVirusTotal { vtSubs := opendb.VirusTotalSubdomains(urlUtils.CleanProtocol(target), input.GetKey("virustotal"), false) subdomains = opendb.AppendDBSubdomains(vtSubs, subdomains) @@ -356,6 +362,8 @@ func SubdomainSubcommandHandler(userInput input.Input, mutex *sync.Mutex, subdomains = opendb.AppendDBSubdomains(anubis, subdomains) threatminer := opendb.ThreatMinerSubdomains(urlUtils.CleanProtocol(target), false) subdomains = opendb.AppendDBSubdomains(threatminer, subdomains) + dnsreponoc := opendb.DNSRepoNocSubdomains(urlUtils.CleanProtocol(target), false) + subdomains = opendb.AppendDBSubdomains(dnsreponoc, subdomains) // Service Not Working // bufferOverrun := opendb.BufferOverrunSubdomains(urlUtils.CleanProtocol(target), userInput.SubdomainPlain) @@ -365,6 +373,10 @@ func SubdomainSubcommandHandler(userInput input.Input, mutex *sync.Mutex, // sonar := opendb.SonarSubdomains(urlUtils.CleanProtocol(target), userInput.SubdomainPlain) // subdomains = opendb.AppendDBSubdomains(sonar, subdomains) + // Service not working + // subdomaincenter := opendb.SubdomainCenterSubdomains(urlUtils.CleanProtocol(target), false) + // subdomains = opendb.AppendDBSubdomains(subdomaincenter, subdomains) + // Service not fully reliable yet // if userInput.SubdomainBuiltWith { // builtWithSubs := opendb.BuiltWithSubdomains(urlUtils.CleanProtocol(target), input.GetKey("builtwith"),