Merge pull request #15 from projectdiscovery/feature-silent-and-raw-o…

…utput

some improvements

README.md

@@ -17,6 +17,7 @@ DNSProbe is a tool built on top of [retryabledns](https://github.com/projectdisc
     - [From Source](#from-source)
     - [Querying host for A record](#querying-host-for-a-record)
     - [Querying host for CNAME record](#querying-host-for-cname-record)
+    - [Querying CNAME records on the Subfinder output](#querying-cname-records-on-the-subfinder-output)
 - [License](#license)
 
 # Features
@@ -45,6 +46,8 @@ This will display help for the tool. Here are all the switches it supports.
 | -t             | Number of concurrent requests to make (default 250)                                                              | dnsprobe -t 500                             |
 | -f             | Output type: ip, domain, response, simple (domain + ip, default), full (domain + response), json (domain + raw response)  | dnsprobe -f json           |
 | -o             | Output file (optional)                                                                                                      | dnsprobe -o result.txt                                |
+| -raw           | Output the full response ignoring output type                                                                  | dnsprobe -raw               |
+| -silent        | Show only found results in output                     | dnsprobe -silent             |
 
 # Installation Instructions
 ### From Source
@@ -59,7 +62,7 @@ In order to update the tool, you can use -u flag with go get command.
 
 ### Querying host for A record
 
-To query a list of domains, you can pass the list via stdin.
+To query a list of domains, you can pass the list via stdin (it also accepts full URLS, in this case the domain is extracted automatically).
 
 ```bash
 > cat domains.txt | dnsprobe

dnsprobe.go

@@ -22,6 +22,8 @@ var (
 	retries      = flag.Int("c", 1, "Max dns retries")
 	outputFormat = flag.String("f", "simple", "Output type: ip, domain, response, simple (domain + ip), full (domain + response), json (domain + raw response)")
 	outputFile   = flag.String("o", "", "Output file")
+	raw          = flag.Bool("raw", false, "Operates like dig")
+	silent       = flag.Bool("silent", false, "Silent output")
 )
 
 type JsonLine struct {
@@ -31,9 +33,15 @@ type JsonLine struct {
 }
 
 func main() {
-	showBanner()
+
 	flag.Parse()
 
+	if *silent {
+		gologger.MaxLevel = gologger.Silent
+	}
+
+	showBanner()
+
 	options := dnsprobe.DefaultOptions
 	options.MaxRetries = *retries
 
@@ -110,7 +118,15 @@ func main() {
 		go func(domain string) {
 			defer wg.Done()
 
-			if rs, err := dnsProbe.LookupRaw(domain); err == nil {
+			if isURL(domain) {
+				domain = extractDomain(domain)
+			}
+
+			if rs, rawResp, err := dnsProbe.LookupRaw(domain); err == nil {
+				if *raw {
+					writequeue <- "\n" + rawResp
+					return
+				}
 				for _, r := range rs {
 					tokens := strings.Split(r, "\t")
 					ip := tokens[len(tokens)-1]
@@ -131,7 +147,6 @@ func main() {
 							writequeue <- fmt.Sprintln(string(jsonls))
 						}
 					}
-
 				}
 			}
 		}(sc.Text())

go.mod

@@ -5,6 +5,6 @@ go 1.14
 require (
 	github.com/miekg/dns v1.1.29
 	github.com/projectdiscovery/gologger v1.0.0
-	github.com/projectdiscovery/retryabledns v1.0.2
+	github.com/projectdiscovery/retryabledns v1.0.3
 	github.com/remeh/sizedwaitgroup v1.0.0
 )

go.sum

@@ -8,6 +8,8 @@ github.com/projectdiscovery/gologger v1.0.0 h1:XAQ8kHeVKXMjY4rLGh7eT5+oHU077BNEv
 github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/retryabledns v1.0.2 h1:vLVABSJbLXfvREf1ApTBX19sCPVa2puxXA5RREM2+uw=
 github.com/projectdiscovery/retryabledns v1.0.2/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
+github.com/projectdiscovery/retryabledns v1.0.3 h1:M54uIgT8xo1fuc3hG81enQFekrwABNQl3yqb978SfMA=
+github.com/projectdiscovery/retryabledns v1.0.3/go.mod h1:/UzJn4I+cPdQl6pKiiQfvVAT636YZvJQYZhYhGB0dUQ=
 github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E=
 github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

lib/dnsprobe.go

@@ -3,13 +3,13 @@ package dnsprobe
 import (
 	"net"
 
-	miekdns "github.com/miekg/dns"
-	dns "github.com/projectdiscovery/retryabledns"
+	miekgdns "github.com/miekg/dns"
+	retryabledns "github.com/projectdiscovery/retryabledns"
 )
 
 // DnsProbe is structure to perform dns lookups
 type DnsProbe struct {
-	dnsClient    *dns.Client
+	dnsClient    *retryabledns.Client
 	questionType uint16
 }
 
@@ -24,7 +24,7 @@ type Options struct {
 var DefaultOptions = Options{
 	BaseResolvers: DefaultResolvers,
 	MaxRetries:    5,
-	QuestionType:  miekdns.TypeA,
+	QuestionType:  miekgdns.TypeA,
 }
 
 // DefaultResolvers contains the list of resolvers known to be trusted.
@@ -38,7 +38,7 @@ var DefaultResolvers = []string{
 
 // New creates a dns resolver
 func New(options Options) (*DnsProbe, error) {
-	dnsClient, err := dns.New(options.BaseResolvers, options.MaxRetries)
+	dnsClient, err := retryabledns.New(options.BaseResolvers, options.MaxRetries)
 	if err != nil {
 		return nil, err
 	}
@@ -61,9 +61,9 @@ func (d *DnsProbe) Lookup(hostname string) ([]string, error) {
 }
 
 // LookupRaw performs a DNS question of a specified type and returns raw responses
-func (d *DnsProbe) LookupRaw(hostname string) ([]string, error) {
+func (d *DnsProbe) LookupRaw(hostname string) ([]string, string, error) {
 	if ip := net.ParseIP(hostname); ip != nil {
-		return []string{hostname}, nil
+		return []string{hostname}, "", nil
 	}
 
 	return d.dnsClient.ResolveRaw(hostname, d.questionType)

util.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"net/url"
 	"os"
 )
 
@@ -19,3 +20,27 @@ func linesInFile(fileName string) ([]string, error) {
 	}
 	return result, nil
 }
+
+// isURL tests a string to determine if it is a well-structured url or not.
+func isURL(toTest string) bool {
+	_, err := url.ParseRequestURI(toTest)
+	if err != nil {
+		return false
+	}
+
+	u, err := url.Parse(toTest)
+	if err != nil || u.Scheme == "" || u.Host == "" {
+		return false
+	}
+
+	return true
+}
+
+func extractDomain(URL string) string {
+	u, err := url.Parse(URL)
+	if err != nil {
+		return ""
+	}
+
+	return u.Hostname()
+}