v0.4c live!

.gitignore

@@ -15,3 +15,6 @@
 
 # Binary 
 gOSINT
+
+#Telegram dump
+*.dump

README.md

@@ -1,22 +1,24 @@
-# gOSINT [![Build Status](https://travis-ci.org/Nhoya/gOSINT.svg?branch=master)](https://travis-ci.org/Nhoya/gOSINT) [![GitHub stars](https://img.shields.io/github/stars/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/stargazers) [![GitHub forks](https://img.shields.io/github/forks/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/network) [![Twitter](https://img.shields.io/twitter/url/https/github.com/Nhoya/gOSINT.svg?style=social&style=plastic)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2FNhoya%2FgOSINT)
+# gOSINT [![Build Status](https://travis-ci.org/Nhoya/gOSINT.svg?branch=master)](https://travis-ci.org/Nhoya/gOSINT) [![GitHub stars](https://img.shields.io/github/stars/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/stargazers) [![GitHub forks](https://img.shields.io/github/forks/Nhoya/gOSINT.svg)](https://github.com/Nhoya/gOSINT/network) [![Twitter](https://img.shields.io/twitter/url/https/github.com/Nhoya/gOSINT.svg?style=social&style=plastic)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2FNhoya%2FgOSINT) [![Go Report Card](https://goreportcard.com/badge/github.com/Nhoya/gOSINT)](https://goreportcard.com/report/github.com/Nhoya/gOSINT) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/76673062a30e48bd99d499d32c0c6af0)](https://www.codacy.com/app/Nhoya/gOSINT?utm_source=github.com&utm_medium=referral&utm_content=Nhoya/gOSINT&utm_campaign=Badge_Grade)
 OSINT framework in Go
 
-you probably want to take a look at the develop branch for more updates.
+Take a look at the [develop branch](https://github.com/Nhoya/gOSINT/tree/develop) for more updates.
 
 ## Introduction
 
+gOSINT is a small OSINT framework in Golang. If you want, feel free to contribute and/or leave a feedback!
 
-gOSINT is a small OSINT framework in Golang, if you want, feel free to contribute!
+## Like my project? Consider donation :)
 
+[![Paypal Badge](https://img.shields.io/badge/Donate-PayPal-yellow.svg)](https://www.paypal.me/Nhoya) [![BTC Badge](https://img.shields.io/badge/Donate-BTC-yellow.svg)](https://pastebin.com/raw/nyDDPwaM) [![Monero Badge](https://img.shields.io/badge/Donate-XMR-yellow.svg)](https://pastebin.com/raw/dNUFqwuC)
 
 ## What gOSINT can do
 
 - [x] Find mails from git repository
 - [x] Find Dumps for mail address
 - [x] Search for  mail address linked to domain/mail address in PGP keyring
-- [x] Retrive Info from domain whois (waiting to be implemented)
+- [x] Retrieve Info from domain whois (waiting to be implemented)
 - [x] Search for mail address in source code
-- [x] Retrive Telegram Public Groups History
+- [x] Retrieve Telegram Public Groups History
 
 ## Building
 
@@ -53,7 +55,7 @@ go get "github.com/jaytaylor/html2text"
 
 Currently `gOSINT` is still an early version and few modules are supported
 
-- [x] git support for mail retriving (using github API, bitbucket API or RAW clone and search) *Now with Pagination*!
+- [x] git support for mail retriving (using github API, bitbucket API or RAW clone and search)
 - [x] Search for mails in PGP Server
 - [x] [https://haveibeenpwned.com/](http://haveibeenpwned.com/) search for mail in databreach
 - [x] Retrive Telegram Public Group Messages
@@ -70,18 +72,23 @@ Usage:
   gOSINT [OPTIONS]
 
 Application Options:
-  -m, --module=[pgp|pwnd|git|plainSearch] Specify module
-      --url=                              Specify target URL
-      --gitAPI=[github|bitbucket]         Specify git website API to use (for git module,optional)
-      --mail=                             Specify mail target (for pgp and pwnd module)
-  -p, --path=                             Specify target path (for plainSearch module)
-  -f, --full                              Make deep search using linked modules
-  -c, --clone                             Enable clone function for plainSearch module (need to specify repo URL)
-      --ask-confirmation                  Ask confirmation before adding mail to set (for plainSearch module)
-  -v, --version                           Print version
+  -m, --module=[pgp|pwnd|git|plainSearch|telegram] Specify module
+  -v, --version                                    Print version
+      --url=                                       Specify target URL
+      --gitAPI=[github|bitbucket]                  Specify git website API to use (for git module,optional)
+  -c, --clone                                      Enable clone function for plainSearch module (need to specify repo URL)
+      --mail=                                      Specify mail target (for pgp and pwnd module)
+      --grace=                                     Specify telegram messages grace period (default: 15)
+  -g, --tgroup=                                    Specify Telegram group/channel name
+  -s, --tgstart=                                   Specify first message to scrape
+  -e, --tgend=                                     Specify last message to scrape
+      --dumpfile                                   Create and resume messages from dumpfile
+      --ask-confirmation                           Ask confirmation before adding mail to set (for plainSearch module)
+  -p, --path=                                      Specify target path (for plainSearch module)
+  -f, --full                                       Make deep search using linked modules
 
 Help Options:
-  -h, --help                              Show this help message
+  -h, --help                                       Show this help message
 ```
 
 ## Examples
@@ -91,7 +98,7 @@ Currently `gOSINT` supports the following actions
 
 `gOSINT -m git --url=[RepoURL] --gitAPI [github|bitbucket] (optional)`
 
-retrive mail from git commits
+retrieve mail from git commits
 
 `gOSINT -m git --url [RepoURL] --gitAPI [github|bitbucket] (optional) -f`
 
@@ -133,10 +140,23 @@ pass the resoult to pgp search and haveibeenpwnd modules
 
 ask confirmation before adding mail to search results
 
-`gOSINT -m telegram --target [PublicGroupName]`
+`gOSINT -m telegram --tgroup | -g  [PublicGroupName]`
 
-retrive message history for telegram public group
+retrieve message history for telegram public group
 
-`gOSINT -m telegram --target [PublicGroupName] --dumpfile`
+`gOSINT -m telegram --tgroup | -g [PublicGroupName] --dumpfile`
 
 the output will be stored in a file, if the file is already populated it will resume from the last ID
+
+`gOSINT -m telegram --tgroup | -g [PublicGroupName] --dumpfile -s [masageID] -e [messageID]`
+
+Set start and end messages for scraping
+
+## PGP module Demo
+[![asciicast](https://asciinema.org/a/21PCpbgFqyHiTbPINexHKEywj.png)](https://asciinema.org/a/21PCpbgFqyHiTbPINexHKEywj)
+
+## Pwnd module Demo
+[![asciicast](https://asciinema.org/a/x9Ap0IRcNNcLfriVujkNUhFSF.png)](https://asciinema.org/a/x9Ap0IRcNNcLfriVujkNUhFSF)
+
+## Telegram Crawler Demo
+[![asciicast](https://asciinema.org/a/nbRO9FNpjiYXAKeI87xn29j9z.png)](https://asciinema.org/a/nbRO9FNpjiYXAKeI87xn29j9z)

build.sh

@@ -6,6 +6,22 @@ readonly YELLOW="\033[00;33m"
 readonly BOLD="\033[01m"
 readonly END="\033[0m"
 
+version=$(go version 2> /dev/null)
+if [[ "$?" != 0 ]]; then
+        echo "Unable to find go, you need go >= 1.8 to build gOSINT"
+        exit 1
+fi
+go_version_regex="([0-9]).([0-9]).[0-9]"
+if [[ "$version" =~ $go_version_regex ]]; then
+        if [[ ${BASH_REMATCH[1]} -le 1 ]]; then
+                if [[ ${BASH_REMATCH[2]} -lt 8 ]]; then
+                        echo "This version of go is not supported, you need go >= 1.8"
+                        echo "Current: $version"
+                        exit 1
+                fi
+        fi
+fi
+
 dependencies=( github.com/deckarep/golang-set github.com/nhoya/goPwned github.com/jessevdk/go-flags gopkg.in/src-d/go-git.v4 github.com/jaytaylor/html2text)
 
 

gOSINT.go

@@ -8,21 +8,28 @@ import (
 	"github.com/jessevdk/go-flags"
 )
 
-const ver = "v0.4b"
+const ver = "v0.4c"
 
 var opts struct {
-	Module     string `short:"m" long:"module" description:"Specify module"  choice:"pgp" choice:"pwnd" choice:"git" choice:"plainSearch" choice:"telegram"`
-	Url        string `long:"url" default:"" description:"Specify target URL"`
-	Target     string `short:"t" long:"target" default:"" description:"Specify target"`
+	Module  string `short:"m" long:"module" description:"Specify module"  choice:"pgp" choice:"pwnd" choice:"git" choice:"plainSearch" choice:"telegram"`
+	Version bool   `short:"v" long:"version" description:"Print version"`
+	// git module
+	URL        string `long:"url" default:"" description:"Specify target URL"`
 	GitAPIType string `long:"gitAPI" default:"" description:"Specify git website API to use (for git module,optional)" choice:"github" choice:"bitbucket"`
-	Mail       string `long:"mail" default:"" description:"Specify mail target (for pgp and pwnd module)"`
-	Path       string `short:"p" long:"path" description:"Specify target path (for plainSearch module)"`
-	TgGrace    int    `long:"grace" default:"15" description:"Specify telegram messages grace period"`
-	DumpFile   bool   `long:"dumpfile" description:"Create and resume messages from dumpfile"`
-	Mode       bool   `short:"f" long:"full" description:"Make deep search using linked modules"`
 	Clone      bool   `short:"c" long:"clone" description:"Enable clone function for plainSearch module (need to specify repo URL)"`
-	Confirm    bool   `long:"ask-confirmation" description:"Ask confirmation before adding mail to set (for plainSearch module)"`
-	Version    bool   `short:"v" long:"version" description:"Print version"`
+	// pwn and pgp module
+	Mail string `long:"mail" default:"" description:"Specify mail target (for pgp and pwnd module)"`
+	// telegram module
+	TgGrace  int    `long:"grace" default:"15" description:"Specify telegram messages grace period"`
+	TgGroup  string `short:"g" long:"tgroup" default:"" description:"Specify Telegram group/channel name"`
+	TgStart  int    `short:"s" long:"tgstart" default:"1" default-mask:"-" description:"Specify first message to scrape"`
+	TgEnd    int    `short:"e" long:"tgend" description:"Specify last message to scrape"`
+	DumpFile bool   `long:"dumpfile" description:"Create and resume messages from dumpfile"`
+	// plainSearch module
+	Confirm bool   `long:"ask-confirmation" description:"Ask confirmation before adding mail to set (for plainSearch module)"`
+	Path    string `short:"p" long:"path" description:"Specify target path (for plainSearch module)"`
+	// generic
+	Mode bool `short:"f" long:"full" description:"Make deep search using linked modules"`
 }
 
 func mailCheck(mailSet mapset.Set) {
@@ -45,53 +52,20 @@ func main() {
 		fmt.Println("gOSINT " + ver)
 		os.Exit(0)
 	}
-	if opts.Url != "" {
-		isUrl(opts.Url)
+	if opts.URL != "" {
+		isURL(opts.URL)
 	}
 
 	switch mod := opts.Module; mod {
 	case "pwnd":
-		mailCheck(mailSet)
-		pwnd(mailSet)
+		initPwnd(mailSet)
 	case "pgp":
-		mailCheck(mailSet)
-		mailSet = pgpSearch(mailSet)
-		if opts.Mode {
-			pwnd(mailSet)
-		}
+		initPGP(mailSet)
 	case "git":
-		if opts.Url == "" {
-			fmt.Println("You must specify target URL")
-			os.Exit(1)
-		}
-		mailSet = gitSearch(opts.Url, opts.GitAPIType, mailSet)
-		if opts.Mode {
-			mailSet = pgpSearch(mailSet)
-			pwnd(mailSet)
-		}
+		initGit(mailSet)
 	case "plainSearch":
-		if opts.Clone {
-			if opts.Url == "" {
-				fmt.Println("You must specify target URL")
-				os.Exit(1)
-			}
-			mailSet = cloneAndSearch(opts.Url, mailSet, opts.Confirm)
-		} else {
-			if opts.Path == "" {
-				fmt.Println("You must specify Path")
-				os.Exit(1)
-			}
-			mailSet = plainMailSearch(opts.Path, mailSet, opts.Confirm)
-		}
-		if opts.Mode {
-			mailSet = pgpSearch(mailSet)
-			pwnd(mailSet)
-		}
+		initPlainSearch(mailSet)
 	case "telegram":
-		if opts.Target == "" {
-			fmt.Println("You must specify target")
-			os.Exit(1)
-		}
-		getTelegramGroupHistory(opts.Target, opts.TgGrace, opts.DumpFile)
+		initTelegram()
 	}
 }

git.go

@@ -14,6 +14,18 @@ import (
 	"gopkg.in/src-d/go-git.v4/storage/memory"
 )
 
+func initGit(mailSet mapset.Set) {
+	if opts.URL == "" {
+		fmt.Println("You must specify target URL")
+		os.Exit(1)
+	}
+	mailSet = gitSearch(opts.URL, opts.GitAPIType, mailSet)
+	if opts.Mode {
+		mailSet = pgpSearch(mailSet)
+		pwnd(mailSet)
+	}
+}
+
 func gitSearch(target string, WebsiteAPI string, mailSet mapset.Set) mapset.Set {
 	// TODO: add worker for pagination
 	domain := ""
@@ -27,11 +39,11 @@ func gitSearch(target string, WebsiteAPI string, mailSet mapset.Set) mapset.Set
 		fmt.Println("[+] Using github API")
 		domain = targetSplit[0] + "//api." + targetSplit[2] + "/repos/" + targetSplit[3] + "/" + targetSplit[4] + "/commits?per_page=100"
 		//GitHub Pagination
-		lastPage := retriveLastGHPage(domain)
+		lastPage := retrieveLastGHPage(domain)
 		fmt.Println("[+] Looping through pages.This MAY take a while...")
 		for page := 1; page < lastPage+1; page++ {
 			fmt.Println("[+] Analyzing commits page: " + strconv.Itoa(page))
-			commits = retriveRequestBody(domain + "&page=" + strconv.Itoa(page))
+			commits = retrieveRequestBody(domain + "&page=" + strconv.Itoa(page))
 			findMailInText(commits, mailSet)
 		}
 	} else if strings.Contains(target, "https://bitbucket.org") || WebsiteAPI == "bitbucket" {
@@ -44,8 +56,8 @@ func gitSearch(target string, WebsiteAPI string, mailSet mapset.Set) mapset.Set
 		for page != 0 {
 			fmt.Println("[+] Analyzing commits page: " + strconv.Itoa(page))
 			pageDom := domain + "&page=" + strconv.Itoa(page)
-			//This is needed because we can't unluckily retrive max_page from one single request
-			pageContent := retriveRequestBody(pageDom)
+			//This is needed because we can't unluckily retrieve max_page from one single request
+			pageContent := retrieveRequestBody(pageDom)
 			nextPage := "\"next\": \"" + domain + "&page="
 
 			findMailInText(pageContent, mailSet)
@@ -60,7 +72,7 @@ func gitSearch(target string, WebsiteAPI string, mailSet mapset.Set) mapset.Set
 		findMailInText(commits, mailSet)
 	}
 
-	//Check if the mailset has been populated (this avoids problems with mispelled repositories too)
+	//Check if the mailset has been populated (this avoids problems with misspelled repositories too)
 	if mailSet == nil {
 		fmt.Println("[-] Nothing Found")
 		os.Exit(1)
@@ -70,7 +82,7 @@ func gitSearch(target string, WebsiteAPI string, mailSet mapset.Set) mapset.Set
 	return mailSet
 }
 
-func retriveLastGHPage(domain string) int {
+func retrieveLastGHPage(domain string) int {
 	req, err := http.Get(domain)
 	if err != nil {
 		panic(err)
@@ -85,10 +97,10 @@ func retriveLastGHPage(domain string) int {
 	return 1
 }
 
-func cloneAndSearchCommit(Url string) string {
+func cloneAndSearchCommit(URL string) string {
 	fmt.Println("[+] Cloning Repo")
 	r, _ := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
-		URL: Url,
+		URL: URL,
 	})
 	ref, _ := r.Head()
 	cIter, _ := r.Log(&git.LogOptions{From: ref.Hash()})

pgp.go

@@ -6,14 +6,22 @@ import (
 	"github.com/deckarep/golang-set"
 )
 
+func initPGP(mailSet mapset.Set) {
+	mailCheck(mailSet)
+	mailSet = pgpSearch(mailSet)
+	if opts.Mode {
+		pwnd(mailSet)
+	}
+}
+
 func pgpSearch(mailSet mapset.Set) mapset.Set {
 	fmt.Println("==== PGP SEARCH ====")
 	mailIterator := mailSet.Iterator()
 	for mail := range mailIterator.C {
 		pgpSet := mapset.NewSet()
 		fmt.Println("[+] pgp search for " + mail.(string))
 		domain := "http://pgp.mit.edu/pks/lookup?search=" + mail.(string)
-		body := retriveRequestBody(domain)
+		body := retrieveRequestBody(domain)
 		findMailInText(body, pgpSet)
 		if pgpSet != nil {
 			pgpIterator := pgpSet.Iterator()

pwnd.go

@@ -8,6 +8,11 @@ import (
 	"github.com/nhoya/goPwned"
 )
 
+func initPwnd(mailSet mapset.Set) {
+	mailCheck(mailSet)
+	pwnd(mailSet)
+}
+
 func pwnd(mailSet mapset.Set) {
 	fmt.Println("==== HAVEIBEENPWND SEARCH ====")
 	mailIterator := mailSet.Iterator()