add detecting site file to gather.py

gather.py

@@ -15,6 +15,7 @@
 import sys,socket
 import json
 import urlparse
+import re
 
 noTitle = False
 def encoding(data):
@@ -147,7 +148,7 @@ def baseInfo(url):
 	except:
 		pass
 
-		req.get_method = lambda: 'HEAD'
+	req.get_method = lambda: 'GET'
 	try:
 		response = urllib2.urlopen(req, timeout = 15)
 		#for k, v in response.info().items():
@@ -163,14 +164,40 @@ def baseInfo(url):
 	except:
 		pass
 
+	req.get_method = lambda: 'DEBUG'
+	try:
+		response = urllib2.urlopen(req, timeout = 15)
+		print '* Support Debug Method'
+		#print response.read()
+	except Exception, e:
+		if hasattr(e, 'code'):
+			if not (e.code == 501 or e.code == 405 or e.code == 403):
+				print 'TRACE: ', e
+
 	req.get_method = lambda: 'TRACE'
 	try:
 		response = urllib2.urlopen(req, timeout = 15)
+		print '* Support TRACE Header'
 	except Exception, e:
 		if hasattr(e, 'code'):
 			if not (e.code == 501 or e.code == 405 or e.code == 403):
 				print 'TRACE: ', e
 
+def querySiteFile(url):
+	files = ( ('robots.txt', 'Allow|Disallow'), ('crossdomain.xml', 'cross-domain-policy'), 
+		('phpinfo.php', 'PHP Version'), ('sitemap.xml', 'schemas\/sitemap'), )
+	for file in files:
+		try:
+			response = urllib2.urlopen(url + '/' + file[0], timeout = 15)
+			html = response.read()
+			if not re.search(file[1], html, re.IGNORECASE):
+				continue
+			print '[%s]' % file[0]
+			print html[:4096]
+		except:
+			#raise
+			pass
+
 if __name__ == '__main__':
 	import locale	
 	reload(sys)
@@ -181,7 +208,7 @@ def baseInfo(url):
 	urllib2.install_opener(opener)
 
 	options = 0
-	opts, args = getopt.getopt(sys.argv[1:], "Nrswb")
+	opts, args = getopt.getopt(sys.argv[1:], "fNrswb")
 	for op, vaule in opts:
 		if op == '-N':
 			noTitle = True
@@ -193,9 +220,11 @@ def baseInfo(url):
 			options |= 4
 		elif op == '-b':
 			options |= 16
+		elif op == '-f':
+			options |= 32
 
 	if options == 0:
-		options = 1 | 2 | 4 | 8 | 16
+		options = 1 | 2 | 4 | 8 | 16 | 32
 	url = args[0]
 	if url[:7] != 'http://' and url[:8] != 'https://':
 		url = 'http://' + url
@@ -216,6 +245,9 @@ def baseInfo(url):
 	if options & 16:
 		print '\n============================== baidu weight ==============================\n'
 		queryWeight(urlP.hostname)
+	if options & 32:
+		print '\n============================== site file ==============================\n'
+		querySiteFile(url)
 	if options & 8:
 		print '\n============================== nmap ==============================\n'
 		sys.stdout.flush()

ghack.py

@@ -38,9 +38,24 @@
 	'inurl:file', 'inurl:down', 'inurl:passwd', 'inurl:path', "#", 
 	'intitle:"index.of"', 'intitle:管理 ', "#",
 	'intext:*@*.com', 'intext:*@*.net', 'intext:*@*.cn', 'intext:ftp://*:* ',  
-	'intext:powered by', 
+	'intext:powered by', 'inurl:phpinfo.php', 
 	'inurl:":8000"', 'inurl:":8080"', 'inurl:":8081"', 'inurl:":81"', 
-	'inurl:":88"', 'inurl:":8888"', 'inurl:":8088"'
+	'inurl:":88"', 'inurl:":8888"', 'inurl:":8088"', 
+	'Microsoft OLE DB Provider for SQL Server', 
+	'microsoft jet database engine error', 
+	'type mismatch', 
+	'you have an error in your sql syntax', 
+	'invalid sql statement', 
+	'DorisDuke error', 
+	'OleDbException', 
+	'JasperException', 
+	'Fatal Error', 
+	'supplied argument is not a valid mysql', 
+	'mysql_', 
+	'odbc',
+	'jdbc', 
+	'ora-00921', 
+	'ADODB',
 	)
 
 def execCmd(url):

utils/bingsearch.py

@@ -61,6 +61,8 @@ def _urlFilter(url):
 		return False
 	if url.find('bing.com') != -1:
 		return False
+	if url.find('.live.com') != -1:
+		return False
 	if url.find('google.com') != -1:
 		return False
 	if url.find('.googleusercontent.com') != -1:

utils/gfsoso.py

@@ -18,6 +18,7 @@
 import time
 import locale
 import webutils
+import googlesearch
 
 #GOOGLE_HOME = 'http://www.google.com.hk/'
 GOOGLE_HOME = 'http://64.233.161.104/'
@@ -27,34 +28,48 @@
 reqDelay = 0.0
 #maxResult = 10
 totalRecord = sys.maxint
-
+RedirectedUrl = GFSOSO_HOME
 _cookieFetched = False
 
 def _refreshCookie(opener, what):
+
+	what = urllib2.quote(what)
 	url = GFSOSO_HOME + '?q=%s' % (what)
 	req = urllib2.Request(url)
 	webutils.setupRequest(req)
 	req.add_header('Referer', GFSOSO_HOME)
 	try:
 		response = opener.open(req, timeout = REQ_TIMEOUT)
+		# print response.geturl()
+		if response.geturl().find(GFSOSO_HOME) == -1:
+			global RedirectedUrl
+			RedirectedUrl = response.geturl()
+			RedirectedUrl = RedirectedUrl[0 : RedirectedUrl.find('/', 7) + 1]
+			# print RedirectedUrl
+			return False
+
 		html = response.read()
 	except Exception, e:
+		print e
 		html = ''
 		if e.code == 301: # moved
 			# html = reduce(lambda x,y: x + y, e.readlines())
 			for line in e.readlines():
 				html += line
 		else:
 			print "Exception: url: %s - " % url, e
-			return
+			return False
 
 	m = re.search(r"_GFTOKEN','([0-9a-f]+)'", html)
 
 	webutils.cookieJar.set_cookie(_makeCookie('AJSTAT_ok_pages', '1'))
 	webutils.cookieJar.set_cookie(_makeCookie('AJSTAT_ok_times', '1'))
-	webutils.cookieJar.set_cookie(_makeCookie('_GFTOKEN', m.group(1)))
+	if m:
+		webutils.cookieJar.set_cookie(_makeCookie('_GFTOKEN', m.group(1)))
+		return False
 	global _cookieFetched
 	_cookieFetched = True
+	return True
 
 def _urlFilter(url):
 	if url.find('http:') == -1 and url.find('ftp:') == -1 and url.find('https:') == -1:
@@ -140,7 +155,6 @@ def _gfsosoSearch(opener, what, resultNum = -1, startNum = 0):
 	#import pdb
 	#pdb.set_trace()
 
-	what = urllib2.quote(what)
 	if resultNum == -1:
 		pageCount = -1
 	else:
@@ -149,11 +163,19 @@ def _gfsosoSearch(opener, what, resultNum = -1, startNum = 0):
 	startPage = int((startNum + NUM_PER_PAGE - 1) / NUM_PER_PAGE)
 
 	if not _cookieFetched:
-		_refreshCookie(opener, what)
+		if not _refreshCookie(opener, what):
+			#global _gfsosoPageHandler
+			googlesearch.GOOGLE_HOME = RedirectedUrl
+			#print RedirectedUrl
+			for url in googlesearch.google(opener, what, resultNum, startNum):
+				yield url
+			return
 
 	global totalRecord
 	totalRecord = sys.maxint
 
+	what = urllib2.quote(what)
+
 	pageNum = 1
 	resCnt = 0
 	while True:

utils/google.py

@@ -20,7 +20,7 @@
 elif search_engine == 'aol':
  	google = aolsearch.google
 elif search_engine == 'bing':
- 	google = gingsearch.google
+ 	google = bingsearch.google
 else:
 	google = gfsoso.google
 

utils/googlesearch.py

@@ -116,7 +116,8 @@ def _googleSearch(opener, what, resultNum = -1, startNum = 0):
 		global GOOGLE_HOME
 		global GOOGLE_SEARCH_URL
 		GOOGLE_HOME = os.environ['google_home']
-		GOOGLE_SEARCH_URL = GOOGLE_HOME + '/search?hl=en_US&start=%d&q=%s'
+
+	GOOGLE_SEARCH_URL = GOOGLE_HOME + '/search?hl=en_US&start=%d&q=%s'
 
 	what = urllib2.quote(what)
 	if resultNum == -1: