BingIP

#!/usr/bin/env python3

import urllib.parse
import dns.resolver
import collections
import traceback
import argparse
import requests
import sys
import os
import re

import utils

# defaults
default_nameserver = '8.8.8.8'
default_user_agent = 'Mozilla/4.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/11.0.1245.0 Safari/537.36'
default_max_threads = 6

def bing_search(search, first=0, proxy=None, user_agent=None):
    """
    Perform a Bing search

    :param search: Search query
    :param first: Pagination parameter, in increments of 10
    :param proxy: Proxy to use with requests.get()
    :param user_agent: User-agent for request
    :return: Results
    """

    encoded_search = urllib.parse.quote(search)
    url = 'https://www.bing.com/search?q={}&go=&qs=n&first={}&FORM=PERE'.format(encoded_search, first)

    # proxy
    if proxy:
        proxies = {
            'http': proxy,
            'https': proxy
        }
    else:
        proxies = None

    # user-agent
    if user_agent:
        headers = {
            'User-Agent': user_agent,
        }
    else:
        headers = None

    response = requests.get(url, proxies=proxies, headers=headers) 

    if response.status_code != 200:
        raise RuntimeError('Search request failed with code {}: {}'.format(response.status_code, search))

    soup = utils.soup_up(response)

    results = []

    # Search results look like this:
    #
    # <ol id="b_results">
    #     <li class="b_algo">
    #         <h2>
    #             <a href="URL">LINK NAME</a>
    #         </h2>
    # 
    #         <div class="b_caption">
    #             <div class="b_attribution" ...>
    #                 ...
    #                 <cite>VISIBLE URL</cite>
    #                 <span class="c_tlbxTrg">
    #                     <span class="c_tlbxH" H="BASE:CACHEDPAGEDEFAULT" K="SERP,5085.1">
    #                     </span>
    #                 </span>
    #             </div>
    #             <p>
    #                 CAPTION
    #             </p>
    #         </div>
    #     </li>
    #     ...
    # </ol>
    #
    b_results = soup.find('ol', {'id': 'b_results'})

    if not b_results:
        raise RuntimeError('Search failed: {}'.format(search))

    for li in b_results.find_all('li', {'class': 'b_algo'}):
        a = li.find('a', href=True)

        if not a:
            utils.debug('Bing scraper failed to find <a> in <li>')
            continue

        href = a['href']
        title = a.text

        b_caption = li.find('div', {'class': 'b_caption'})
        if b_caption:
            caption = b_caption.find('p').text
        else:
            utils.debug('Bing scraper failed to find div b_caption')
            caption = None

        result = {
                    'query': search,
                    'href': href,
                    'title': title,
                    'caption': caption,
                 }

        results.append(result)

    return results

def main():
    parser = argparse.ArgumentParser()
    group = parser.add_argument_group('input arguments')
    group.add_argument('item', nargs='*', help='IP address, CIDR range, or domain name to perform recon on')
    group.add_argument('-f', '--file', action='append', help='file containing IP addresses, CIDR ranges, and/or domain names to perform recon on')

    group = parser.add_argument_group('http arguments')
    group.add_argument('-p', '--proxy', help='use proxy')
    group.add_argument('-A', '--user-agent',
            default=default_user_agent,
            help='user-agent for HTTP requests (default: {})'.format(default_user_agent))

    group = parser.add_argument_group('dns arguments')
    group.add_argument('-n', '--nameserver', default=[default_nameserver], action='append',
            help='use these nameservers (default: {})'.format(default_nameserver))
    group.add_argument('--system-nameservers', action='store_true',
            help='use system nameservers')
    group.add_argument('-u', '--udp', action='store_true',
            help='use udp instead of tcp to resolve domains')

    group = parser.add_argument_group('threading arguments')
    group.add_argument('-t', '--max-threads', type=int, default=default_max_threads,
            help='maximum number of threads to use at once (default: {})'.format(default_max_threads))

    group = parser.add_argument_group('output arguments')
    group.add_argument('-s', '--short', help='short output format')
    group.add_argument('-d', '--domains-only', action='store_true', help='only output domains')
    group.add_argument('-o', '--output', help='tee output to a file')
    group.add_argument('-q', '--quiet', action='store_true', help='do not print status messages to stderr')
    group.add_argument('-D', '--debug', action='store_true', help='enable debug output')
    args = parser.parse_args()

    # -D/--debug
    if args.debug:
        # enable debug messages
        utils.enable_debug()

    # -q/--quiet
    if args.quiet:
        # disable prefixed stderr messages
        utils.disable_status()

    # -o/--output
    if args.output:
        # set log file
        utils.set_log(args.output)

    # --system-nameservers
    if not args.system_nameservers:
        # -n/--nameserver
        dns.resolver.default_resolver = dns.resolver.Resolver(configure=False)
        dns.resolver.default_resolver.nameservers = args.nameserver
        dns.resolver.default_resolver.rotate = True

    items = set()

    # <item>
    if args.item:
        items |= set(args.item)

    # -f/--file
    items |= set(utils.file_items(args.file))

    if not items:
        utils.die('Provide some domain names, IP addresses, or CIDR ranges')

    # {ip: set(domain...)}
    by_ip = collections.defaultdict(set)
    # {ip: set(parent_domain...)}
    parent_domains = collections.defaultdict(set)

    def resolve_items(items):
        for ip, item, ports in utils.resolve_to_ips(items, tcp=not args.udp):
            # track parent items
            if ip != item and not utils.is_cidr(item):
                parent_domains[ip].add(item)

            # handle errors
            if isinstance(ip, Exception):
                exception = ip
                utils.bad('Failed to resolve {}: {}'.format(item, str(exception)))
                utils.debug_exception(exception)
                continue

            yield ip

    ips = resolve_items(items)

    # check to make sure domains resolved
    ips = utils.check_iterator(ips)
    if not ips:
        utils.die('No domains resolved. No IP addresses to perform recon on.')

    def thread_helper(ip):
        domain_note = ' ({})'.format(', '.join(parent_domains[ip])) if ip in parent_domains else ''
        utils.info('Performing Bing search for IP {}{}'.format(ip, domain_note))

        # -p/--proxy
        # -A/--user-agent
        # TODO pagination
        results = bing_search('ip:{}'.format(ip),
                proxy=args.proxy,
                user_agent=args.user_agent)

        return results

    total_success = 0
    total_results = 0
    for ip, results in utils.threadify(thread_helper, ips, max_threads=args.max_threads):
        if isinstance(results, Exception):
            # handle exception
            exception = results
            utils.bad('Bing search for IP {} failed: {}'.format(ip, str(exception)))
            utils.debug_exception(exception)
        else:
            result_domains = set()
            for result in results:
                parsed_href = urllib.parse.urlparse(result['href'])
                result_domain = parsed_href.netloc.lower().strip()
                result_domains.add(result_domain)

                total_results += 1

            # display results
            # -s/--short
            if args.short:
                # short result format
                # looks like: IP (DOMAIN): RESULTS...
                # or: IP (PARENT): Failed
                # or: IP (PARENT): No results

                out = ip
                if ip in parent_domains:
                    # add parent domain info
                    out += ' ({})'.format(', '.join(parent_domains[ip]))

                out += ': '

                if result_domains:
                    out += ' '.join(result_domains)
                elif result_domains is None:
                    out += 'Failed'
                else:
                    # not reached
                    out += 'No results'

                utils.log(out)
            # -d/--domains-only
            elif args.domains_only:
                # only output domains
                if result_domains:
                    for result in result_domains:
                        utils.log(result)
            else:
                # default/long result format
                # looks like:
                #
                #    Host: IP (DOMAIN)
                #    Results (COUNT):
                #     - RESULT
                #     - RESULT
                #     - ...
                #
                # or:
                #
                #    Host: IP (DOMAIN)
                #    Result: Failed
                #
                # or:
                #
                #    Host: IP (DOMAIN)
                #    Result: No results

                out = 'Host: {}'.format(ip)
                if ip in parent_domains:
                    # add parent domain info
                    out += ' ({})'.format(', '.join(parent_domains[ip]))
                out += '\n'

                if result_domains:
                    out += 'Results ({}):\n'.format(len(result_domains))
                    for result in result_domains:
                        out += ' - {}\n'.format(result)
                elif result_domains is None:
                    out += 'Result: Failed\n'
                else:
                    # not reached
                    out += 'Result: No results\n'

                utils.log(out)

            total_success += 1

    if total_results:
        # display results
        utils.good('Found {} results for {} queries'.format(total_results, total_success))
    else:
        # no results
        utils.bad('No results found')

if __name__ == '__main__':
    main()