Merge pull request #1 from alairock/master

Adding a docker compose, for quick database creation/configuration

.gitignore

@@ -2,6 +2,7 @@
 
 # Configuration
 conf*.yml
+conf*.yaml
 
 # Logging
 *.log
@@ -13,3 +14,6 @@ log.log.*
 /lib/
 /pip-selfcheck.json
 /share/
+instabotenv/
+db_data/
+

docker-compose.yaml

@@ -0,0 +1,13 @@
+mysql:
+  image: mysql
+  ports:
+    - "3306:3306"
+  expose:
+     - "3306"
+  environment:
+    - MYSQL_ROOT_PASSWORD=instabotisgrate
+    - MYSQL_DATABASE=instagram
+    - MYSQL_USER=instabot
+    - MYSQL_PASSWORD=GT8H!b]5,9}A7
+  volumes:
+    - ./db_data:/var/lib/mysql

instabot/db.py

@@ -1,6 +1,5 @@
 import logging
 import sys
-from .errors import DBError
 from instabot import user
 from peewee import *
 from playhouse.shortcuts import RetryOperationalError
@@ -9,18 +8,21 @@
 
 
 class RetryingMySQLDatabase(RetryOperationalError, MySQLDatabase):
-    '''
+    """
     Automatically reconnecting database class.
     @see {@link
-    http://docs.peewee-orm.com/en/latest/peewee/database.html#automatic-reconnect}
-    '''
-    pass
+    http://docs.peewee-orm.com/en/latest/peewee/database.html#automatic
+    -reconnect}
+    """
+
+    def sequence_exists(self, seq):
+        pass
 
 
 def get_db(configuration):
-    '''
+    """
     @raise DBError
-    '''
+    """
     db = RetryingMySQLDatabase(
         configuration.db_name,
         host=configuration.db_host,

instabot/following_service.py

@@ -34,15 +34,15 @@ async def run(self):
                 await asyncio.sleep(10)
 
     async def _follow(self):
-        '''
+        """
         @raise APIError
         @raise APIJSONError
         @raise APILimitError
-        '''
+        """
         unfollowing_threshold = datetime.datetime.utcnow() - \
             self._following_timedelta
         for user in User.select().where(
-                User.was_followed_at == None,
+                        User.was_followed_at is None,
                 ).order_by(User.following_depth, User.created):
             try:
                 await self._client.follow(user)
@@ -59,15 +59,15 @@ async def _follow(self):
             user.save()
 
     async def _unfollow(self):
-        '''
+        """
         @raise APIError
         @raise APIJSONError
         @raise APILimitError
-        '''
+        """
         unfollowing_threshold = datetime.datetime.utcnow() - \
             self._following_timedelta
         for user in User.select().where(
-                (User.is_followed == True) &
+                (User.is_followed is True) &
                 (User.was_followed_at <= unfollowing_threshold),
                 ):
             try:

instabot/instabot.py

@@ -2,7 +2,6 @@
 import datetime
 import logging
 import logging.config
-import sys
 from .configuration import Configuration
 from .db import get_db
 from .errors import ConfigurationError

instabot/instagram.py

@@ -1,7 +1,9 @@
 import asyncio
 import logging
 import json
-from .errors import APIError, APIJSONError, APILimitError, \
+import random
+
+from .errors import APIError, APILimitError, \
     APINotAllowedError, APINotFoundError
 from aiohttp import ClientSession
 
@@ -43,7 +45,7 @@ def __init__(self, configuration):
         loop.run_until_complete(self._do_login())
 
     async def _ajax(self, url, data=None, referer=None):
-        '''Simulates AJAX request.
+        """Simulates AJAX request.
 
         Args:
             url (str): URL path. e.g.: 'query/'
@@ -57,7 +59,7 @@ async def _ajax(self, url, data=None, referer=None):
             APINotAllowedError
             APINotFoundError
 
-        '''
+        """
         if referer is not None:
             self._referer = referer
         url = BASE_URL + url
@@ -114,12 +116,12 @@ async def _ajax(self, url, data=None, referer=None):
         return response_json
 
     async def _do_login(self):
-        '''
+        """
         @raise APIJSONError
         @raise APILimitError
         @raise APINotAllowedError
         @raise APIError
-        '''
+        """
         await self._open(BASE_URL)
         self._update_csrf_token()
         await self._ajax(
@@ -133,13 +135,13 @@ async def _do_login(self):
         self.id = self._session.cookies['ds_user_id'].value
 
     async def follow(self, user):
-        '''
+        """
         @raise APIJSONError
         @raise APILimitError
         @raise APINotAllowedError
         @raise APINotFoundError
         @raise APIError
-        '''
+        """
         try:
             await self._ajax(
                 'web/friendships/{}/follow/'.format(user.instagram_id),
@@ -159,7 +161,7 @@ async def follow(self, user):
             LOGGER.debug('{} was followed'.format(user.username))
 
     async def get_followed(self, user):
-        '''Fetches information about people followed by given user.
+        """Fetches information about people followed by given user.
 
         Args:
             user (User): Whose subscriptions should be fetched.
@@ -177,8 +179,8 @@ async def get_followed(self, user):
             APINotAllowedError
             APIError
 
-        '''
-        SINGLE_RESPONSE_SIZE = 50
+        """
+        single_response_size = 50
 
         response = await self._ajax(
             'query/',
@@ -191,7 +193,7 @@ async def get_followed(self, user):
                 '      username    }}  }}}}'
                 .format(
                     id=user.instagram_id,
-                    count=SINGLE_RESPONSE_SIZE,
+                    count=single_response_size,
                     ),
                 'ref': 'relationships::follow_list',
                 },
@@ -212,7 +214,7 @@ async def get_followed(self, user):
                     .format(
                         id=user.instagram_id,
                         end_cursor=end_cursor,
-                        count=SINGLE_RESPONSE_SIZE,
+                        count=single_response_size,
                         ),
                     'ref': 'relationships::follow_list',
                     },
@@ -222,8 +224,58 @@ async def get_followed(self, user):
         LOGGER.debug('{} followed users were fetched'.format(len(followed)))
         return followed
 
+    async def get_following_page(self, user, cursor=None):
+        """
+        Args:
+            user (User): User whose followers should be fetched
+            cursor: The next page to retrieve, if possible.
+        :param user:
+        :param cursor:
+        :return:
+        """
+        q = 'ig_user(%s) {' % user.instagram_id,
+        qcursor = ' followed_by.first(20) {'
+        if cursor is not None:
+            qcursor = ' followed_by.after(%s, 20) {' % cursor
+        q += qcursor + '''
+            count,
+            page_info {
+              end_cursor,
+              has_next_page
+            },
+            nodes {
+                id,
+                is_verified,
+                followed_by {count},
+                follows {count},
+                followed_by_viewer,
+                follows_viewer,
+                requested_by_viewer,
+                full_name,
+                profile_pic_url,
+                username
+                }
+            }
+        }
+        '''
+        data = {'q': q, 'ref': 'relationships::follow_list'}
+        response = await self._ajax('query/', data, referer=user.get_url())
+        try:
+            followers = response['followed_by']['nodes']
+            page_info = response['followed_by']['page_info']
+        except (KeyError, TypeError) as e:
+            raise APINotAllowedError(
+                'Instagram have given unexpected data in '
+                '`get_some_followers`. Response JSON: {response} '
+                'Error: {error}'.format(
+                    response=response,
+                    error=e,
+                )
+            )
+        return followers, page_info['end_cursor'], page_info['has_next_page']
+
     async def get_some_followers(self, user):
-        '''Fetches some amount of followers of given user.
+        """Fetches some amount of followers of given user.
 
         Args:
             user (User): Whose followers should be fetched.
@@ -241,48 +293,33 @@ async def get_some_followers(self, user):
             APINotAllowedError
             APIError
 
-        '''
-        SINGLE_RESPONSE_SIZE = 50
-
-        response = await self._ajax(
-            'query/',
-            {
-                'q': 'ig_user({id}) {{  followed_by.first({count}) {{'
-                '    count,    page_info {{      end_cursor,'
-                '      has_next_page    }},    nodes {{      id,'
-                '      is_verified,      followed_by_viewer,'
-                '      requested_by_viewer,      full_name,'
-                '      profile_pic_url,      username    }}  }}}}'
-                .format(
-                    id=user.instagram_id,
-                    count=SINGLE_RESPONSE_SIZE,
-                    ),
-                'ref': 'relationships::follow_list',
-                },
-            referer=user.get_url(),
-            )
-        try:
-            followers = response['followed_by']['nodes']
-        except (KeyError, TypeError) as e:
-            raise APINotAllowedError(
-                'Instagram have given unexpected data in '
-                '`get_some_followers`. Response JSON: {response} '
-                'Error: {error}'
-                .format(
-                    response=response,
-                    error=e,
-                    )
-                )
+        """
+        page_limit = 3
+        followers = []
+        get_next = True
+        cursor = None  # Eventually we will check if we have a
+        # cached page and use that.
+        LOGGER.debug('Fetching followers of {}'.format(user.username))
+        while get_next and page_limit > 0:
+            next_followers, cursor, get_next = await self.get_following_page(
+                user=user,
+                cursor=cursor)
+            followers = followers + next_followers
+            page_limit -= 1
+            await asyncio.sleep(random.randint(1, 5))
+        # TODO: Cache cursor for continuation of this, if needed.
+        LOGGER.debug('Fetched {} followers of {}'
+                     .format(20 * page_limit, user.username))
         return followers
 
     async def like(self, media):
-        '''
+        """
         @raise APIError
         @raise APIJSONError
         @raise APILimitError
         @raise APINotAllowedError
         @raise APINotFoundError
-        '''
+        """
         try:
             await self._ajax('web/likes/{}/like/'.format(media))
         except APILimitError as e:
@@ -324,15 +361,15 @@ async def _sleep_success(self):
             self._success_sleep_time_coefficient
 
     async def unfollow(self, user):
-        '''
+        """
         @raise APIError
         @raise APIJSONError
         @raise APILimitError
         @raise APINotAllowedError
         @raise APINotFoundError
-        '''
+        """
         try:
-            response = await self._ajax(
+            await self._ajax(
                 'web/friendships/{}/unfollow/'.format(user.instagram_id),
                 referer=user.get_url(),
                 )

instabot/stats_service.py

@@ -20,6 +20,8 @@ def report(self, prefix):
 
 
 class StatsService:
+    _instance = None
+
     def __init__(self):
         self._hourly_counter = Counter()
         self._daily_counter = Counter()

instabot/user_service.py

@@ -36,13 +36,13 @@ async def run(self):
 
     async def _ensure_enough_users(self):
         users_to_follow_count = User.select() \
-            .where(User.was_followed_at == None) \
+            .where(User.was_followed_at is None) \
             .count()
         LOGGER.debug('{} users to follow found'.format(users_to_follow_count))
         if users_to_follow_count < self._users_to_follow_cache_size:
             last_users_to_follow_count = users_to_follow_count
             for user in User.select() \
-                .where(User.were_followers_fetched == False) \
+                .where(User.were_followers_fetched is False) \
                 .order_by(
                     User.following_depth,
                     User.created,