Cache recent forts (for forts.max_circle_size) (#3556)

* added bool option to cache recent forts -crf --forts.cache_recent_forts (default true)

saves recent_forts in data/recent-forts-{username}.json on spin

loads recent_forts from same file on start up

bot doesn't start a new recent_forts on every reset

* forgot contributor

* typo fix no_cached_forts

* changed all events related to caching forts to debug level

* caching of forts happens on sigterm/exception

handling of SIGTERM
-Note handling of SIGTERM in python2.7 with multi threads is not reliable.  Child thread can recieve SIGTERM and it is not handled in pokecli.py; pokecli.py continues to run.

CONTRIBUTORS.md

@@ -67,4 +67,5 @@
  * simonsmh
  * joaodragao
  * extink
+ * Quantra
 

configs/config.json.example

@@ -101,7 +101,8 @@
     "map_object_cache_time": 5,
     "forts": {
       "avoid_circles": true,
-      "max_circle_size": 50
+      "max_circle_size": 50,
+      "cache_recent_forts": true
     },
     "websocket_server": false,
     "walk": 4.16,

pokecli.py

@@ -33,6 +33,7 @@
 import ssl
 import sys
 import time
+import signal
 from datetime import timedelta
 from getpass import getpass
 from pgoapi.exceptions import NotLoggedInException, ServerSideRequestThrottlingException, ServerBusyOrOfflineException
@@ -58,10 +59,16 @@
 logger = logging.getLogger('cli')
 logger.setLevel(logging.INFO)
 
+class SIGINTRecieved(Exception): pass
+
 def main():
     bot = False
 
     try:
+        def handle_sigint(*args):
+            raise SIGINTRecieved
+        signal.signal(signal.SIGINT, handle_sigint)
+
         logger.info('PokemonGO Bot v1.0')
         sys.stdout = codecs.getwriter('utf8')(sys.stdout)
         sys.stderr = codecs.getwriter('utf8')(sys.stderr)
@@ -95,7 +102,7 @@ def main():
                 while True:
                     bot.tick()
 
-            except KeyboardInterrupt:
+            except (KeyboardInterrupt, SIGINTRecieved):
                 bot.event_manager.emit(
                     'bot_exit',
                     sender=bot,
@@ -138,6 +145,32 @@ def main():
             report_summary(bot)
 
         raise
+    finally:
+        # Cache here on SIGTERM, or Exception.  Check data is available and worth caching.
+        if bot:
+            if bot.recent_forts[-1] is not None and bot.config.forts_cache_recent_forts:
+                cached_forts_path = os.path.join(
+                    _base_dir, 'data', 'recent-forts-%s.json' % bot.config.username
+                )
+                try:
+                    with open(cached_forts_path, 'w') as outfile:
+                        json.dump(bot.recent_forts, outfile)
+                    bot.event_manager.emit(
+                        'cached_fort',
+                        sender=bot,
+                        level='debug',
+                        formatted='Forts cached.',
+                    )
+                except IOError as e:
+                    bot.event_manager.emit(
+                        'error_caching_forts',
+                        sender=bot,
+                        level='debug',
+                        formatted='Error caching forts for {path}',
+                        data={'path': cached_forts_path}
+                        )
+
+
 
 def report_summary(bot):
     if bot.metrics.start_time is None:
@@ -362,6 +395,15 @@ def _json_loader(filename):
         type=int,
         default=10,
     )
+    add_config(
+        parser,
+        load,
+        short_flag="-crf",
+        long_flag="--forts.cache_recent_forts",
+        help="Caches recent forts used by max_circle_size",
+        type=bool,
+        default=True,
+    )
     add_config(
         parser,
         load,

pokemongo_bot/__init__.py

@@ -84,6 +84,7 @@ def start(self):
         self._setup_event_system()
         self._setup_logging()
         self._setup_api()
+        self._load_recent_forts()
 
         random.seed()
 
@@ -456,6 +457,18 @@ def _register_events(self):
             parameters=('last_lat', 'last_lon')
         )
 
+        # cached recent_forts
+        self.event_manager.register_event('loaded_cached_forts')
+        self.event_manager.register_event('cached_fort')
+        self.event_manager.register_event(
+            'no_cached_forts',
+            parameters=('path', )
+        )
+        self.event_manager.register_event(
+            'error_caching_forts',
+            parameters=('path', )
+        )
+
     def tick(self):
         self.health_record.heartbeat()
         self.cell = self.get_meta_cell()
@@ -1093,3 +1106,42 @@ def get_map_objects(self, lat, lng, timestamp, cellid):
         self.last_time_map_object = time.time()
 
         return self.last_map_object
+
+    def _load_recent_forts(self):
+        if not self.config.forts_cache_recent_forts:
+            return
+
+
+        cached_forts_path = os.path.join(_base_dir, 'data', 'recent-forts-%s.json' % self.config.username)
+        try:
+            # load the cached recent forts
+            with open(cached_forts_path) as f:
+                cached_recent_forts = json.load(f)
+
+            num_cached_recent_forts = len(cached_recent_forts)
+            num_recent_forts = len(self.recent_forts)
+
+            # Handles changes in max_circle_size
+            if not num_recent_forts:
+                self.recent_forts = []
+            elif num_recent_forts > num_cached_recent_forts:
+                self.recent_forts[-num_cached_recent_forts:] = cached_recent_forts
+            elif num_recent_forts < num_cached_recent_forts:
+                self.recent_forts = cached_recent_forts[-num_recent_forts:]
+            else:
+                self.recent_forts = cached_recent_forts
+
+            self.event_manager.emit(
+                'loaded_cached_forts',
+                sender=self,
+                level='debug',
+                formatted='Loaded cached forts...'
+            )
+        except IOError:
+            self.event_manager.emit(
+                'no_cached_forts',
+                sender=self,
+                level='debug',
+                formatted='Starting new cached forts for {path}',
+                data={'path': cached_forts_path}
+            )

pokemongo_bot/cell_workers/spin_fort.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import json
+import os
 import time
 
 from pgoapi.utilities import f2i
@@ -9,6 +11,7 @@
 from pokemongo_bot.human_behaviour import sleep
 from pokemongo_bot.worker_result import WorkerResult
 from pokemongo_bot.base_task import BaseTask
+from pokemongo_bot.base_dir import _base_dir
 from utils import distance, format_time, fort_details