From 12b47d367814d4d38ef3ca2cc3ae039f9ac96525 Mon Sep 17 00:00:00 2001 From: Quantra Date: Fri, 12 Aug 2016 18:45:56 +0100 Subject: [PATCH] Cache recent forts (for forts.max_circle_size) (#3556) * added bool option to cache recent forts -crf --forts.cache_recent_forts (default true) saves recent_forts in data/recent-forts-{username}.json on spin loads recent_forts from same file on start up bot doesn't start a new recent_forts on every reset * forgot contributor * typo fix no_cached_forts * changed all events related to caching forts to debug level * caching of forts happens on sigterm/exception handling of SIGTERM -Note handling of SIGTERM in python2.7 with multi threads is not reliable. Child thread can recieve SIGTERM and it is not handled in pokecli.py; pokecli.py continues to run. --- CONTRIBUTORS.md | 1 + configs/config.json.example | 3 +- pokecli.py | 44 ++++++++++++++++++++- pokemongo_bot/__init__.py | 52 +++++++++++++++++++++++++ pokemongo_bot/cell_workers/spin_fort.py | 3 ++ 5 files changed, 101 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 14a7d95514..1e463bb8d3 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -67,4 +67,5 @@ * simonsmh * joaodragao * extink + * Quantra diff --git a/configs/config.json.example b/configs/config.json.example index f306d3a920..838acc1d4b 100644 --- a/configs/config.json.example +++ b/configs/config.json.example @@ -101,7 +101,8 @@ "map_object_cache_time": 5, "forts": { "avoid_circles": true, - "max_circle_size": 50 + "max_circle_size": 50, + "cache_recent_forts": true }, "websocket_server": false, "walk": 4.16, diff --git a/pokecli.py b/pokecli.py index 4ccedb5fa6..ef50212d9f 100644 --- a/pokecli.py +++ b/pokecli.py @@ -33,6 +33,7 @@ import ssl import sys import time +import signal from datetime import timedelta from getpass import getpass from pgoapi.exceptions import NotLoggedInException, ServerSideRequestThrottlingException, ServerBusyOrOfflineException @@ -58,10 +59,16 @@ logger = logging.getLogger('cli') logger.setLevel(logging.INFO) +class SIGINTRecieved(Exception): pass + def main(): bot = False try: + def handle_sigint(*args): + raise SIGINTRecieved + signal.signal(signal.SIGINT, handle_sigint) + logger.info('PokemonGO Bot v1.0') sys.stdout = codecs.getwriter('utf8')(sys.stdout) sys.stderr = codecs.getwriter('utf8')(sys.stderr) @@ -95,7 +102,7 @@ def main(): while True: bot.tick() - except KeyboardInterrupt: + except (KeyboardInterrupt, SIGINTRecieved): bot.event_manager.emit( 'bot_exit', sender=bot, @@ -138,6 +145,32 @@ def main(): report_summary(bot) raise + finally: + # Cache here on SIGTERM, or Exception. Check data is available and worth caching. + if bot: + if bot.recent_forts[-1] is not None and bot.config.forts_cache_recent_forts: + cached_forts_path = os.path.join( + _base_dir, 'data', 'recent-forts-%s.json' % bot.config.username + ) + try: + with open(cached_forts_path, 'w') as outfile: + json.dump(bot.recent_forts, outfile) + bot.event_manager.emit( + 'cached_fort', + sender=bot, + level='debug', + formatted='Forts cached.', + ) + except IOError as e: + bot.event_manager.emit( + 'error_caching_forts', + sender=bot, + level='debug', + formatted='Error caching forts for {path}', + data={'path': cached_forts_path} + ) + + def report_summary(bot): if bot.metrics.start_time is None: @@ -362,6 +395,15 @@ def _json_loader(filename): type=int, default=10, ) + add_config( + parser, + load, + short_flag="-crf", + long_flag="--forts.cache_recent_forts", + help="Caches recent forts used by max_circle_size", + type=bool, + default=True, + ) add_config( parser, load, diff --git a/pokemongo_bot/__init__.py b/pokemongo_bot/__init__.py index 7d8a207ece..dcf83bdc84 100644 --- a/pokemongo_bot/__init__.py +++ b/pokemongo_bot/__init__.py @@ -84,6 +84,7 @@ def start(self): self._setup_event_system() self._setup_logging() self._setup_api() + self._load_recent_forts() random.seed() @@ -456,6 +457,18 @@ def _register_events(self): parameters=('last_lat', 'last_lon') ) + # cached recent_forts + self.event_manager.register_event('loaded_cached_forts') + self.event_manager.register_event('cached_fort') + self.event_manager.register_event( + 'no_cached_forts', + parameters=('path', ) + ) + self.event_manager.register_event( + 'error_caching_forts', + parameters=('path', ) + ) + def tick(self): self.health_record.heartbeat() self.cell = self.get_meta_cell() @@ -1093,3 +1106,42 @@ def get_map_objects(self, lat, lng, timestamp, cellid): self.last_time_map_object = time.time() return self.last_map_object + + def _load_recent_forts(self): + if not self.config.forts_cache_recent_forts: + return + + + cached_forts_path = os.path.join(_base_dir, 'data', 'recent-forts-%s.json' % self.config.username) + try: + # load the cached recent forts + with open(cached_forts_path) as f: + cached_recent_forts = json.load(f) + + num_cached_recent_forts = len(cached_recent_forts) + num_recent_forts = len(self.recent_forts) + + # Handles changes in max_circle_size + if not num_recent_forts: + self.recent_forts = [] + elif num_recent_forts > num_cached_recent_forts: + self.recent_forts[-num_cached_recent_forts:] = cached_recent_forts + elif num_recent_forts < num_cached_recent_forts: + self.recent_forts = cached_recent_forts[-num_recent_forts:] + else: + self.recent_forts = cached_recent_forts + + self.event_manager.emit( + 'loaded_cached_forts', + sender=self, + level='debug', + formatted='Loaded cached forts...' + ) + except IOError: + self.event_manager.emit( + 'no_cached_forts', + sender=self, + level='debug', + formatted='Starting new cached forts for {path}', + data={'path': cached_forts_path} + ) diff --git a/pokemongo_bot/cell_workers/spin_fort.py b/pokemongo_bot/cell_workers/spin_fort.py index 61d3eb02bd..0ba09ca633 100644 --- a/pokemongo_bot/cell_workers/spin_fort.py +++ b/pokemongo_bot/cell_workers/spin_fort.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +import json +import os import time from pgoapi.utilities import f2i @@ -9,6 +11,7 @@ from pokemongo_bot.human_behaviour import sleep from pokemongo_bot.worker_result import WorkerResult from pokemongo_bot.base_task import BaseTask +from pokemongo_bot.base_dir import _base_dir from utils import distance, format_time, fort_details