fix early cache and safe exit limit bugs

CHANGES.md

@@ -1,4 +1,9 @@
-## 1.0.2 (2021/02/11)
+## 1.0.3 (2021/02/19)
+
+- fixed early cache bug
+- fixed limit being retrieved from next search window when resuming from safe exit
+
+## 1.0.2 (2021/02/16)
 
 - fixed comments returning 25 by default
 

pmaw/PushshiftAPIBase.py

@@ -114,11 +114,11 @@ def _multithread(self, check_total=False):
                     self._shutdown(executor)
                     raise RuntimeError(
                         shards_down_message + f' {len(self.req.req_list)} unfinished requests.')
-            if self.num_batches % self.file_checkpoint == 0:
-                # cache current results
-                executor.submit(self.req.save_cache())
             if not check_total:
                 self.num_batches += 1
+                if self.num_batches % self.file_checkpoint == 0:
+                    # cache current results
+                    executor.submit(self.req.save_cache())
                 self._print_stats('Checkpoint')
             else:
                 break
@@ -235,7 +235,7 @@ def _search(self,
 
         while (self.req.limit is None or self.req.limit > 0) and not self.req.exit.is_set():
             # set/update limit
-            if 'ids' not in self.req.payload:
+            if 'ids' not in self.req.payload and len(self.req.req_list) == 0:
                 # check to see how many results are remaining
                 self.req.req_list.appendleft((url, self.req.payload))
                 self._multithread(check_total=True)

pmaw/__init__.py

@@ -5,7 +5,7 @@
 """
 PMAW: Pushshift Multithread API Wrapper
 """
-__version__ = '1.0.2'
+__version__ = '1.0.3'
 __author__ = 'Matthew Podolak'
 __license__ = 'MIT'