Only filter out already-present logs if they are up-to-date

Logs may be updated. The log index includes the time the log was last updated in its response. Using it, we can re-parse logs which have been updated since we last imported them. To do this, we add an optional second argument to the logid iterator. Then, in filter_logs, we also yield updated logs. Closes: #30 Signed-off-by: Sean Anderson <[email protected]>
Forty-Bot · Aug 27, 2021 · 8fb8c59 · 8fb8c59
1 parent 0a2f7a6
commit 8fb8c59
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 4 deletions.
diff --git a/fetch.py b/fetch.py
@@ -51,7 +51,7 @@ def fetch_players_logids(s, players=None, since=0, count=None, offset=0, limit=1
 
             for log in log_list['logs']:
                 if log['date'] >= since:
-                    yield log['id']
+                    yield log['id'], log['date']
                 else:
                     # Don't fetch any more pages
                     count = fetched

diff --git a/import.py b/import.py
@@ -26,10 +26,21 @@ def filter_logids(c, logids):
     """
 
     for logid in logids:
+        try:
+            time = logid[1]
+            logid = logid[0]
+        except TypeError:
+            time = None
+
         cur = c.cursor()
-        cur.execute("SELECT 1 FROM public.log WHERE logid=%s", (logid,))
-        for _ in cur:
-            break
+        cur.execute("""SELECT
+                           time < %s
+                       FROM public.log
+                       WHERE logid = %s""", (time, logid))
+
+        for row in cur:
+            if row[0]:
+                yield logid
         else:
             yield logid