add new graph real time. Improve harvester

backend/backend.py

@@ -210,7 +210,9 @@ def sentiment_weather_queries_endpoint():
         if resource == AVG_MONTHLY_ANALYSIS:
             return jsonify({'result': get_averaged_by_month(es)}), 200
         if resource == UPDATING_ANALYSIS:
-            return jsonify({'result': get_recent_averaged_by_daily(es)}), 200
+            return jsonify({'result': get_recent_averaged_sentiment_by_hourly(es)}), 200
+        if resource == MESSAGE_COUNTS_HOURLY:
+            return jsonify({'result': get_recent_total_sentiment_by_hourly(es)}), 200
         else:
             return jsonify({'Resource in headers is not valid': resource}), 400
 

backend/harvesters/Mastodon/mharvester.py

@@ -90,7 +90,8 @@ def ingest(recents_only = True, max_id = None):
         since_date = datetime.now().replace(tzinfo=utc) - timedelta(minutes=5)
         if len(doc)>0:
             val = doc[0]
-            since_id = val['_id']
+            print(val)
+            since_id = val['_source']['id']
     else:
         print('backfilling old')
         since_date = datetime.now().replace(tzinfo=utc) - timedelta(days=5) # a random post on 15/05/2024

backend/querying/query_path_constants.py

@@ -4,4 +4,5 @@
 FOR_STATISTICAL_ANALYSIS = 'stat_analysis'
 FOR_SPATIAL_ANALYSIS = 'spatial_analysis'
 AVG_MONTHLY_ANALYSIS = 'avg_monthly_analysis'
-UPDATING_ANALYSIS = 'updateing_analysis'
+UPDATING_ANALYSIS = 'updateing_analysis'
+MESSAGE_COUNTS_HOURLY = 'message_counts_hourly'

backend/querying/sentiment_weather_queries.py

@@ -5,7 +5,7 @@
 from querying.make_query import make_query
 import numpy as np
 
-def get_recent_averaged_by_daily(es):
+def get_recent_averaged_sentiment_by_hourly(es):
     data_start = datetime(2024, 5, 17, tzinfo=UTC)
     joined = {}
 
@@ -26,7 +26,7 @@ def get_recent_averaged_by_daily(es):
     query = f"""    
         SELECT    
             HISTOGRAM(created_at, INTERVAL 1 HOUR) as hour,
-            SUM(sentiment) as avg_sentiment
+            SUM(sentiment) as sum_sentiment
         FROM {MASTODON}
         WHERE sentiment <> 0.0 
         GROUP BY hour 
@@ -38,7 +38,44 @@ def get_recent_averaged_by_daily(es):
                 joined[hour][2] = avg_sent
 
     rows = [[hour]+stats for hour, stats in joined.items()]
-    columns = ['hour', 'average air temp', 'total rainfall', 'average sentiment']
+    columns = ['hour', 'average air temp', 'total rainfall', 'sum sentiment']
+    return {'columns': columns, 'rows': rows }
+
+
+def get_recent_total_sentiment_by_hourly(es):
+    data_start = datetime(2024, 5, 17, tzinfo=UTC)
+    joined = {}
+
+    query = f"""    
+        SELECT    
+            HISTOGRAM(DATETIME_PARSE(aifstime_utc,'yyyyMMddHHmmss'), INTERVAL 1 HOUR) as hour,
+                AVG(air_temp) as air_temp,
+                SUM(rain_trace) as total_rain
+        
+        FROM {BOM_OBSERVATIONS}
+        GROUP BY hour
+    """
+    weather_rows = make_query(es, query)['rows']
+    for hour, avg_temp, total_rain in weather_rows:
+        if datetime.strptime(hour, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=UTC) >=  data_start:
+            joined[hour] = joined.get(hour, [avg_temp, total_rain, 0])
+
+    query = f"""    
+        SELECT    
+            HISTOGRAM(created_at, INTERVAL 1 HOUR) as hour,
+            SUM(1) as message_count
+        FROM {MASTODON}
+        WHERE sentiment <> 0.0 
+        GROUP BY hour 
+    """
+    sentiment_rows = make_query(es, query)['rows']
+    for hour, avg_sent in sentiment_rows:
+        if datetime.strptime(hour, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=UTC) >=  data_start:  
+            if hour in joined.keys():
+                joined[hour][2] = avg_sent
+
+    rows = [[hour]+stats for hour, stats in joined.items()]
+    columns = ['hour', 'average air temp', 'total rainfall', 'message counts']
     return {'columns': columns, 'rows': rows }