Define new conjunctivitis signal in google-symptoms (#2112)

* define new conjunctivitis signal * add s07 to params * update test data * change behavior to only log missing set * lint and clean test --------- Co-authored-by: Amaris Sim <[email protected]>
cmu-delphi · Jan 29, 2025 · 51620ea · 51620ea
1 parent d407d6c
commit 51620ea
Show file tree

Hide file tree

Showing 10 changed files with 27,790 additions and 27,778 deletions.
diff --git a/ansible/templates/google_symptoms-params-prod.json.j2 b/ansible/templates/google_symptoms-params-prod.json.j2
@@ -52,6 +52,7 @@
         "s04_smoothed_search",
         "s05_smoothed_search",
         "s06_smoothed_search",
+        "s07_smoothed_search",
         "scontrol_smoothed_search"
       ]
     }

diff --git a/google_symptoms/delphi_google_symptoms/constants.py b/google_symptoms/delphi_google_symptoms/constants.py
@@ -11,14 +11,11 @@
     "s02": ["Nasal congestion", "Post nasal drip", "Rhinorrhea", "Sinusitis",
             "Rhinitis", "Common cold"],
     "s03": ["Fever", "Hyperthermia", "Chills", "Shivering", "Low grade fever"],
-    #"s04": ["Fatigue", "Weakness", "Muscle weakness", "Myalgia", "Pain"],
     "s04": ["Shortness of breath", "Wheeze", "Croup", "Pneumonia", "Asthma",
             "Crackles", "Acute bronchitis", "Bronchitis"],
     "s05": ["Anosmia", "Dysgeusia", "Ageusia"],
-    #"s07": ["Nausea", "Vomiting", "Diarrhea", "Indigestion", "Abdominal pain"],
     "s06": ["Laryngitis", "Sore throat", "Throat irritation"],
-    #"s09": ["Headache", "Migraine", "Cluster headache", "Dizziness", "Lightheadedness"],
-    #"s10": ["Night sweats","Perspiration", "hyperhidrosis"],
+    "s07": ["Conjunctivitis", "Red eye", "Epiphora", "Eye pain", "Rheum"],
     "scontrol": ["Type 2 diabetes", "Urinary tract infection", "Hair loss",
                  "Candidiasis", "Weight gain"]
 }

diff --git a/google_symptoms/delphi_google_symptoms/date_utils.py b/google_symptoms/delphi_google_symptoms/date_utils.py
@@ -8,7 +8,7 @@
 from delphi_utils.validator.utils import lag_converter
 from pandas import to_datetime
 
-from .constants import COMBINED_METRIC, FULL_BKFILL_START_DATE, PAD_DAYS, SMOOTHERS
+from .constants import COMBINED_METRIC, PAD_DAYS, SMOOTHERS
 
 
 def generate_patch_dates(params: Dict) -> Dict[date, Dict[str, Union[date, int]]]:
@@ -85,22 +85,20 @@ def generate_num_export_days(params: Dict, logger) -> [int]:
 
         if sensor_names.difference(set(gs_metadata.signal)):
             # If any signal not in metadata yet, we need to backfill its full history.
-            logger.warning("Signals missing in the epidata; backfilling full history")
-            num_export_days = (export_end_date - FULL_BKFILL_START_DATE).days + 1
-        else:
-            latest_date_diff = (datetime.today() - to_datetime(min(gs_metadata.max_time))).days + 1
+            logger.warning("Signals missing in the epidata; need to patch full history")
+        latest_date_diff = (datetime.today() - to_datetime(min(gs_metadata.max_time))).days + 1
 
-            expected_date_diff = params["validation"]["common"].get("span_length", 14)
+        expected_date_diff = params["validation"]["common"].get("span_length", 14)
 
-            # there's an expected lag of 4 days behind if running from today
-            if export_end_date.date() == datetime.today().date():
-                global_max_expected_lag = get_max_lag(params)
-                expected_date_diff += global_max_expected_lag
+        # there's an expected lag of 4 days behind if running from today
+        if export_end_date.date() == datetime.today().date():
+            global_max_expected_lag = get_max_lag(params)
+            expected_date_diff += global_max_expected_lag
 
-            if latest_date_diff > expected_date_diff:
-                logger.info("Lag is more than expected", expected_lag=expected_date_diff, lag=latest_date_diff)
+        if latest_date_diff > expected_date_diff:
+            logger.info("Lag is more than expected", expected_lag=expected_date_diff, lag=latest_date_diff)
 
-            num_export_days = expected_date_diff
+        num_export_days = expected_date_diff
 
     return num_export_days
 

diff --git a/google_symptoms/params.json.template b/google_symptoms/params.json.template
@@ -41,6 +41,7 @@
         "s04_smoothed_search",
         "s05_smoothed_search",
         "s06_smoothed_search",
+        "s07_smoothed_search",
         "scontrol_smoothed_search"
       ]
     }

diff --git a/google_symptoms/tests/test_data/covid_metadata.csv b/google_symptoms/tests/test_data/covid_metadata.csv
@@ -1955,6 +1955,18 @@ google-symptoms,s06_smoothed_search,day,hrr,2020-02-20,2024-07-13,304,0.0,2.4442
 google-symptoms,s06_smoothed_search,day,msa,2020-02-20,2024-07-13,379,0.0,2.942381,0.6269093,0.2474402,2024-07-17 13:15:36,20240717,4,732
 google-symptoms,s06_smoothed_search,day,nation,2020-02-20,2024-07-13,1,0.4007222,1.5691818,0.8043518,0.2252896,2024-07-17 13:15:36,20240717,4,732
 google-symptoms,s06_smoothed_search,day,state,2020-02-20,2024-07-13,51,0.1852381,2.0328571,0.7772562,0.2522011,2024-07-17 13:15:36,20240717,4,732
+google-symptoms,s07_raw_search,day,county,2020-02-14,2024-07-13,869,0.0733333,3.8166667,0.7171405,0.2509621,2024-07-17 13:15:32,20240717,4,738
+google-symptoms,s07_raw_search,day,hhs,2020-02-14,2024-07-13,10,0.3065122,1.9331217,0.7831431,0.2417782,2024-07-17 13:15:33,20240717,4,738
+google-symptoms,s07_raw_search,day,hrr,2020-02-14,2024-07-13,304,8.36e-05,2.6907692,0.5563906,0.2612786,2024-07-17 13:15:34,20240717,4,738
+google-symptoms,s07_raw_search,day,msa,2020-02-14,2024-07-13,379,0.0698392,3.6766667,0.6513573,0.2451345,2024-07-17 13:15:36,20240717,4,738
+google-symptoms,s07_raw_search,day,nation,2020-02-14,2024-07-13,1,0.3923812,1.7562711,0.8039715,0.2280758,2024-07-17 13:15:36,20240717,4,738
+google-symptoms,s07_raw_search,day,state,2020-02-14,2024-07-13,51,0.1533333,2.2033333,0.7768827,0.2577945,2024-07-17 13:15:36,20240717,4,738
+google-symptoms,s07_smoothed_search,day,county,2020-02-20,2024-07-13,869,0.0,2.942381,0.6715739,0.2519801,2024-07-17 13:15:32,20240717,4,732
+google-symptoms,s07_smoothed_search,day,hhs,2020-02-20,2024-07-13,10,0.3288689,1.804973,0.783512,0.2387222,2024-07-17 13:15:33,20240717,4,732
+google-symptoms,s07_smoothed_search,day,hrr,2020-02-20,2024-07-13,304,0.0,2.4442007,0.5435988,0.263496,2024-07-17 13:15:34,20240717,4,732
+google-symptoms,s07_smoothed_search,day,msa,2020-02-20,2024-07-13,379,0.0,2.942381,0.6269093,0.2474402,2024-07-17 13:15:36,20240717,4,732
+google-symptoms,s07_smoothed_search,day,nation,2020-02-20,2024-07-13,1,0.4007222,1.5691818,0.8043518,0.2252896,2024-07-17 13:15:36,20240717,4,732
+google-symptoms,s07_smoothed_search,day,state,2020-02-20,2024-07-13,51,0.1852381,2.0328571,0.7772562,0.2522011,2024-07-17 13:15:36,20240717,4,732
 google-symptoms,scontrol_raw_search,day,county,2020-02-14,2024-07-13,1438,0.25,14.124,3.3171389,1.0021697,2024-07-17 13:15:32,20240717,4,738
 google-symptoms,scontrol_raw_search,day,hhs,2020-02-14,2024-07-13,10,2.097142,10.5574026,3.561163,0.4858404,2024-07-17 13:15:33,20240717,4,738
 google-symptoms,scontrol_raw_search,day,hrr,2020-02-14,2024-07-13,306,0.0535164,12.348618,3.1551263,0.8088133,2024-07-17 13:15:34,20240717,4,738