Bugfix/reducing down mem foot print (#54)

* Experimental Changes -- this should help clean up stuff that is no longer needed on each loop run. * Update PurpleAirDataLogger.py - another small tweak. Cleaning up old data * Update PurpleAirDataLogger.py -- more smaller tweaks that seem to be working * More Deletes -- more clean up. maybe it'll help. * Update PurpleAirDataLogger.py -- a few more tweaks that may help maybe. * prepared PSQL statements -- moving to using prepare psql statements for the store sensor method. * Update setup.py - bumping pg8000 version * Black - added black to vscode settings file - ran black * Update setup.py - Making an alpha release with the memory foot print fixes. Testing has show on average it is now using 100MBs less than 1.2.0.
carlkidcrypto · May 18, 2023 · cc7da06 · cc7da06
1 parent 679bddf
commit cc7da06
Show file tree

Hide file tree

Showing 4 changed files with 74 additions and 22 deletions.
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -15,5 +15,6 @@
         "timescaledb",
         "PSQL",
         "deciviews"
-    ]
+    ],
+    "python.formatting.provider": "black"
 }
diff --git a/purpleair_data_logger/PurpleAirDataLogger.py b/purpleair_data_logger/PurpleAirDataLogger.py
@@ -165,6 +165,9 @@ def _validate_sensor_data_before_insert(self, the_modified_sensor_data):
                     field
                 ]
 
+        # Delete some stuff
+        del the_modified_sensor_data
+
         # Then return the modified copy
         return temp_the_modified_sensor_data
 
@@ -185,6 +188,7 @@ def _run_loop_for_storing_single_sensor_data(self, the_json_file):
                       {the_json_file['sensor_index']}..."""
             )
 
+            sensor_data = None
             sensor_data = self._purpleair_api_obj.request_sensor_data(
                 the_json_file["sensor_index"],
                 the_json_file["read_key"],
@@ -245,6 +249,11 @@ def _run_loop_for_storing_single_sensor_data(self, the_json_file):
                 f"""Waiting {self._send_request_every_x_seconds} seconds before
                   requesting new data again..."""
             )
+
+            # Delete some stuff
+            del sensor_data
+            del the_modified_sensor_data
+
             sleep(self.send_request_every_x_seconds)
 
     def _run_loop_for_storing_multiple_sensors_data(self, json_config_file):
@@ -264,6 +273,7 @@ def _run_loop_for_storing_multiple_sensors_data(self, json_config_file):
                       {json_config_file["fields"]}..."""
             )
 
+            sensors_data = None
             sensors_data = self._purpleair_api_obj.request_multiple_sensors_data(
                 fields=json_config_file["fields"],
                 location_type=json_config_file["location_type"],
@@ -285,7 +295,7 @@ def _run_loop_for_storing_multiple_sensors_data(self, json_config_file):
             # It is important to know that the order of 'fields' provided as an argument to request_multiple_sensors_data()
             # will determine the order of data items. In a nutshell it is a 1:1 mapping from fields to data.
             # Now lets build and feed what the store_sensor_data() method expects.
-
+            store_sensor_data_type_list = []
             store_sensor_data_type_list = self._construct_store_sensor_data_type(
                 sensors_data
             )
@@ -298,6 +308,11 @@ def _run_loop_for_storing_multiple_sensors_data(self, json_config_file):
                 f"""Waiting {self._send_request_every_x_seconds} seconds before
                   requesting new data again..."""
             )
+
+            # Delete some stuff
+            del sensors_data
+            del store_sensor_data_type_list
+
             sleep(self.send_request_every_x_seconds)
 
     def _run_loop_for_storing_group_sensors_data(self, json_config_file):
@@ -430,6 +445,8 @@ def _construct_store_sensor_data_type(self, raw_data):
         """
 
         # Extract the 'fields' and 'data' parts to make it easier on ourselves
+        extracted_fields = None
+        extracted_data = None
         extracted_fields = raw_data["fields"]
         extracted_data = raw_data["data"]
         store_sensor_data_type_list = []
@@ -453,6 +470,11 @@ def _construct_store_sensor_data_type(self, raw_data):
 
             store_sensor_data_type_list.append(the_modified_sensor_data_dict)
 
+        # Delete some stuff
+        del extracted_fields
+        del extracted_data
+        del raw_data
+
         return store_sensor_data_type_list
 
     def validate_parameters_and_run(
@@ -483,6 +505,7 @@ def validate_parameters_and_run(
             # Now load up that json file
             file_obj = open(paa_multiple_sensor_request_json_file, "r")
             the_json_file = json.load(file_obj)
+            file_obj.close()
             self._run_loop_for_storing_multiple_sensors_data(the_json_file)
 
         elif (
@@ -493,6 +516,7 @@ def validate_parameters_and_run(
             # Now load up that json file
             file_obj = open(paa_single_sensor_request_json_file, "r")
             the_json_file = json.load(file_obj)
+            file_obj.close()
             self._run_loop_for_storing_single_sensor_data(the_json_file)
 
         elif (
@@ -503,6 +527,7 @@ def validate_parameters_and_run(
             # Now load up that json file
             file_obj = open(paa_group_sensor_request_json_file, "r")
             the_json_file = json.load(file_obj)
+            file_obj.close()
             self._run_loop_for_storing_group_sensors_data(the_json_file)
 
         elif (

diff --git a/purpleair_data_logger/PurpleAirPSQLDataLogger.py b/purpleair_data_logger/PurpleAirPSQLDataLogger.py
@@ -90,6 +90,38 @@ def __init__(self, PurpleAirAPIReadKey, PurpleAirAPIWriteKey, psql_db_conn):
         # Create continuous aggregates and materialized views
         self._configure_continuous_aggregates()
 
+        # Create some prepared statements
+        self._db_prepared_statements = {}
+        self._db_prepared_statements[
+            "station_information_and_status_fields"
+        ] = self._db_conn.prepare(
+            PSQL_INSERT_STATEMENT_STATION_INFORMATION_AND_STATUS_FIELDS
+        )
+        self._db_prepared_statements["environmental_fields"] = self._db_conn.prepare(
+            PSQL_INSERT_STATEMENT_ENVIRONMENTAL_FIELDS
+        )
+        self._db_prepared_statements["miscellaneous_fields"] = self._db_conn.prepare(
+            PSQL_INSERT_STATEMENT_MISCELLANEOUS_FIELDS
+        )
+        self._db_prepared_statements["pm1_0_fields"] = self._db_conn.prepare(
+            PSQL_INSERT_STATEMENT_PM1_0_FIELDS
+        )
+        self._db_prepared_statements["pm2_5_fields"] = self._db_conn.prepare(
+            PSQL_INSERT_STATEMENT_PM2_5_FIELDS
+        )
+        self._db_prepared_statements[
+            "pm2_5_pseudo_average_fields"
+        ] = self._db_conn.prepare(PSQL_INSERT_STATEMENT_PM2_5_PSEUDO_AVERAGE_FIELDS)
+        self._db_prepared_statements["pm10_0_fields"] = self._db_conn.prepare(
+            PSQL_INSERT_STATEMENT_PM10_0_FIELDS
+        )
+        self._db_prepared_statements["particle_count_fields"] = self._db_conn.prepare(
+            PSQL_INSERT_STATEMENT_PARTICLE_COUNT_FIELDS
+        )
+        self._db_prepared_statements["thingspeak_fields"] = self._db_conn.prepare(
+            PSQL_INSERT_STATEMENT_THINGSPEAK_FIELDS
+        )
+
         # Commit to the db
         self._db_conn.commit()
 
@@ -201,8 +233,7 @@ def store_sensor_data(self, single_sensor_data_dict):
         """
 
         # Run the queries
-        self._db_conn.run(
-            PSQL_INSERT_STATEMENT_STATION_INFORMATION_AND_STATUS_FIELDS,
+        self._db_prepared_statements["station_information_and_status_fields"].run(
             data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp(
                 single_sensor_data_dict["data_time_stamp"]
             ),
@@ -242,8 +273,7 @@ def store_sensor_data(self, single_sensor_data_dict):
             confidence_auto=single_sensor_data_dict["confidence_auto"],
         )
 
-        self._db_conn.run(
-            PSQL_INSERT_STATEMENT_ENVIRONMENTAL_FIELDS,
+        self._db_prepared_statements["environmental_fields"].run(
             data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp(
                 single_sensor_data_dict["data_time_stamp"]
             ),
@@ -259,8 +289,7 @@ def store_sensor_data(self, single_sensor_data_dict):
             pressure_b=single_sensor_data_dict["pressure_b"],
         )
 
-        self._db_conn.run(
-            PSQL_INSERT_STATEMENT_MISCELLANEOUS_FIELDS,
+        self._db_prepared_statements["miscellaneous_fields"].run(
             data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp(
                 single_sensor_data_dict["data_time_stamp"]
             ),
@@ -272,8 +301,7 @@ def store_sensor_data(self, single_sensor_data_dict):
             analog_input=single_sensor_data_dict["analog_input"],
         )
 
-        self._db_conn.run(
-            PSQL_INSERT_STATEMENT_PM1_0_FIELDS,
+        self._db_prepared_statements["pm1_0_fields"].run(
             data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp(
                 single_sensor_data_dict["data_time_stamp"]
             ),
@@ -289,8 +317,7 @@ def store_sensor_data(self, single_sensor_data_dict):
             pm1_0_cf_1_b=single_sensor_data_dict["pm1.0_cf_1_b"],
         )
 
-        self._db_conn.run(
-            PSQL_INSERT_STATEMENT_PM2_5_FIELDS,
+        self._db_prepared_statements["pm2_5_fields"].run(
             data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp(
                 single_sensor_data_dict["data_time_stamp"]
             ),
@@ -309,8 +336,7 @@ def store_sensor_data(self, single_sensor_data_dict):
             pm2_5_cf_1_b=single_sensor_data_dict["pm2.5_cf_1_b"],
         )
 
-        self._db_conn.run(
-            PSQL_INSERT_STATEMENT_PM2_5_PSEUDO_AVERAGE_FIELDS,
+        self._db_prepared_statements["pm2_5_pseudo_average_fields"].run(
             data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp(
                 single_sensor_data_dict["data_time_stamp"]
             ),
@@ -335,8 +361,7 @@ def store_sensor_data(self, single_sensor_data_dict):
             pm2_5_1week_b=single_sensor_data_dict["pm2.5_1week_b"],
         )
 
-        self._db_conn.run(
-            PSQL_INSERT_STATEMENT_PM10_0_FIELDS,
+        self._db_prepared_statements["pm10_0_fields"].run(
             data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp(
                 single_sensor_data_dict["data_time_stamp"]
             ),
@@ -352,8 +377,7 @@ def store_sensor_data(self, single_sensor_data_dict):
             pm10_0_cf_1_b=single_sensor_data_dict["pm10.0_cf_1_b"],
         )
 
-        self._db_conn.run(
-            PSQL_INSERT_STATEMENT_PARTICLE_COUNT_FIELDS,
+        self._db_prepared_statements["particle_count_fields"].run(
             data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp(
                 single_sensor_data_dict["data_time_stamp"]
             ),
@@ -378,8 +402,7 @@ def store_sensor_data(self, single_sensor_data_dict):
             um_count_b_10_0=single_sensor_data_dict["10.0_um_count_b"],
         )
 
-        self._db_conn.run(
-            PSQL_INSERT_STATEMENT_THINGSPEAK_FIELDS,
+        self._db_prepared_statements["thingspeak_fields"].run(
             data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp(
                 single_sensor_data_dict["data_time_stamp"]
             ),
@@ -397,6 +420,9 @@ def store_sensor_data(self, single_sensor_data_dict):
         # Commit to the db
         self._db_conn.commit()
 
+        # Delete some stuff
+        del single_sensor_data_dict
+
 
 if __name__ == "__main__":
     parser = generate_common_arg_parser(

diff --git a/setup.py b/setup.py
@@ -12,7 +12,7 @@ def read_file(filename):
 
 setup(
     name="purpleair_data_logger",
-    version="1.2.0",
+    version="1.2.1a1",
     license="MIT",
     author="Carlos Santos",
     author_email="[email protected]",
@@ -29,6 +29,6 @@ def read_file(filename):
         "purple air api",
         "PurpleAirSQLiteDataLogger",
     ],
-    install_requires=["pg8000==1.29.4", "requests", "purpleair_api==1.0.1"],
+    install_requires=["pg8000==1.29.5", "requests", "purpleair_api==1.0.2a1"],
     platforms=["Windows 32/64", "Linux 32/64", "MacOS 32/64"],
 )