diff --git a/.vscode/settings.json b/.vscode/settings.json index 514c82e..a8764e6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -15,5 +15,6 @@ "timescaledb", "PSQL", "deciviews" - ] + ], + "python.formatting.provider": "black" } \ No newline at end of file diff --git a/purpleair_data_logger/PurpleAirDataLogger.py b/purpleair_data_logger/PurpleAirDataLogger.py index b062d42..06d3374 100644 --- a/purpleair_data_logger/PurpleAirDataLogger.py +++ b/purpleair_data_logger/PurpleAirDataLogger.py @@ -165,6 +165,9 @@ def _validate_sensor_data_before_insert(self, the_modified_sensor_data): field ] + # Delete some stuff + del the_modified_sensor_data + # Then return the modified copy return temp_the_modified_sensor_data @@ -185,6 +188,7 @@ def _run_loop_for_storing_single_sensor_data(self, the_json_file): {the_json_file['sensor_index']}...""" ) + sensor_data = None sensor_data = self._purpleair_api_obj.request_sensor_data( the_json_file["sensor_index"], the_json_file["read_key"], @@ -245,6 +249,11 @@ def _run_loop_for_storing_single_sensor_data(self, the_json_file): f"""Waiting {self._send_request_every_x_seconds} seconds before requesting new data again...""" ) + + # Delete some stuff + del sensor_data + del the_modified_sensor_data + sleep(self.send_request_every_x_seconds) def _run_loop_for_storing_multiple_sensors_data(self, json_config_file): @@ -264,6 +273,7 @@ def _run_loop_for_storing_multiple_sensors_data(self, json_config_file): {json_config_file["fields"]}...""" ) + sensors_data = None sensors_data = self._purpleair_api_obj.request_multiple_sensors_data( fields=json_config_file["fields"], location_type=json_config_file["location_type"], @@ -285,7 +295,7 @@ def _run_loop_for_storing_multiple_sensors_data(self, json_config_file): # It is important to know that the order of 'fields' provided as an argument to request_multiple_sensors_data() # will determine the order of data items. In a nutshell it is a 1:1 mapping from fields to data. # Now lets build and feed what the store_sensor_data() method expects. - + store_sensor_data_type_list = [] store_sensor_data_type_list = self._construct_store_sensor_data_type( sensors_data ) @@ -298,6 +308,11 @@ def _run_loop_for_storing_multiple_sensors_data(self, json_config_file): f"""Waiting {self._send_request_every_x_seconds} seconds before requesting new data again...""" ) + + # Delete some stuff + del sensors_data + del store_sensor_data_type_list + sleep(self.send_request_every_x_seconds) def _run_loop_for_storing_group_sensors_data(self, json_config_file): @@ -430,6 +445,8 @@ def _construct_store_sensor_data_type(self, raw_data): """ # Extract the 'fields' and 'data' parts to make it easier on ourselves + extracted_fields = None + extracted_data = None extracted_fields = raw_data["fields"] extracted_data = raw_data["data"] store_sensor_data_type_list = [] @@ -453,6 +470,11 @@ def _construct_store_sensor_data_type(self, raw_data): store_sensor_data_type_list.append(the_modified_sensor_data_dict) + # Delete some stuff + del extracted_fields + del extracted_data + del raw_data + return store_sensor_data_type_list def validate_parameters_and_run( @@ -483,6 +505,7 @@ def validate_parameters_and_run( # Now load up that json file file_obj = open(paa_multiple_sensor_request_json_file, "r") the_json_file = json.load(file_obj) + file_obj.close() self._run_loop_for_storing_multiple_sensors_data(the_json_file) elif ( @@ -493,6 +516,7 @@ def validate_parameters_and_run( # Now load up that json file file_obj = open(paa_single_sensor_request_json_file, "r") the_json_file = json.load(file_obj) + file_obj.close() self._run_loop_for_storing_single_sensor_data(the_json_file) elif ( @@ -503,6 +527,7 @@ def validate_parameters_and_run( # Now load up that json file file_obj = open(paa_group_sensor_request_json_file, "r") the_json_file = json.load(file_obj) + file_obj.close() self._run_loop_for_storing_group_sensors_data(the_json_file) elif ( diff --git a/purpleair_data_logger/PurpleAirPSQLDataLogger.py b/purpleair_data_logger/PurpleAirPSQLDataLogger.py index aa5a3ba..431fdf8 100644 --- a/purpleair_data_logger/PurpleAirPSQLDataLogger.py +++ b/purpleair_data_logger/PurpleAirPSQLDataLogger.py @@ -90,6 +90,38 @@ def __init__(self, PurpleAirAPIReadKey, PurpleAirAPIWriteKey, psql_db_conn): # Create continuous aggregates and materialized views self._configure_continuous_aggregates() + # Create some prepared statements + self._db_prepared_statements = {} + self._db_prepared_statements[ + "station_information_and_status_fields" + ] = self._db_conn.prepare( + PSQL_INSERT_STATEMENT_STATION_INFORMATION_AND_STATUS_FIELDS + ) + self._db_prepared_statements["environmental_fields"] = self._db_conn.prepare( + PSQL_INSERT_STATEMENT_ENVIRONMENTAL_FIELDS + ) + self._db_prepared_statements["miscellaneous_fields"] = self._db_conn.prepare( + PSQL_INSERT_STATEMENT_MISCELLANEOUS_FIELDS + ) + self._db_prepared_statements["pm1_0_fields"] = self._db_conn.prepare( + PSQL_INSERT_STATEMENT_PM1_0_FIELDS + ) + self._db_prepared_statements["pm2_5_fields"] = self._db_conn.prepare( + PSQL_INSERT_STATEMENT_PM2_5_FIELDS + ) + self._db_prepared_statements[ + "pm2_5_pseudo_average_fields" + ] = self._db_conn.prepare(PSQL_INSERT_STATEMENT_PM2_5_PSEUDO_AVERAGE_FIELDS) + self._db_prepared_statements["pm10_0_fields"] = self._db_conn.prepare( + PSQL_INSERT_STATEMENT_PM10_0_FIELDS + ) + self._db_prepared_statements["particle_count_fields"] = self._db_conn.prepare( + PSQL_INSERT_STATEMENT_PARTICLE_COUNT_FIELDS + ) + self._db_prepared_statements["thingspeak_fields"] = self._db_conn.prepare( + PSQL_INSERT_STATEMENT_THINGSPEAK_FIELDS + ) + # Commit to the db self._db_conn.commit() @@ -201,8 +233,7 @@ def store_sensor_data(self, single_sensor_data_dict): """ # Run the queries - self._db_conn.run( - PSQL_INSERT_STATEMENT_STATION_INFORMATION_AND_STATUS_FIELDS, + self._db_prepared_statements["station_information_and_status_fields"].run( data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp( single_sensor_data_dict["data_time_stamp"] ), @@ -242,8 +273,7 @@ def store_sensor_data(self, single_sensor_data_dict): confidence_auto=single_sensor_data_dict["confidence_auto"], ) - self._db_conn.run( - PSQL_INSERT_STATEMENT_ENVIRONMENTAL_FIELDS, + self._db_prepared_statements["environmental_fields"].run( data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp( single_sensor_data_dict["data_time_stamp"] ), @@ -259,8 +289,7 @@ def store_sensor_data(self, single_sensor_data_dict): pressure_b=single_sensor_data_dict["pressure_b"], ) - self._db_conn.run( - PSQL_INSERT_STATEMENT_MISCELLANEOUS_FIELDS, + self._db_prepared_statements["miscellaneous_fields"].run( data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp( single_sensor_data_dict["data_time_stamp"] ), @@ -272,8 +301,7 @@ def store_sensor_data(self, single_sensor_data_dict): analog_input=single_sensor_data_dict["analog_input"], ) - self._db_conn.run( - PSQL_INSERT_STATEMENT_PM1_0_FIELDS, + self._db_prepared_statements["pm1_0_fields"].run( data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp( single_sensor_data_dict["data_time_stamp"] ), @@ -289,8 +317,7 @@ def store_sensor_data(self, single_sensor_data_dict): pm1_0_cf_1_b=single_sensor_data_dict["pm1.0_cf_1_b"], ) - self._db_conn.run( - PSQL_INSERT_STATEMENT_PM2_5_FIELDS, + self._db_prepared_statements["pm2_5_fields"].run( data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp( single_sensor_data_dict["data_time_stamp"] ), @@ -309,8 +336,7 @@ def store_sensor_data(self, single_sensor_data_dict): pm2_5_cf_1_b=single_sensor_data_dict["pm2.5_cf_1_b"], ) - self._db_conn.run( - PSQL_INSERT_STATEMENT_PM2_5_PSEUDO_AVERAGE_FIELDS, + self._db_prepared_statements["pm2_5_pseudo_average_fields"].run( data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp( single_sensor_data_dict["data_time_stamp"] ), @@ -335,8 +361,7 @@ def store_sensor_data(self, single_sensor_data_dict): pm2_5_1week_b=single_sensor_data_dict["pm2.5_1week_b"], ) - self._db_conn.run( - PSQL_INSERT_STATEMENT_PM10_0_FIELDS, + self._db_prepared_statements["pm10_0_fields"].run( data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp( single_sensor_data_dict["data_time_stamp"] ), @@ -352,8 +377,7 @@ def store_sensor_data(self, single_sensor_data_dict): pm10_0_cf_1_b=single_sensor_data_dict["pm10.0_cf_1_b"], ) - self._db_conn.run( - PSQL_INSERT_STATEMENT_PARTICLE_COUNT_FIELDS, + self._db_prepared_statements["particle_count_fields"].run( data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp( single_sensor_data_dict["data_time_stamp"] ), @@ -378,8 +402,7 @@ def store_sensor_data(self, single_sensor_data_dict): um_count_b_10_0=single_sensor_data_dict["10.0_um_count_b"], ) - self._db_conn.run( - PSQL_INSERT_STATEMENT_THINGSPEAK_FIELDS, + self._db_prepared_statements["thingspeak_fields"].run( data_time_stamp=self._convert_unix_epoch_timestamp_to_psql_timestamp( single_sensor_data_dict["data_time_stamp"] ), @@ -397,6 +420,9 @@ def store_sensor_data(self, single_sensor_data_dict): # Commit to the db self._db_conn.commit() + # Delete some stuff + del single_sensor_data_dict + if __name__ == "__main__": parser = generate_common_arg_parser( diff --git a/setup.py b/setup.py index 42ceaf5..49b667b 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def read_file(filename): setup( name="purpleair_data_logger", - version="1.2.0", + version="1.2.1a1", license="MIT", author="Carlos Santos", author_email="dose.lucky.sake@cloak.id", @@ -29,6 +29,6 @@ def read_file(filename): "purple air api", "PurpleAirSQLiteDataLogger", ], - install_requires=["pg8000==1.29.4", "requests", "purpleair_api==1.0.1"], + install_requires=["pg8000==1.29.5", "requests", "purpleair_api==1.0.2a1"], platforms=["Windows 32/64", "Linux 32/64", "MacOS 32/64"], )