harvester_influx.cfg

# Smart Emission Data Harvester - Stetl config
#
# Just van den Broecke - 2016
#
# This config reads raw measurements from InfluxDB.
# It inserts this data as JSON blobs in Postgres.
# The progres of harvesting (date/time) is tracked, so this allows to harvest all data
# in multiple steps.
# The measurement data in Postgres is then available for further refinement (validation, aggregation, etc).

# The main Stetl ETL chain
[etl]
chains = input_influxdb_1|output_postgres_insert

# Read all history data from an InfluxDB with AirSensEUR data
[input_influxdb_1]
class = smartem.harvester.harvestinfluxdb.HarvesterInfluxDbInput
content_type = application/x-www-form-urlencoded
host = {influx_dc1_host}
port = {influx_port}
database = {influx_as_database}
user = {influx_as_reader}
password = {influx_as_reader_password}
pg_host = {pg_host}
pg_port = {pg_port}
pg_database = {pg_database}
pg_user = {pg_user}
pg_password = {pg_password}
pg_schema = {pg_schema_raw}
max_proc_time_secs = {max_proc_time_secs}
device_type = ase
device_version = 1
meas_name_to_device_id = {{'ASE_NL_01': '11820001', 'ASE_NL_02': '11820002', 'ASE_NL_03': '11820003', 'ASE_NL_04': '11820004', 'ASE_NL_05': '11820005'}}
progress_table = harvester_progress

# for testing/debugging
[output_std]
class = outputs.standardoutput.StandardOutput

# Insert file records
[output_postgres_insert]
class = outputs.dboutput.PostgresInsertOutput
input_format = record_array
host = {pg_host}
database = {pg_database}
user = {pg_user}
password = {pg_password}
schema = {pg_schema_raw}
table = timeseries
key=unique_id
replace=True