-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathcalibrator.cfg
116 lines (102 loc) · 3.42 KB
/
calibrator.cfg
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
# Smart Emission Data Extract ETL - Stetl config
#
# Pieter Marsman - 2016
#
# This config reads the raw timeseries measurements from the DB, and RIVM
# timeseries from their API, merges those and uses it to create a calibration
# model.
# The main Stetl ETL chain
[etl]
#chains = input_rivm|input_jose|merge|output_dataset
#chains = input_dataset|calibrate
chains = input_rivm|input_jose|merge|calibrate|(output_model)(output_parameters)(vis1)(vis2)(vis3)(vis4)
[input_rivm]
class = smartem.calibrator.calibration_input.CalibrationInfluxDbInput
method = POST
content_type = application/x-www-form-urlencoded
host = {influx_host}
port = {influx_port}
database = {influx_se_database}
user = {influx_se_writer}
password = {influx_se_writer_password}
# currently assuming that all components from influxdb are predicted
query = SELECT time, component, geohash, value FROM {influx_se_measurement_rivm} WHERE {calibration_location_geohash} AND time > '%s' LIMIT {calibrator_max_input_records}
key = rivm
cache_result = True
output_format = record
[input_jose]
class = smartem.calibrator.calibration_input.CalibrationInfluxDbInput
input_format = record
method = POST
content_type = application/x-www-form-urlencoded
host = {influx_host}
port = {influx_port}
database = {influx_se_database}
user = {influx_se_writer}
password = {influx_se_writer_password}
# currently assuming that all components from influxdb are used
query = SELECT time, component, geohash, value FROM {influx_se_measurement_extract} WHERE {calibration_location_geohash} AND time > '%s' LIMIT {calibrator_max_input_records}
key = jose
cache_result = True
output_format = record
[merge]
class = smartem.calibrator.mergerefdata.MergeRivmJose
input = record
output = record_array
[input_dataset]
class = smartem.calibrator.calibration_input.CalibrationDataInput
file_path = calibration/dataset.csv,calibration/dataset.csv,calibration/dataset.csv
output = record
[output_dataset]
class = smartem.calibrator.calibration_output.CalibrationDataOutput
input = record
output = record
file_path = calibration/dataset.csv
[calibrate]
class = smartem.calibrator.calibration.Calibrator
input = record_array
running_means = {{'s_coresistance':0.05,'s_o3resistance':0.05,'s_no2resistance':0.05}}
targets = {calibration_components}
inverse_sample_fraction = 50
random_search_iterations = 50
cv_k = 10
output = record
n_jobs = 1
[output_model]
class = smartem.calibrator.calibration_output.CalibrationModelOutput
input_format = record
host = {pg_host}
database = {pg_database}
user = {pg_user}
password = {pg_password}
schema = {pg_schema_calibrated}
table = calibration_models
[output_parameters]
class = smartem.calibrator.calibration_output.ParameterOutput
input_format = record
host = {pg_host}
database = {pg_database}
user = {pg_user}
password = {pg_password}
schema = {pg_schema_calibrated}
table = calibration_parameters
[vis1]
class = smartem.calibrator.calibration_visualization.PerformanceVisualization
input = record
clear_output_folder = True
file_path = calibration/%s/%s
[vis2]
class = smartem.calibrator.calibration_visualization.ModelVisualization
input = record
file_path = calibration/%s/%s
[vis3]
class = smartem.calibrator.calibration_visualization.DataVisualization
input = record
file_path = calibration/%s/%s
[vis4]
class = smartem.calibrator.calibration_visualization.SearchVisualization
input = record
file_path = calibration/%s/%s
# for testing/debugging
[output_std]
class = outputs.standardoutput.StandardOutput