Signal agnostic default calibration files, load actions and sensor def in settings, initialize signal analyzer in gunicorn.

README.md

@@ -368,6 +368,177 @@ specific to the sensor you are using.
 }
 ```
 
+### Sensor Calibration File
+By default, scos-sensor will use `configs/default_calibration.json` as the sensor calibration file. However, if
+`configs/sensor_calibration.json` or `configs/sigan_calibration.json` exist they will be used instead of the default
+calibration file. Sensor calibration files allow scos-sensor to apply a gain based on a laboratory calibration of
+the sensor and may also contain other useful metadata that characterizes the sensor performance. For additional
+information on the calibration data, see the [NTIA-Sensor SigMF Calibration Object](https://github.com/NTIA/sigmf-ns-ntia/blob/master/ntia-sensor.sigmf-ext.md#08-the-calibration-object).
+The default calibration file is shown below:
+```json
+{
+  "calibration_data":{
+    "datetime": "1970-01-01T00:00:00.000000Z",
+    "gain": 0,
+    "noise_figure": null,
+    "1db_compression_point": null,
+    "enbw": null,
+    "temperature": 26.85
+  },
+  "last_calibration_datetime": "1970-01-01T00:00:00.000000Z",
+  "calibration_parameters": [],
+  "clock_rate_lookup_by_sample_rate": [
+  ],
+  "sensor_uid": "DEFAULT CALIBRATION",
+  "calibration_reference": "noise source output"
+}
+```
+The `calibration_parameters` key lists the parameters that will be used to obtain
+the calibration data. In the case of the default calibration, there are no
+`calibration_parameters` so the calibration data is found directly within the
+`calibration_data` element and by default scos-sensor will not apply any additional
+gain. Typically, a sensor would be calibrated at particular
+sensing parameters. The calibration data for specific parameters should be listed
+within the calibration_data object and accessed by the values of the settings listed in the 
+calibration_parameters element. For example, the calibration below provides an example of a
+sensor calibrated at a sample rate of 14000000.0 samples per second at several 
+frequencies with a signal analyzer reference level setting of -25.
+```json
+{
+	"calibration_parameters": [
+		"sample_rate",
+		"frequency",
+		"reference_level"
+	],
+	"calibration_datetime": "2020-11-18T23:13:09.156274Z",
+	"calibration_data": {
+		"14000000.0":{
+			"3555000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3565000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3575000000":{
+
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3585000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3595000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3605000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3615000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3625000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3635000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3645000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3655000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3665000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3675000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3685000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			},
+			"3695000000":{
+				"-25":{
+					"noise_figure": 46.03993010994134,
+					"enbw": 15723428.858731967,
+					"gain": 0.40803345928877379
+				}
+			}
+		}
+	},
+	"clock_rate_lookup_by_sample_rate": [
+		{
+			"sample_rate": 3500000.0,
+			"clock_frequency": 3500000.0
+		},
+		{
+			"sample_rate": 28000000.0,
+			"clock_frequency": 28000000.0
+		}
+	],
+	"sensor_uid": "US55120115"
+}
+
+```
+When an action is run with the above calibration, scos will expect the action to have
+a sample_rate, frequency, and reference_level specified in the action config. The values
+specified for these parameters will then be used to retrieve the calibration data.
+
 ## Security
 
 This section covers authentication, permissions, and certificates used to access the

configs/default_calibration.json

@@ -0,0 +1,16 @@
+{
+  "calibration_data":{
+    "datetime": "1970-01-01T00:00:00.000000Z",
+    "gain": 0,
+    "noise_figure": 0.0,
+    "1db_compression_point": null,
+    "enbw": null,
+    "temperature": 26.85
+  },
+  "last_calibration_datetime": "1970-01-01T00:00:00.000000Z",
+  "calibration_parameters": [],
+  "clock_rate_lookup_by_sample_rate": [
+  ],
+  "sensor_uid": "DEFAULT CALIBRATION",
+  "calibration_reference": "noise source output"
+}

docker-compose.yml

@@ -60,6 +60,8 @@ services:
       - POSTGRES_PASSWORD
       - SCOS_SENSOR_GIT_TAG
       - SECRET_KEY
+      - SIGAN_MODULE
+      - SIGAN_CLASS
       - SIGAN_POWER_SWITCH
       - SIGAN_POWER_CYCLE_STATES
     expose:

docs/openapi.json

@@ -1636,16 +1636,7 @@
                     "title": "Action",
                     "description": "[Required] The name of the action to be scheduled",
                     "type": "string",
-                    "enum": [
-                        "logger",
-                        "test_monitor_sigan",
-                        "test_multi_frequency_iq_action",
-                        "test_nasctn_sea_data_product",
-                        "test_single_frequency_iq_action",
-                        "test_single_frequency_m4s_action",
-                        "test_survey_iq_action",
-                        "test_sync_gps"
-                    ]
+                    "enum": []
                 },
                 "priority": {
                     "title": "Priority",
@@ -1941,4 +1932,4 @@
             }
         }
     }
-}
+}

entrypoints/api_entrypoint.sh

@@ -10,16 +10,11 @@ function cleanup_demodb {
 
 trap cleanup_demodb SIGTERM
 trap cleanup_demodb SIGINT
-# This is done to avoid loading actions and connecting to the sigan when migrations are applied and when
-# the super user is created.
-cp sensor/migration_settings.py sensor/settings.py
 echo "Starting Migrations"
 python3.8 manage.py migrate
 
-
 echo "Creating superuser (if managed)"
 python3.8 /scripts/create_superuser.py
-cp sensor/runtime_settings.py sensor/settings.py
 
 echo "Starting Gunicorn"
 exec gunicorn sensor.wsgi -c ../gunicorn/config.py &

gunicorn/config.py

@@ -1,14 +1,18 @@
+import importlib
+import logging
 import os
 import sys
 from multiprocessing import cpu_count
 
+
+
 bind = ":8000"
 workers = 1
 worker_class = "gthread"
 threads = cpu_count()
 
 loglevel = os.environ.get("GUNICORN_LOG_LEVEL", "info")
-
+logger = logging.getLogger(__name__)
 
 def _modify_path():
     """Ensure Django project is on sys.path."""
@@ -27,9 +31,67 @@ def post_worker_init(worker):
     import django
 
     django.setup()
-
     from scheduler import scheduler
-
+    from initialization import (
+        load_preselector,
+        load_switches,
+    )
+    from initialization import (
+        get_sensor_calibration,
+        get_sigan_calibration
+    )
+    from django.conf import settings
+    from status.models import Location
+    from scos_actions.hardware.sensor import Sensor
+    from scos_actions.metadata.utils import construct_geojson_point
+    from scos_actions.signals import register_component_with_status
+    from scos_actions.signals import register_signal_analyzer
+    from scos_actions.signals import register_sensor
+
+    sigan_module_setting = settings.SIGAN_MODULE
+    sigan_module = importlib.import_module(sigan_module_setting)
+    logger.info("Creating " + settings.SIGAN_CLASS + " from " + settings.SIGAN_MODULE)
+    sigan_constructor = getattr(sigan_module, settings.SIGAN_CLASS)
+    sensor_cal = get_sensor_calibration(settings.SENSOR_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE)
+    sigan_cal = get_sigan_calibration(settings.SIGAN_CALIBRATION_FILE, settings.DEFAULT_CALIBRATION_FILE)
+    sigan = sigan_constructor(sensor_cal=sensor_cal, sigan_cal=sigan_cal)
+    register_component_with_status.send(sigan, component=sigan)
+    register_signal_analyzer.send(sigan, signal_analyzer=sigan)
+
+    switches = load_switches(settings.SWITCH_CONFIGS_DIR)
+    capabilities = settings.CAPABILITIES
+    preselector = load_preselector(settings.PRESELECTOR_CONFIG, settings.PRESELECTOR_MODULE, settings.PRESELECTOR_CLASS, capabilities["sensor"])
+    location = None
+    if "location" in capabilities["sensor"]:
+        try:
+            sensor_loc = capabilities["sensor"].pop("location")
+            try:
+                #if there is an active database location, use it over the value in the sensor def.
+                db_location = Location.objects.get(active=True)
+                location = construct_geojson_point(db_location.longitude, db_location.latitude, db_location.height)
+            except Location.DoesNotExist:
+                # This should never occur because status/migrations/0003_auto_20211217_2229.py
+                # will load the No DB location. Use sensor def location and save to DB.
+                location = construct_geojson_point(
+                    sensor_loc["x"],
+                    sensor_loc["y"],
+                    sensor_loc["z"] if "z" in sensor_loc else None,
+                )
+                #Save the sensor location from the sensor def to the database
+                db_location = Location()
+                db_location.longitude = sensor_loc["x"]
+                db_location.latitude = sensor_loc["y"]
+                db_location.height = sensor_loc["z"]
+                db_location.gps = False
+                db_location.description = sensor_loc["description"]
+                db_location.save()
+        except:
+            logger.exception("Failed to get sensor location from sensor definition.")
+
+
+    sensor = Sensor(signal_analyzer=sigan, preselector = preselector, switches = switches, capabilities = capabilities, location = location)
+    scheduler.thread.sensor = sensor
+    register_sensor.send(sensor, sensor=sensor)
     scheduler.thread.start()
 
 

scripts/print_action_docstring.py

@@ -6,7 +6,7 @@
 
 import django
 
-from utils.action_registrar import registered_actions
+from actions import actions
 
 PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "src")
 
@@ -16,7 +16,7 @@
 django.setup()
 
 
-action_names = sorted(registered_actions.keys())
+action_names = sorted(actions.keys())
 
 
 if __name__ == "__main__":
@@ -28,4 +28,4 @@
     ),
 
     args = parser.parse_args()
-    print(registered_actions[args.action].description)
+    print(actions[args.action].description)