Merge pull request #626 from projecthorus/testing

v1.5.10 release - DFM updates, APRS limits

README.md

@@ -21,7 +21,8 @@ Vaisala | RS41-SG/SGP/SGM | :heavy_check_mark: | :heavy_check_mark: | :heavy_che
 Graw | DFM06/09/17 | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | :x:
 Meteomodem | M10 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | Not Sent | :x:
 Meteomodem | M20 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: (For some models) | :x:
-Intermet Systems | iMet-4 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | Not Sent | :x:
+Intermet Systems | iMet-1 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | Not Sent | :heavy_check_mark:
+Intermet Systems | iMet-4 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | Not Sent | :heavy_check_mark:
 Intermet Systems | iMet-54 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | Not Sent | :x:
 Lockheed Martin | LMS6-400/1680 | :heavy_check_mark: | :x: | :x: | :x: | Not Sent
 Meisei | iMS-100 | :heavy_check_mark: | :x: | :x: | :x: | Not Sent

auto_rx/auto_rx.py

@@ -876,7 +876,7 @@ def main():
             position_report=config["aprs_position_report"],
             aprsis_host=config["aprs_server"],
             aprsis_port=config["aprs_port"],
-            synchronous_upload_time=config["aprs_upload_rate"],
+            upload_time=config["aprs_upload_rate"],
             callsign_validity_threshold=config["payload_id_valid"],
             station_beacon=config["station_beacon_enabled"],
             station_beacon_rate=config["station_beacon_rate"],

auto_rx/autorx/__init__.py

@@ -17,7 +17,7 @@
 # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus.
 # PATCH - Small changes, or minor feature additions.
 
-__version__ = "1.5.9"
+__version__ = "1.5.10"
 
 
 # Global Variables

auto_rx/autorx/aprs.py

@@ -89,9 +89,6 @@ def telemetry_to_aprs_position(
 
     # TODO: RS41 Burst Timer
 
-    # Add on auto_rx version
-    _aprs_comment += " auto_rx v" + auto_rx_version
-
     # Convert float latitude to APRS format (DDMM.MM)
     lat = float(sonde_data["lat"])
     lat_degree = abs(int(lat))
@@ -308,7 +305,7 @@ def __init__(
         station_beacon_position=(0.0, 0.0, 0.0),
         station_beacon_comment="radiosonde_auto_rx SondeGate v<version>",
         station_beacon_icon="/r",
-        synchronous_upload_time=30,
+        upload_time=60,
         callsign_validity_threshold=5,
         upload_queue_size=16,
         upload_timeout=5,
@@ -339,9 +336,7 @@ def __init__(
             station_beacon_comment (str): Comment field for the station beacon. <version> will be replaced with the current auto_rx version.
             station_beacon_icon (str): The APRS icon to be used, as the two characters (symbol table, symbol index), as per http://www.aprs.org/symbols.html
 
-            synchronous_upload_time (int): Upload the most recent telemetry when time.time()%synchronous_upload_time == 0
-                This is done in an attempt to get multiple stations uploading the same telemetry sentence simultaneously,
-                and also acts as decimation on the number of sentences uploaded to APRS-IS.
+            upload_time (int): Upload the most recent telemetry after this time is up.
 
             callsign_validity_threshold (int): Only upload telemetry data if the callsign has been observed more than N times. Default = 5
 
@@ -362,7 +357,8 @@ def __init__(
         self.aprsis_reconnect = aprsis_reconnect
         self.upload_timeout = upload_timeout
         self.upload_queue_size = upload_queue_size
-        self.synchronous_upload_time = synchronous_upload_time
+        self.upload_time = upload_time
+        self.next_upload = time.monotonic() + upload_time
         self.callsign_validity_threshold = callsign_validity_threshold
         self.inhibit = inhibit
 
@@ -653,7 +649,7 @@ def upload_timer(self):
         """ Add packets to the aprs upload queue if it is time for us to upload. """
 
         while self.timer_thread_running:
-            if int(time.time()) % self.synchronous_upload_time == 0:
+            if time.monotonic() > self.next_upload:
                 # Time to upload!
                 for _id in self.observed_payloads.keys():
                     # If no data, continue...
@@ -677,6 +673,9 @@ def upload_timer(self):
 
                 # Flush APRS-IS RX buffer
                 self.flush_rx()
+
+                # Reset upload timer
+                self.next_upload = time.monotonic() + self.upload_time
             else:
                 # Not yet time to upload, wait for a bit.
                 time.sleep(0.1)

auto_rx/autorx/config.py

@@ -352,7 +352,7 @@ def read_auto_rx_config(filename, no_sdr_test=False):
 
         if auto_rx_config["aprs_upload_rate"] < MINIMUM_APRS_UPDATE_RATE:
             logging.warning(
-                "Config - APRS Update Rate clipped to minimum of %d seconds. Please be respectful of other users of APRS-IS."
+                "Config - APRS Update Rate clipped to minimum of %d seconds."
                 % MINIMUM_APRS_UPDATE_RATE
             )
             auto_rx_config["aprs_upload_rate"] = MINIMUM_APRS_UPDATE_RATE
@@ -517,7 +517,7 @@ def read_auto_rx_config(filename, no_sdr_test=False):
             auto_rx_config["aprs_port"] = config.getint("aprs", "aprs_port")
         except:
             logging.warning(
-                "Config - Did not find aprs_port setting - using default of 14590. APRS packets might not be forwarded out to the wider APRS-IS network!"
+                "Config - Did not find aprs_port setting - using default of 14590."
             )
             auto_rx_config["aprs_port"] = 14590
 
@@ -652,6 +652,39 @@ def read_auto_rx_config(filename, no_sdr_test=False):
             )
             auto_rx_config["experimental_decoders"]["MK2LMS"] = False
 
+
+        # As of auto_rx version 1.5.10, we are limiting APRS output to only radiosondy.info,
+        # and only on the non-forwarding port. 
+        # This decision was not made lightly, and is a result of the considerable amount of
+        # non-amateur traffic that radiosonde flights are causing within the APRS-IS network.
+        # Until some form of common format can be agreed to amongst the developers of *all* 
+        # radiosonde tracking software to enable radiosonde telemetry to be de-duped, 
+        # I have decided to help reduce the impact on the wider APRS-IS network by restricting 
+        # the allowed servers and ports.
+        # If you are using another APRS-IS server that *does not* forward to the wider APRS-IS
+        # network and want it allowed, then please raise an issue at
+        # https://github.com/projecthorus/radiosonde_auto_rx/issues
+        #
+        # You are of course free to fork and modify this codebase as you wish, but please be aware
+        # that this goes against the wishes of the radiosonde_auto_rx developers to not be part
+        # of the bigger problem of APRS-IS congestion. 
+
+        ALLOWED_APRS_SERVERS = ["radiosondy.info"]
+        ALLOWED_APRS_PORTS = [14590]
+
+        if auto_rx_config["aprs_server"] not in ALLOWED_APRS_SERVERS:
+            logging.warning(
+                "Please do not upload to servers which forward to the wider APRS-IS network and cause network congestion. Switching to default server of radiosondy.info. If you believe this to be in error, please raise an issue at https://github.com/projecthorus/radiosonde_auto_rx/issues"
+            )
+            auto_rx_config["aprs_server"] = "radiosondy.info"
+
+        if auto_rx_config["aprs_port"] not in ALLOWED_APRS_PORTS:
+            logging.warning(
+                "Please do not use APRS ports which forward data out to the wider APRS-IS network and cause network congestion. Switching to default port of 14590. If you believe this to be in error, please raise an issue at https://github.com/projecthorus/radiosonde_auto_rx/issues"
+            )
+            auto_rx_config["aprs_port"] = 14590
+
+
         # If we are being called as part of a unit test, just return the config now.
         if no_sdr_test:
             return auto_rx_config

auto_rx/autorx/decode.py

@@ -808,8 +808,12 @@ def generate_decoder_command_experimental(self):
             )
 
             # DFM decoder
+            if len(self.raw_file_option)>0:
+                # Use raw ecc detailed raw output for DFM sondes.
+                self.raw_file_option = "--rawecc"
+
             decode_cmd = (
-                f"./dfm09mod -vv --ecc --json --dist --auto --softin -i {self.raw_file_option.upper()} 2>/dev/null"
+                f"./dfm09mod -vv --ecc --json --dist --auto --softin -i {self.raw_file_option} 2>/dev/null"
             )
 
             # DFM sondes transmit continuously - average over the last 2 frames, and peak hold
@@ -1282,8 +1286,18 @@ def handle_decoder_line(self, data):
                     # in the subtype field, so we can use this directly.
                     _telemetry["type"] = _telemetry["subtype"]
                 elif self.sonde_type == "DFM":
-                    # For DFM sondes, we need to use a lookup to convert the subtype field into a model.
-                    _telemetry["type"] = decode_dfm_subtype(_telemetry["subtype"])
+                    # As of 2021-2, the decoder provides a guess of the DFM subtype, provided as
+                    # a subtype field of "0xX:GUESS", e.g. "0xD:DFM17P"
+                    if ":" in _telemetry["subtype"]:
+                        _subtype = _telemetry["subtype"].split(":")[1]
+                        _telemetry["dfmcode"] = _telemetry["subtype"].split(":")[0]
+                        _telemetry["type"] = _subtype
+                        _telemetry["subtype"] = _subtype
+                    else:
+                        _telemetry["type"] = "DFM"
+                        _telemetry["subtype"] = "DFM"
+
+
 
                     # Check frame ID here to ensure we are on dfm09mod version with the frame number fixes (2020-12).
                     if _telemetry["frame"] < 256:

auto_rx/autorx/sondehub.py

@@ -172,6 +172,8 @@ def reformat_data(self, telemetry):
             _output["type"] = "DFM"
             _output["subtype"] = telemetry["type"]
             _output["serial"] = telemetry["id"].split("-")[1]
+            if "dfmcode" in telemetry:
+                _output["dfmcode"] = telemetry["dfmcode"]
 
         elif telemetry["type"].startswith("M10") or telemetry["type"].startswith("M20"):
             _output["manufacturer"] = "Meteomodem"
@@ -383,6 +385,14 @@ def upload_telemetry(self, telem_list):
                 _retries += 1
                 continue
 
+            elif _req.status_code == 201:
+                self.log_debug(
+                    "Sondehub reported issue when adding packets to DB. Status Code: %d %s."
+                    % (_req.status_code, _req.text)
+                )
+                _upload_success = True
+                break
+
             else:
                 self.log_error(
                     "Error uploading to Sondehub. Status Code: %d %s."

auto_rx/station.cfg.example

@@ -159,37 +159,61 @@ sondehub_contact_email = [email protected]
 ########################
 # APRS UPLOAD SETTINGS #
 ########################
-# Settings for uploading to APRS-IS
+# Settings for uploading to radiosondy.info
+#
+# IMPORTANT APRS NOTE
+#
+# As of auto_rx version 1.5.10, we are limiting APRS output to only radiosondy.info,
+# and only on the non-forwarding port. 
+# This decision was not made lightly, and is a result of the considerable amount of
+# non-amateur traffic that radiosonde flights are causing within the APRS-IS network.
+# Until some form of common format can be agreed to amongst the developers of *all* 
+# radiosonde tracking software to enable radiosonde telemetry to be de-duped, 
+# I have decided to help reduce the impact on the wider APRS-IS network by restricting 
+# the allowed servers and ports.
+# If you are using another APRS-IS server that *does not* forward to the wider APRS-IS
+# network and want it allowed, then please raise an issue at
+# https://github.com/projecthorus/radiosonde_auto_rx/issues
+#
+# You are of course free to fork and modify this codebase as you wish, but please be aware
+# that this goes against the wishes of the radiosonde_auto_rx developers to not be part
+# of the bigger problem of APRS-IS congestion. 
+# As of 2022-03-01, radiosonde traffic has been filtered from aprs.fi, so even if you do
+# modify the code, you still won't see sondes on that map.
+# APRS-IS is a *shared resource*, intended for the use of all amateur radio operators, and 
+# for many years we have been treating it as a playground to dump large amounts of non-amateur
+# traffic into, so we can see weather balloons on a map. 
+# Instead of congesting this shared resource with this non-amateur traffic, we should instead
+# be moving to using databases and sites specialised for this purpose, for example sondehub.org
+
 [aprs]
 # Enable APRS upload (you will also need to change some options below!)
 aprs_enabled = False
 
 # APRS-IS Login Information
 # The aprs_user field can have an SSID on the end if desired, i.e. N0CALL-4
-# If you are a licensed amateur radio operator, you may want to change the aprs_port number below 
-# to 14580, so that your uploaded telemetry makes its way out to the wider APRS network.
 aprs_user = N0CALL
 # APRS-IS Passcode. You can generate one for your callsign here: https://apps.magicbug.co.uk/passcode/
 aprs_pass = 00000
 
 # APRS Upload Rate - Upload a packet every X seconds.
-# This has a lower limit of 30 seconds, to avoid flooding the APRS-IS network.
-# Please be respectful of other uses of the APRS-IS network, and do not attempt
-# to upload faster than this. 
+# This has a lower limit of 30 seconds, to avoid flooding radiosondy.info
+# Please be respectful, and do not attempt to upload faster than this. 
 upload_rate = 30
 
 # APRS-IS server to upload to.
-# Default to radiosondy.info for now, to allow stats to show up on http://radiosondy.info
-# Packets are forwarded onto the rest of the APRS-IS network from radiosondy.info.
-# If you wish to inject packets directly into the APRS-IS network, use rotate.aprs2.net
+# Currently we only support uploading to radiosondy.info 
+# When using port 14580, packets are not forwarded to the wider APRS-IS network, and hence
+# are help reduce the huge amount of non-amateur traffic that ends up in APRS-IS from
+# radiosondes.
 aprs_server = radiosondy.info
 
 # APRS-IS Port Number to upload to.
-# When using radiosondy.info:
-#   Port 14590 - Packets stay within radiosondy.info. Non-licensed operators can use this.
-#   Port 14580 - Packets are forwarded out to the wider APRS-IS network. Only licensed amateur radio operators should use this!!
-# For all other APRS-IS servers (licensed amateur radio operators only!), use port 14580.
-aprs_port = 14590
+# 
+#   Port 14590 - Packets stay within radiosondy.info and do not congest the wider APRS-IS
+#                network.
+#
+aprs_port = 14580
 
 # APRS Station Location Beaconing
 # If enabled, you will show up on APRS using the aprs_user callsign set above.