From a4eab3cb999c52b7c94394b13723a29a9ba18952 Mon Sep 17 00:00:00 2001 From: Luke Prior <22492406+LukePrior@users.noreply.github.com> Date: Fri, 4 Feb 2022 21:06:37 +1100 Subject: [PATCH 1/8] Update Radiosonde Support Matrix --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cf2f5bea..33025acd 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,8 @@ Vaisala | RS41-SG/SGP/SGM | :heavy_check_mark: | :heavy_check_mark: | :heavy_che Graw | DFM06/09/17 | :heavy_check_mark: | :heavy_check_mark: | :x: | :x: | :x: Meteomodem | M10 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | Not Sent | :x: Meteomodem | M20 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: (For some models) | :x: -Intermet Systems | iMet-4 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | Not Sent | :x: +Intermet Systems | iMet-1 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | Not Sent | :heavy_check_mark: +Intermet Systems | iMet-4 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | Not Sent | :heavy_check_mark: Intermet Systems | iMet-54 | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | Not Sent | :x: Lockheed Martin | LMS6-400/1680 | :heavy_check_mark: | :x: | :x: | :x: | Not Sent Meisei | iMS-100 | :heavy_check_mark: | :x: | :x: | :x: | Not Sent From c4d227f4448da39dc31904c67344b2a2916ecb32 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Tue, 8 Feb 2022 18:19:18 +1030 Subject: [PATCH 2/8] Update DFM decoder, handle new subtype format --- auto_rx/autorx/__init__.py | 2 +- auto_rx/autorx/decode.py | 19 +- auto_rx/autorx/sondehub.py | 2 + demod/mod/dfm09mod.c | 766 ++++++++++++++++++++++++------------- 4 files changed, 511 insertions(+), 278 deletions(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 2c00724e..879e4999 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -17,7 +17,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.5.9" +__version__ = "1.5.10-beta1" # Global Variables diff --git a/auto_rx/autorx/decode.py b/auto_rx/autorx/decode.py index 84db559f..818467ae 100644 --- a/auto_rx/autorx/decode.py +++ b/auto_rx/autorx/decode.py @@ -808,8 +808,12 @@ def generate_decoder_command_experimental(self): ) # DFM decoder + if len(self.raw_file_option)>0: + # Use raw ecc detailed raw output for DFM sondes. + self.raw_file_option = "--rawecc" + decode_cmd = ( - f"./dfm09mod -vv --ecc --json --dist --auto --softin -i {self.raw_file_option.upper()} 2>/dev/null" + f"./dfm09mod -vv --ecc --json --dist --auto --softin -i {self.raw_file_option} 2>/dev/null" ) # DFM sondes transmit continuously - average over the last 2 frames, and peak hold @@ -1282,8 +1286,17 @@ def handle_decoder_line(self, data): # in the subtype field, so we can use this directly. _telemetry["type"] = _telemetry["subtype"] elif self.sonde_type == "DFM": - # For DFM sondes, we need to use a lookup to convert the subtype field into a model. - _telemetry["type"] = decode_dfm_subtype(_telemetry["subtype"]) + # As of 2021-2, the decoder provides a guess of the DFM subtype, provided as + # a subtype field of "0xX:GUESS", e.g. "0xD:DFM17P" + if ":" in _telemetry["subtype"]: + _subtype = _telemetry["subtype"].split(":")[1] + _telemetry["dfmcode"] = _telemetry["subtype"].split(":")[0] + _telemetry["type"] = _subtype + _telemetry["subtype"] = _subtype + else: + _telemetry["type"] = "DFM" + _telemetry["subtype"] = "DFM" + # Check frame ID here to ensure we are on dfm09mod version with the frame number fixes (2020-12). if _telemetry["frame"] < 256: diff --git a/auto_rx/autorx/sondehub.py b/auto_rx/autorx/sondehub.py index 35b18652..90501a1b 100644 --- a/auto_rx/autorx/sondehub.py +++ b/auto_rx/autorx/sondehub.py @@ -172,6 +172,8 @@ def reformat_data(self, telemetry): _output["type"] = "DFM" _output["subtype"] = telemetry["type"] _output["serial"] = telemetry["id"].split("-")[1] + if "dfmcode" in telemetry: + _output["dfmcode"] = telemetry["dfmcode"] elif telemetry["type"].startswith("M10") or telemetry["type"].startswith("M20"): _output["manufacturer"] = "Meteomodem" diff --git a/demod/mod/dfm09mod.c b/demod/mod/dfm09mod.c index 17795346..5054c59c 100644 --- a/demod/mod/dfm09mod.c +++ b/demod/mod/dfm09mod.c @@ -34,6 +34,28 @@ #include "demod_mod.h" +enum dfmtyp_keys_t { + UNDEF, + UNKNW, + DFM06, + PS15, + DFM09, + DFM09P, + DFM17, + DFM17P +}; + +static char *DFM_types[] = { + [UNDEF] = "", + [UNKNW] = "DFMxX", + [DFM06] = "DFM06", + [PS15] = "PS15", + [DFM09] = "DFM09", + [DFM09P] = "DFM09P", + [DFM17] = "DFM17", + [DFM17P] = "DFM17P" +}; + typedef struct { i8_t vbs; // verbose output i8_t raw; // raw frames @@ -75,14 +97,19 @@ typedef struct { int sonde_typ; ui32_t SN6; ui32_t SN; - int week; int gpssec; + int week; int tow; ui32_t sec_gps; int jahr; int monat; int tag; int std; int min; float sek; double lat; double lon; double alt; double dir; double horiV; double vertV; - float meas24[5+2]; - float status[2]; + //float T; + float Rf; float _frmcnt; + float meas24[9]; + float status[2]; + ui32_t val24[9]; + ui8_t cfgchk24[9]; + int cfgchk; char sonde_id[16]; // "ID__:xxxxxxxx\0\0" hsbit_t frame[BITFRAME_LEN+4]; // char frame_bits[BITFRAME_LEN+4]; char dat_str[9][13+1]; @@ -90,8 +117,12 @@ typedef struct { pcksts_t pck[9]; option_t option; int ptu_out; + char sensortyp0xC; + char *dfmtyp; int jsn_freq; // freq/kHz (SDR) gpsdat_t gps; + int prev_cntsec_diff; + int prev_manpol; } gpx_t; @@ -434,24 +465,26 @@ static float get_Temp(gpx_t *gpx) { // meas[0..4] // meas0 = g*(R + Rs) // meas3 = g*Rs , Rs: dfm6:10k, dfm9:20k // meas4 = g*Rf , Rf=220k + float T = 0; // T/Kelvin float f = gpx->meas24[0], f1 = gpx->meas24[3], f2 = gpx->meas24[4]; - if (gpx->ptu_out >= 0xC) { + if (gpx->sensortyp0xC == 'P') { // 0xC: "P+" DFM-09P , "T-" DFM-17TU ; 0xD: "P-" DFM-17P ? f = gpx->meas24[0+1]; f1 = gpx->meas24[3+2]; f2 = gpx->meas24[4+2]; } - //float *meas = gpx->meas24; - float B0 = 3260.0; // B/Kelvin, fit -55C..+40C - float T0 = 25 + 273.15; // t0=25C - float R0 = 5.0e3; // R0=R25=5k - float Rf = 220e3; // Rf = 220k - float g = f2/Rf; - float R = (f-f1) / g; // meas[0,3,4] > 0 ? - float T = 0; // T/Kelvin - if (f*f1*f2 == 0) R = 0; - if (R > 0) T = 1/(1/T0 + 1/B0 * log(R/R0)); + if (gpx->cfgchk) { + //float *meas = gpx->meas24; + float B0 = 3260.0; // B/Kelvin, fit -55C..+40C + float T0 = 25 + 273.15; // t0=25C + float R0 = 5.0e3; // R0=R25=5k + float Rf = gpx->Rf; // Rf = DFM09:220k , DFM17:332k + float g = f2/Rf; + float R = (f-f1) / g; // meas[0,3,4] > 0 ? + if (f*f1*f2 == 0) R = 0; + if (R > 0) T = 1/(1/T0 + 1/B0 * log(R/R0)); + } return T - 273.15; // Celsius // DFM-06: meas20 * 16 = meas24 // -> (meas24[0]-meas24[3])/meas24[4]=(meas20[0]-meas20[3])/meas20[4] @@ -468,7 +501,7 @@ static float get_Temp2(gpx_t *gpx) { // meas[0..4] float f = gpx->meas24[0], f1 = gpx->meas24[3], f2 = gpx->meas24[4]; - if (gpx->ptu_out >= 0xC) { + if (gpx->ptu_out >= 0xC && gpx->meas24[6] < 220e3) { f = gpx->meas24[0+1]; f1 = gpx->meas24[3+2]; f2 = gpx->meas24[4+2]; @@ -476,7 +509,7 @@ static float get_Temp2(gpx_t *gpx) { // meas[0..4] float B0 = 3260.0; // B/Kelvin, fit -55C..+40C float T0 = 25 + 273.15; // t0=25C float R0 = 5.0e3; // R0=R25=5k - float Rf2 = 220e3; // Rf2 = Rf = 220k + float Rf2 = 220e3; // Rf2 = Rf = DFM09:220k , DFM17:332k float g_o = f2/Rf2; // approx gain float Rs_o = f1/g_o; // = Rf2 * f1/f2; float Rf1 = Rs_o; // Rf1 = Rs: dfm6:10k, dfm9:20k @@ -494,7 +527,7 @@ static float get_Temp2(gpx_t *gpx) { // meas[0..4] R = (f-f1)/g; // meas[0,3,4] > 0 ? if (R > 0) T = 1/(1/T0 + 1/B0 * log(R/R0)); - if (gpx->option.ptu && gpx->ptu_out && gpx->option.dbg) { + if (gpx->option.ptu && gpx->ptu_out && gpx->option.dbg && gpx->option.vbs == 3) { printf(" (Rso: %.1f , Rb: %.1f)", Rs_o/1e3, Rb/1e3); } @@ -533,13 +566,13 @@ static float get_Temp4(gpx_t *gpx) { // meas[0..4] float f = gpx->meas24[0], f1 = gpx->meas24[3], f2 = gpx->meas24[4]; - if (gpx->ptu_out >= 0xC) { + if (gpx->ptu_out >= 0xC && gpx->meas24[6] < 220e3) { f = gpx->meas24[0+1]; f1 = gpx->meas24[3+2]; f2 = gpx->meas24[4+2]; } //float *meas = gpx->meas24; - float Rf = 220e3; // Rf = 220k + float Rf = 220e3; // Rf = DFM09:220k , DFM17:332k float g = f2/Rf; float R = (f-f1) / g; // f,f1,f2 > 0 ? float T = 0; // T/Kelvin @@ -550,6 +583,14 @@ static float get_Temp4(gpx_t *gpx) { // meas[0..4] } +static int reset_cfgchk(gpx_t *gpx) { + int j; + for (j = 0; j < 9; j++) gpx->cfgchk24[j] = 0; + gpx->cfgchk = 0; + gpx->ptu_out = 0; + return 0; +} + #define SNbit 0x0100 static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { int ret = 0; @@ -571,17 +612,18 @@ static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { { // reset if 0x5A, 0x5B (DFM-06) gpx->sonde_typ = 0; gpx->snc.max_ch = conf_id; + reset_cfgchk(gpx); } if (conf_id > 5 && conf_id > gpx->snc.max_ch && ec == 0) { // mind. 6 Kanaele if (bits2val(conf_bits+4, 4) == 0xC) { // 0xsCaaaab gpx->snc.max_ch = conf_id; // reset? } -/* + /* if (bits2val(conf_bits, 8) == 0x70) { // 0x70aaaab gpx->snc.max_ch = conf_id; // reset? } -*/ + */ } // SN: mind. 6 Kanaele @@ -595,10 +637,11 @@ static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { if (SN6 == gpx->SN6 && SN6 != 0) { // nur Nibble-Werte 0..9 gpx->sonde_typ = SNbit | 6; gpx->ptu_out = 6; // <-> DFM-06 - sprintf(gpx->sonde_id, "ID06:%6X", gpx->SN6); + sprintf(gpx->sonde_id, "IDx%1X:%6X", gpx->sonde_typ & 0xF, gpx->SN6); } else { // reset gpx->sonde_typ = 0; + reset_cfgchk(gpx); } gpx->SN6 = SN6; } // SN in last pck/channel, #{pcks} depends on (sensor) config; observed: @@ -614,6 +657,7 @@ static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { gpx->snc.chXbit = 0; gpx->snc.chX[0] = 0; gpx->snc.chX[1] = 0; + reset_cfgchk(gpx); } gpx->snc.sn_ch = sn_ch; gpx->snc.chX[hl] = (val >> 4) & 0xFFFF; @@ -628,19 +672,17 @@ static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { gpx->ptu_out = 0; if (sn_ch == 0xA /*&& (sn2_ch & 0xF) == 0xC*/) gpx->ptu_out = sn_ch; // <+> DFM-09 if (sn_ch == 0xB /*&& (sn2_ch & 0xF) == 0xC*/) gpx->ptu_out = sn_ch; // <-> DFM-17 - if (sn_ch == 0xC) gpx->ptu_out = sn_ch; // <+> DFM-09P(?) - if (sn_ch == 0xD) gpx->ptu_out = sn_ch; // <-> DFM-17P? + if (sn_ch == 0xC) gpx->ptu_out = sn_ch; // <+> DFM-09P(?) , <-> DFM-17TU(?) + if (sn_ch == 0xD) gpx->ptu_out = sn_ch; // <-> DFM-17P(?) // PS-15 ? (sn2_ch & 0xF) == 0x0 : gpx->ptu_out = 0 // <-> PS-15 - if ( (gpx->sonde_typ & 0xF) == 0xA) { - sprintf(gpx->sonde_id, "ID09:%6u", gpx->SN); - } - else { - sprintf(gpx->sonde_id, "ID-%1X:%6u", gpx->sonde_typ & 0xF, gpx->SN); + if ( (gpx->sonde_typ & 0xF) > 6) { + sprintf(gpx->sonde_id, "IDx%1X:%6u", gpx->sonde_typ & 0xF, gpx->SN); } } else { // reset gpx->sonde_typ = 0; + reset_cfgchk(gpx); } gpx->snc.SN_X = SN; gpx->snc.chXbit = 0; @@ -651,41 +693,80 @@ static int conf_out(gpx_t *gpx, ui8_t *conf_bits, int ec) { } - if (conf_id >= 0 && conf_id <= 4) { + if (conf_id >= 0 && conf_id <= 8) { + gpx->cfgchk24[conf_id] = 1; val = bits2val(conf_bits+4, 4*6); - gpx->meas24[conf_id] = fl24(val); + gpx->val24[conf_id] = val; + gpx->meas24[conf_id] = fl24(val); //0xA: 0..4 // DFM-09 (STM32): 24bit 0exxxxx // DFM-06 (NXP8): 20bit 0exxxx0 // fl20(bits2val(conf_bits+4, 4*5)) // = fl20(exxxx) // = fl24(exxxx0)/2^4 // meas20 * 16 = meas24 - } - if (gpx->ptu_out >= 0xC) { // DFM>=09(P) - if (conf_id >= 5 && conf_id <= 6) { - val = bits2val(conf_bits+4, 4*6); - gpx->meas24[conf_id] = fl24(val); - } + gpx->cfgchk = 0; + if (gpx->ptu_out >= 0x5) gpx->cfgchk = gpx->cfgchk24[0]*gpx->cfgchk24[1]*gpx->cfgchk24[2] + *gpx->cfgchk24[3]*gpx->cfgchk24[4]*gpx->cfgchk24[5]; + if (gpx->ptu_out >= 0x7) gpx->cfgchk *= gpx->cfgchk24[6]*gpx->cfgchk24[7]; + if (gpx->ptu_out >= 0x8) gpx->cfgchk *= gpx->cfgchk24[8]; } - // STM32-status: Bat, MCU-Temp - if (gpx->ptu_out >= 0xA) { // DFM>=09(P) (STM32) - ui8_t ofs = 0; - if (gpx->ptu_out >= 0xC) ofs = 2; - if (conf_id == 0x5+ofs) { // voltage - val = bits2val(conf_bits+8, 4*4); - gpx->status[0] = val/1000.0; + gpx->sensortyp0xC = 'T'; + gpx->Rf = 220e3; + if (gpx->cfgchk) + { // 0xC: "P+" DFM-09P , "T-" DFM-17TU ; 0xD: "P-" DFM-17P ? + if (gpx->ptu_out >= 0xD || (gpx->ptu_out >= 0xC && gpx->meas24[6] < 220e3)) { // gpx->meas24[6] < 220e3 <=> gpx->meas24[0] > 1e6 ? + gpx->sensortyp0xC = 'P'; // gpx->meas24[0] > 1e6 ? } - if (conf_id == 0x6+ofs) { // T-intern (STM32) - val = bits2val(conf_bits+8, 4*4); - gpx->status[1] = val/100.0; + if ( ((gpx->ptu_out == 0xB || gpx->ptu_out == 0xC) && gpx->sensortyp0xC == 'T') || gpx->ptu_out >= 0xD) gpx->Rf = 332e3; // DFM-17 ? + + // STM32-status: Bat, MCU-Temp + if (gpx->ptu_out >= 0xA) { // DFM>=09(P) (STM32) + ui8_t ofs = 0; + if (gpx->sensortyp0xC == 'P') ofs = 2; + // + if (conf_id == 0x5+ofs) { // voltage + val = bits2val(conf_bits+8, 4*4); + gpx->status[0] = val/1000.0; + } + if (conf_id == 0x6+ofs) { // T-intern (STM32) + val = bits2val(conf_bits+8, 4*4); + gpx->status[1] = val/100.0; + } + } + else { + gpx->status[0] = 0; + gpx->status[1] = 0; } - } - else { - gpx->status[0] = 0; - gpx->status[1] = 0; } + /* guess DFM type + V/Ti Tf012 Rf + 0xA DFM-09 5/6 0,3,4 'T+' 220k + 0xC DFM-09P 7/8 1,5,6 'P+' 220k + 0xB DFM-17 5/6 0,3,4 'T-' 332k + 0xC DFM-17TU 5/6 0,3,4 'T-' 332k + 0xD DFM-17P 7/8 1,5,6 'P-' 332k + */ + gpx->dfmtyp = DFM_types[UNDEF]; + switch (gpx->sonde_typ & 0xF) { + case 0x6: gpx->dfmtyp = DFM_types[DFM06]; + break; + case 0x7: + case 0x8: gpx->dfmtyp = DFM_types[PS15]; + break; + case 0xA: gpx->dfmtyp = DFM_types[DFM09]; + break; + case 0xB: gpx->dfmtyp = DFM_types[DFM17]; + break; + case 0xC: if (gpx->sensortyp0xC == 'P') gpx->dfmtyp = DFM_types[DFM09P]; + else /*'T'*/ gpx->dfmtyp = DFM_types[DFM17]; + break; + case 0xD: gpx->dfmtyp = DFM_types[DFM17P]; + break; + default: gpx->dfmtyp = DFM_types[UNKNW]; + break; + } return ret; } @@ -721,6 +802,29 @@ static void print_gpx(gpx_t *gpx) { jsonout = 0; } + if (!gpx->option.raw || gpx->option.jsn) { + + // seconds since GPS (ignoring leap seconds, DFM=UTC) + datetime2GPSweek(gpx->jahr, gpx->monat, gpx->tag, gpx->std, gpx->min, (int)(gpx->sek+0.5), &(gpx->week), &(gpx->tow)); + gpx->sec_gps = gpx->week*604800 + gpx->tow; // SECONDS_IN_WEEK=7*86400=604800 + + if (contgps) { + ui8_t secmod256 = (ui8_t)gpx->sec_gps; // % 256 + int cntsec_diff = secmod256 - gpx->frnr; + if (cntsec_diff < 0) cntsec_diff += 256; + // DFM06: cntsec_diff might drift slowly (30sec sync), but recovers faster + // DFM09: delta(diff)=1 could indicate decoding error + if (gpx->option.jsn && (cntsec_diff != gpx->prev_cntsec_diff || gpx->option.inv != gpx->prev_manpol)) { + // initial state not relevant + jsonout = 0; + gpx->sonde_typ = 0; + reset_cfgchk(gpx); + } + gpx->prev_cntsec_diff = cntsec_diff; + gpx->prev_manpol = gpx->option.inv; + } + } + if (output & 0xF000) { if (gpx->option.raw == 2) { @@ -731,8 +835,9 @@ static void print_gpx(gpx_t *gpx) { for (i = 0; i < 9; i++) { for (j = 0; j < 13; j++) gpx->dat_str[i][j] = ' '; } + printf("\n"); } - else { + else if (!gpx->option.raw) { if (gpx->option.aut && gpx->option.vbs >= 2) printf("<%c> ", gpx->option.inv?'-':'+'); printf("[%3d] ", gpx->frnr); printf("%4d-%02d-%02d ", gpx->jahr, gpx->monat, gpx->tag); @@ -745,73 +850,79 @@ static void print_gpx(gpx_t *gpx) { printf(" vH: %5.2f ", gpx->horiV); printf(" D: %5.1f ", gpx->dir); printf(" vV: %5.2f ", gpx->vertV); - if (gpx->option.ptu && gpx->ptu_out) { - float t = get_Temp(gpx); - if (t > -270.0) printf(" T=%.1fC ", t); - if (gpx->option.dbg) { - float t2 = get_Temp2(gpx); - float t4 = get_Temp4(gpx); - if (t2 > -270.0) printf(" T2=%.1fC ", t2); - if (t4 > -270.0) printf(" T4=%.1fC ", t4); - printf(" f0: %.2f ", gpx->meas24[0]); - printf(" f1: %.2f ", gpx->meas24[1]); - printf(" f2: %.2f ", gpx->meas24[2]); - printf(" f3: %.2f ", gpx->meas24[3]); - printf(" f4: %.2f ", gpx->meas24[4]); - if (gpx->ptu_out >= 0xC) { - printf(" f5: %.2f ", gpx->meas24[5]); - printf(" f6: %.2f ", gpx->meas24[6]); - } + if (gpx->cfgchk) + { + if (gpx->option.ptu && gpx->ptu_out) { + float t = get_Temp(gpx); + if (t > -270.0) { + printf(" T=%.1fC ", t); // 0xC:P+ DFM-09P , 0xC:T- DFM-17TU , 0xD:P- DFM-17P ? + if (gpx->option.vbs == 3) printf(" (0x%X:%c%c) ", gpx->sonde_typ & 0xF, gpx->sensortyp0xC, gpx->option.inv?'-':'+'); + } + if (gpx->option.dbg) { + float t2 = get_Temp2(gpx); + float t4 = get_Temp4(gpx); + if (t2 > -270.0) printf(" T2=%.1fC ", t2); + if (t4 > -270.0) printf(" T4=%.1fC ", t4); + } + } + if (gpx->option.vbs == 3 && gpx->ptu_out >= 0xA) { + if (gpx->status[0]> 0.0) printf(" U: %.2fV ", gpx->status[0]); + if (gpx->status[1]> 0.0) printf(" Ti: %.1fK ", gpx->status[1]); } } - if (gpx->option.vbs == 3 && gpx->ptu_out >= 0xA) { - printf(" U: %.2fV ", gpx->status[0]); - printf(" Ti: %.1fK ", gpx->status[1]); + if (gpx->option.dbg) { + printf(" f0:%.1f", gpx->meas24[0]); + printf(" f1:%.1f", gpx->meas24[1]); + printf(" f2:%.1f", gpx->meas24[2]); + printf(" f3:%.1f", gpx->meas24[3]); + printf(" f4:%.1f", gpx->meas24[4]); + if (gpx->ptu_out >= 0xA /*0xC*/) { + printf(" f5:%.1f", gpx->meas24[5]); + printf(" f6:%.1f", gpx->meas24[6]); + } + printf(" "); } if (gpx->option.vbs) { if (gpx->sonde_typ & SNbit) { - printf(" (%s) ", gpx->sonde_id); + printf(" (%s", gpx->sonde_id); + if (gpx->option.vbs > 1 && *gpx->dfmtyp) printf(":%s", gpx->dfmtyp); + printf(") "); gpx->sonde_typ ^= SNbit; } } - } - printf("\n"); - - if (gpx->option.sat) { - printf(" "); - printf(" dMSL: %+.2f", gpx->gps.dMSL); // MSL = alt + gps.dMSL - printf(" sats: %d", gpx->gps.nSV); - printf(" ("); - for (j = 0; j < 32; j++) { if ((gpx->gps.prn >> j)&1) printf(" %02d", j+1); } - printf(" )"); printf("\n"); + + if (gpx->option.sat) { + printf(" "); + printf(" dMSL: %+.2f", gpx->gps.dMSL); // MSL = alt + gps.dMSL + printf(" sats: %d", gpx->gps.nSV); + printf(" ("); + for (j = 0; j < 32; j++) { if ((gpx->gps.prn >> j)&1) printf(" %02d", j+1); } + printf(" )"); + printf("\n"); + } } if (gpx->option.jsn && jsonout && gpx->sek < 60.0) { char *ver_jsn = NULL; - unsigned long sec_gps = 0; - int week = 0; - int tow = 0; char json_sonde_id[] = "DFM-xxxxxxxx\0\0"; - ui8_t dfm_typ = (gpx->sonde_typ & 0xF); - switch ( dfm_typ ) { + ui8_t dfmXtyp = (gpx->sonde_typ & 0xF); + switch ( dfmXtyp ) { case 0: sprintf(json_sonde_id, "DFM-xxxxxxxx"); break; //json_sonde_id[0] = '\0'; case 6: sprintf(json_sonde_id, "DFM-%6X", gpx->SN6); break; // DFM-06 case 0xA: sprintf(json_sonde_id, "DFM-%6u", gpx->SN); break; // DFM-09 - // 0x7:PS-15?, 0xB:DFM-17? 0xC:DFM-09P? 0xD:DFM-17P? + // 0x7:PS-15?, 0xB:DFM-17? 0xC:DFM-09P?DFM-17TU? 0xD:DFM-17P? default : sprintf(json_sonde_id, "DFM-%6u", gpx->SN); } - // JSON frame counter: seconds since GPS (ignoring leap seconds, DFM=UTC) - datetime2GPSweek(gpx->jahr, gpx->monat, gpx->tag, gpx->std, gpx->min, (int)(gpx->sek+0.5), &week, &tow); - sec_gps = week*604800 + tow; // SECONDS_IN_WEEK=7*86400=604800 + // JSON frame counter: gpx->sec_gps , seconds since GPS (ignoring leap seconds, DFM=UTC) // Print JSON blob // valid sonde_ID? printf("{ \"type\": \"%s\"", "DFM"); - printf(", \"frame\": %lu, ", sec_gps); // gpx->frnr + printf(", \"frame\": %u, ", gpx->sec_gps); // gpx->frnr printf("\"id\": \"%s\", \"datetime\": \"%04d-%02d-%02dT%02d:%02d:%06.3fZ\", \"lat\": %.5f, \"lon\": %.5f, \"alt\": %.5f, \"vel_h\": %.5f, \"heading\": %.5f, \"vel_v\": %.5f, \"sats\": %d", json_sonde_id, gpx->jahr, gpx->monat, gpx->tag, gpx->std, gpx->min, gpx->sek, gpx->lat, gpx->lon, gpx->alt, gpx->horiV, gpx->dir, gpx->vertV, gpx->gps.nSV); if (gpx->ptu_out >= 0xA && gpx->status[0] > 0) { // DFM>=09(P): Battery (STM32) @@ -821,7 +932,12 @@ static void print_gpx(gpx_t *gpx) { float t = get_Temp(gpx); // ecc-valid temperature? if (t > -270.0) printf(", \"temp\": %.1f", t); } - if (dfm_typ > 0) printf(", \"subtype\": \"0x%1X\"", dfm_typ); + //if (dfmXtyp > 0) printf(", \"subtype\": \"0x%1X\"", dfmXtyp); + if (dfmXtyp > 0) { + printf(", \"subtype\": \"0x%1X", dfmXtyp); + if (*gpx->dfmtyp) printf(":%s", gpx->dfmtyp); + printf("\""); + } if (gpx->jsn_freq > 0) { printf(", \"freq\": %d", gpx->jsn_freq); } @@ -862,7 +978,28 @@ static int print_frame(gpx_t *gpx) { ret2 = hamming(gpx->option.ecc, hamming_dat2, 13, block_dat2); ret = ret0 | ret1 | ret2; - if (gpx->option.raw == 1) { + if (gpx->option.raw == 9) { + int diff = 0; + for (i = 0; i < CONF; i++) { + diff += ( (gpx->frame[i].hb & 1) != (dfm_header[i] & 1) ); + } + if (diff == 0) + { + ui8_t byte = 0; + printf("%c", gpx->option.inv?'-':'+'); + printf("<%7.1f> ", gpx->_frmcnt); + for (i = CONF; i < BITFRAME_LEN; i++) { + if (i == DAT1 || i == DAT2) printf(" "); + byte |= (gpx->frame[i].hb&1)<<(i%4); // little endian + if (i % 4 == 3) { + printf("%1X", byte & 0xF); + byte = 0; + } + } + printf("\n"); + } + } + else if (gpx->option.raw == 1) { for (i = 0; i < 7; i++) { nib = bits2val(block_conf+S*i, S); @@ -902,29 +1039,32 @@ static int print_frame(gpx_t *gpx) { printf("\n"); } - else if (gpx->option.ecc) { + if ( (gpx->option.raw&1) != 1 || gpx->option.jsn ) + { + if (gpx->option.ecc) { + + if (ret0 == 0 || ret0 > 0 || gpx->option.ecc == 2) { + conf_out(gpx, block_conf, ret0); + } + if (ret1 == 0 || ret1 > 0 || gpx->option.ecc == 2) { + frid = dat_out(gpx, block_dat1, ret1); + if (frid == 8) print_gpx(gpx); + } + if (ret2 == 0 || ret2 > 0 || gpx->option.ecc == 2) { + frid = dat_out(gpx, block_dat2, ret2); + if (frid == 8) print_gpx(gpx); + } - if (ret0 == 0 || ret0 > 0 || gpx->option.ecc == 2) { - conf_out(gpx, block_conf, ret0); } - if (ret1 == 0 || ret1 > 0 || gpx->option.ecc == 2) { + else { + + conf_out(gpx, block_conf, ret0); frid = dat_out(gpx, block_dat1, ret1); if (frid == 8) print_gpx(gpx); - } - if (ret2 == 0 || ret2 > 0 || gpx->option.ecc == 2) { frid = dat_out(gpx, block_dat2, ret2); if (frid == 8) print_gpx(gpx); - } - - } - else { - - conf_out(gpx, block_conf, ret0); - frid = dat_out(gpx, block_dat1, ret1); - if (frid == 8) print_gpx(gpx); - frid = dat_out(gpx, block_dat2, ret2); - if (frid == 8) print_gpx(gpx); + } } return ret; @@ -947,6 +1087,7 @@ int main(int argc, char **argv) { int option_iqdc = 0; int option_lp = 0; int option_dc = 0; + int option_noLUT = 0; int option_bin = 0; int option_softin = 0; int option_json = 0; // JSON blob output (for auto_rx) @@ -954,6 +1095,7 @@ int main(int argc, char **argv) { int wavloaded = 0; int sel_wavch = 0; // audio channel: left int spike = 0; + int rawhex = 0; int cfreq = -1; FILE *fp = NULL; @@ -1021,6 +1163,9 @@ int main(int argc, char **argv) { else if ( (strcmp(*argv, "-R") == 0) || (strcmp(*argv, "--RAW") == 0) ) { option_raw = 2; } + else if ( (strcmp(*argv, "--rawecc") == 0) ) { + option_raw = 9; + } else if ( (strcmp(*argv, "-i") == 0) || (strcmp(*argv, "--invert") == 0) ) { option_inv = 0x1; } @@ -1076,21 +1221,24 @@ int main(int argc, char **argv) { dsp.xlt_fq = -fq; // S(t) -> S(t)*exp(-f*2pi*I*t) option_iq = 5; } - else if (strcmp(*argv, "--lp") == 0) { option_lp = 1; } // IQ lowpass - else if (strcmp(*argv, "--dc") == 0) { option_dc = 1; } - else if (strcmp(*argv, "--min") == 0) { - option_min = 1; - } - else if (strcmp(*argv, "--dbg") == 0) { gpx.option.dbg = 1; } + else if (strcmp(*argv, "--lpIQ") == 0) { option_lp |= LP_IQ; } // IQ/IF lowpass else if (strcmp(*argv, "--lpbw") == 0) { // IQ lowpass BW / kHz double bw = 0.0; ++argv; if (*argv) bw = atof(*argv); else return -1; if (bw > 4.6 && bw < 24.0) lpIQ_bw = bw*1e3; - option_lp = 1; + option_lp |= LP_IQ; } + else if (strcmp(*argv, "--lpFM") == 0) { option_lp |= LP_FM; } // FM lowpass + else if (strcmp(*argv, "--dc") == 0) { option_dc = 1; } + else if (strcmp(*argv, "--noLUT") == 0) { option_noLUT = 1; } + else if (strcmp(*argv, "--min") == 0) { + option_min = 1; + } + else if (strcmp(*argv, "--dbg") == 0) { gpx.option.dbg = 1; } else if (strcmp(*argv, "--sat") == 0) { gpx.option.sat = 1; } + else if (strcmp(*argv, "--rawhex") == 0) { rawhex = 1; } // raw hex input else if (strcmp(*argv, "-") == 0) { int sample_rate = 0, bits_sample = 0, channels = 0; ++argv; @@ -1119,6 +1267,14 @@ int main(int argc, char **argv) { } if (!wavloaded) fp = stdin; + if (option_iq == 5 && option_dc) option_lp |= LP_FM; + + // LUT faster for decM, however frequency correction after decimation + // LUT recommonded if decM > 2 + // + if (option_noLUT && option_iq == 5) dsp.opt_nolut = 1; else dsp.opt_nolut = 0; + + // ecc2-soft_decision accepts also 2-error words, // so the probability for 3 errors is high and will // produce wrong codewords. hence ecc2 is not recommended @@ -1167,196 +1323,258 @@ int main(int argc, char **argv) { } #endif - if (!option_bin && !option_softin) { - if (option_iq == 0 && option_pcmraw) { - fclose(fp); - fprintf(stderr, "error: raw data not IQ\n"); - return -1; - } - if (option_iq) sel_wavch = 0; + if (!rawhex) { - pcm.sel_ch = sel_wavch; - if (option_pcmraw == 0) { - k = read_wav_header(&pcm, fp); - if ( k < 0 ) { + if (!option_bin && !option_softin) { + + if (option_iq == 0 && option_pcmraw) { fclose(fp); - fprintf(stderr, "error: wav header\n"); + fprintf(stderr, "error: raw data not IQ\n"); return -1; } - } + if (option_iq) sel_wavch = 0; + + pcm.sel_ch = sel_wavch; + if (option_pcmraw == 0) { + k = read_wav_header(&pcm, fp); + if ( k < 0 ) { + fclose(fp); + fprintf(stderr, "error: wav header\n"); + return -1; + } + } - if (cfreq > 0) { - int fq_kHz = (cfreq - dsp.xlt_fq*pcm.sr + 500)/1e3; - gpx.jsn_freq = fq_kHz; - } + if (cfreq > 0) { + int fq_kHz = (cfreq - dsp.xlt_fq*pcm.sr + 500)/1e3; + gpx.jsn_freq = fq_kHz; + } - // dfm: BT=1?, h=2.4? - symlen = 2; - - // init dsp - // - dsp.fp = fp; - dsp.sr = pcm.sr; - dsp.bps = pcm.bps; - dsp.nch = pcm.nch; - dsp.ch = pcm.sel_ch; - dsp.br = (float)BAUD_RATE; - dsp.sps = (float)dsp.sr/dsp.br; - dsp.symlen = symlen; - dsp.symhd = symlen; - dsp._spb = dsp.sps*symlen; - dsp.hdr = dfm_rawheader; - dsp.hdrlen = strlen(dfm_rawheader); - dsp.BT = 0.5; // bw/time (ISI) // 0.3..0.5 - dsp.h = 1.8; // 2.4 modulation index abzgl. BT - dsp.opt_iq = option_iq; - dsp.opt_iqdc = option_iqdc; - dsp.opt_lp = option_lp; - dsp.lpIQ_bw = lpIQ_bw; // 12e3; // IF lowpass bandwidth - dsp.lpFM_bw = 4e3; // FM audio lowpass - dsp.opt_dc = option_dc; - dsp.opt_IFmin = option_min; - - if ( dsp.sps < 8 ) { - fprintf(stderr, "note: sample rate low\n"); - } + // dfm: BT=1?, h=2.4? + symlen = 2; + + // init dsp + // + dsp.fp = fp; + dsp.sr = pcm.sr; + dsp.bps = pcm.bps; + dsp.nch = pcm.nch; + dsp.ch = pcm.sel_ch; + dsp.br = (float)BAUD_RATE; + dsp.sps = (float)dsp.sr/dsp.br; + dsp.symlen = symlen; + dsp.symhd = symlen; + dsp._spb = dsp.sps*symlen; + dsp.hdr = dfm_rawheader; + dsp.hdrlen = strlen(dfm_rawheader); + dsp.BT = 0.5; // bw/time (ISI) // 0.3..0.5 + dsp.h = 1.8; // 2.4 modulation index abzgl. BT + dsp.opt_iq = option_iq; + dsp.opt_iqdc = option_iqdc; + dsp.opt_lp = option_lp; + dsp.lpIQ_bw = lpIQ_bw; // 12e3; // IF lowpass bandwidth + dsp.lpFM_bw = 4e3; // FM audio lowpass + dsp.opt_dc = option_dc; + dsp.opt_IFmin = option_min; + + if ( dsp.sps < 8 ) { + fprintf(stderr, "note: sample rate low\n"); + } - k = init_buffers(&dsp); - if ( k < 0 ) { - fprintf(stderr, "error: init buffers\n"); - return -1; - } + k = init_buffers(&dsp); + if ( k < 0 ) { + fprintf(stderr, "error: init buffers\n"); + return -1; + } - bitofs += shift; - } - else { - if (option_bin && option_softin) option_bin = 0; - // init circular header bit buffer - hdb.hdr = dfm_rawheader; - hdb.len = strlen(dfm_rawheader); - hdb.thb = 1.0 - 2.1/(float)hdb.len; // 1.0-max_bit_errors/hdrlen // max 1.1 !! - hdb.bufpos = -1; - hdb.buf = calloc(hdb.len, sizeof(char)); - if (hdb.buf == NULL) { - fprintf(stderr, "error: malloc\n"); - return -1; + bitofs += shift; } - hdb.ths = 0.7; // caution/test false positive - hdb.sbuf = calloc(hdb.len, sizeof(float)); - if (hdb.sbuf == NULL) { - fprintf(stderr, "error: malloc\n"); - return -1; + else { + if (option_bin && option_softin) option_bin = 0; + // init circular header bit buffer + hdb.hdr = dfm_rawheader; + hdb.len = strlen(dfm_rawheader); + hdb.thb = 1.0 - 2.1/(float)hdb.len; // 1.0-max_bit_errors/hdrlen // max 1.1 !! + hdb.bufpos = -1; + hdb.buf = calloc(hdb.len, sizeof(char)); + if (hdb.buf == NULL) { + fprintf(stderr, "error: malloc\n"); + return -1; + } + hdb.ths = 0.7; // caution/test false positive + hdb.sbuf = calloc(hdb.len, sizeof(float)); + if (hdb.sbuf == NULL) { + fprintf(stderr, "error: malloc\n"); + return -1; + } } - } - while ( 1 ) - { - if (option_bin) { // aka find_binrawhead() - header_found = find_binhead(fp, &hdb, &_mv); // symbols or bits? - hdrcnt += nfrms; - } - else if (option_softin) { - header_found = find_softbinhead(fp, &hdb, &_mv); - hdrcnt += nfrms; - } - else { //2 (false positive) // FM-audio: - header_found = find_header(&dsp, thres, 2, bitofs, dsp.opt_dc); // optional 2nd pass: dc=0 - _mv = dsp.mv; - } - if (header_found == EOF) break; + while ( 1 ) + { + if (option_bin) { // aka find_binrawhead() + header_found = find_binhead(fp, &hdb, &_mv); // symbols or bits? + hdrcnt += nfrms; + } + else if (option_softin) { + header_found = find_softbinhead(fp, &hdb, &_mv); + hdrcnt += nfrms; + } + else { //2 (false positive) // FM-audio: + header_found = find_header(&dsp, thres, 2, bitofs, dsp.opt_dc); // optional 2nd pass: dc=0 + _mv = dsp.mv; + } + if (header_found == EOF) break; - // mv == correlation score - if (_mv *(0.5-gpx.option.inv) < 0) { - if (gpx.option.aut == 0) header_found = 0; - else gpx.option.inv ^= 0x1; - } + // mv == correlation score + if (_mv *(0.5-gpx.option.inv) < 0) { + if (gpx.option.aut == 0) header_found = 0; + else gpx.option.inv ^= 0x1; + } - if (header_found) - { - bitpos = 0; - pos = headerlen; - pos /= 2; + if (header_found) + { + bitpos = 0; + pos = headerlen; + pos /= 2; - //if (fabs(mv) > 0.85) nfrms = 8; else nfrms = 4; // test OK/KO/NO count + //if (fabs(mv) > 0.85) nfrms = 8; else nfrms = 4; // test OK/KO/NO count - frm = 0; - while ( frm < nfrms ) { // nfrms=1,2,4,8 - if (option_bin || option_softin) { - gpx._frmcnt = hdrcnt + frm; - } - else { - gpx._frmcnt = dsp.mv_pos/(2.0*dsp.sps*BITFRAME_LEN) + frm; - } - while ( pos < BITFRAME_LEN ) - { - if (option_bin) { - // symbols or bits? - // manchester1 1->10,0->01: 1.bit (DFM-06) - // manchester2 0->10,1->01: 2.bit (DFM-09) - bitQ = fgetc(fp); - if (bitQ != EOF) { - hsbit.hb = bitQ & 0x1; - bitQ = fgetc(fp); // check: rbit0^rbit1=1 (Manchester) - if (bitQ != EOF) hsbit.hb = bitQ & 0x1; // 2.bit (DFM-09) - hsbit.sb = 2*hsbit.hb - 1; - } + frm = 0; + while ( frm < nfrms ) { // nfrms=1,2,4,8 + if (option_bin || option_softin) { + gpx._frmcnt = hdrcnt + frm; + } + else { + gpx._frmcnt = dsp.mv_pos/(2.0*dsp.sps*BITFRAME_LEN) + frm; } - else if (option_softin) { - float s1 = 0.0; - float s2 = 0.0; - float s = 0.0; - bitQ = f32soft_read(fp, &s1); - if (bitQ != EOF) { - bitQ = f32soft_read(fp, &s2); + while ( pos < BITFRAME_LEN ) + { + if (option_bin) { + // symbols or bits? + // manchester1 1->10,0->01: 1.bit (DFM-06) + // manchester2 0->10,1->01: 2.bit (DFM-09) + bitQ = fgetc(fp); if (bitQ != EOF) { - s = s2-s1; // integrate both symbols // only 2nd Manchester symbol: s2 - hsbit.sb = s; - hsbit.hb = (s>=0.0); + hsbit.hb = bitQ & 0x1; + bitQ = fgetc(fp); // check: rbit0^rbit1=1 (Manchester) + if (bitQ != EOF) hsbit.hb = bitQ & 0x1; // 2.bit (DFM-09) + hsbit.sb = 2*hsbit.hb - 1; } } - } - else { - float bl = -1; - if (option_iq >= 2) spike = 0; - if (option_iq > 2) bl = 4.0; - bitQ = read_softbit(&dsp, &hsbit, 0, bitofs, bitpos, bl, spike); // symlen=2 - // optional: - // normalize soft bit s_j by - // rhsbit.sb /= dsp._spb+1; // all samples in [-1,+1] - } - if ( bitQ == EOF ) { frm = nfrms; break; } // liest 2x EOF + else if (option_softin) { + float s1 = 0.0; + float s2 = 0.0; + float s = 0.0; + bitQ = f32soft_read(fp, &s1); + if (bitQ != EOF) { + bitQ = f32soft_read(fp, &s2); + if (bitQ != EOF) { + s = s2-s1; // integrate both symbols // only 2nd Manchester symbol: s2 + hsbit.sb = s; + hsbit.hb = (s>=0.0); + } + } + } + else { + float bl = -1; + if (option_iq >= 2) spike = 0; + if (option_iq > 2) bl = 4.0; + bitQ = read_softbit(&dsp, &hsbit, 0, bitofs, bitpos, bl, spike); // symlen=2 + // optional: + // normalize soft bit s_j by + // rhsbit.sb /= dsp._spb+1; // all samples in [-1,+1] + } + if ( bitQ == EOF ) { frm = nfrms; break; } // liest 2x EOF + + if (gpx.option.inv) { + hsbit.hb ^= 1; + hsbit.sb = -hsbit.sb; + } - if (gpx.option.inv) { - hsbit.hb ^= 1; - hsbit.sb = -hsbit.sb; + gpx.frame[pos] = hsbit; + pos++; + bitpos += 1; } - gpx.frame[pos] = hsbit; - pos++; - bitpos += 1; + if (pos < BITFRAME_LEN) break; + ret = print_frame(&gpx); + pos = 0; + frm += 1; + //if (ret < 0) frms += 1; } - - ret = print_frame(&gpx); - if (pos < BITFRAME_LEN) break; - pos = 0; - frm += 1; - //if (ret < 0) frms += 1; } + + header_found = 0; + pos = headerlen; } - header_found = 0; - pos = headerlen; + + if (!option_bin && !option_softin) free_buffers(&dsp); + else { + if (hdb.buf) { free(hdb.buf); hdb.buf = NULL; } + } } + else //if (rawhex) + { + #define BUFLEN (BITFRAME_LEN/4+12) + char buffer_rawhex[BUFLEN+1]; // no header + char *pbuf = NULL; + int len, i, j; + int ch = 0; + int pos = 0; + float _frmcnt = -1.0f; + + memset(buffer_rawhex, BUFLEN+1, 0); + while ( (ch=fgetc(fp)) != EOF) + { + if (ch == ' ') continue; + if (ch == '\n') { + buffer_rawhex[pos] = '\0'; + // + // +/- gpx.frame[].hb ... + // + gpx.option.inv = buffer_rawhex[0]=='-' ? 1 : 0; + sscanf(buffer_rawhex+1, "<%f>", &_frmcnt); + pbuf = strchr(buffer_rawhex, '>'); + if (pbuf != NULL) + { + pbuf++; + len = strlen(pbuf); + if (len*4 == BITFRAME_LEN-CONF) { + gpx._frmcnt = _frmcnt; + for (i = 0; i < len; i++) { + ui8_t nib = 0xFF; + sscanf(pbuf+i, "%1hhx", &nib); + for (j = 0; j < 4; j++) { + gpx.frame[CONF+4*i+j].hb = (nib>>j) & 0x1; // little endian + gpx.frame[CONF+4*i+j].sb = 2*gpx.frame[CONF+4*i+j].hb - 1; + } + } + print_frame(&gpx); + } + } + pos = 0; + } + else { + if ( ch >= '0' && ch <= '9' || ch >= 'a' && ch <= 'f' || ch >= 'A' && ch <= 'F' + || ch == '+' || ch == '-' || ch == '<' || ch == '.' || ch == '>') { + if (pos < BUFLEN) { + buffer_rawhex[pos] = ch; + pos++; + } + } + else { + // frame error + } + } - if (!option_bin && !option_softin) free_buffers(&dsp); - else { - if (hdb.buf) { free(hdb.buf); hdb.buf = NULL; } + } } + fclose(fp); return 0; From a44ace3d539ec4409fe98f24689550952132f37d Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Fri, 18 Feb 2022 17:04:06 +1030 Subject: [PATCH 3/8] Inhibit 201 errors from sondehub --- auto_rx/autorx/__init__.py | 2 +- auto_rx/autorx/decode.py | 1 + auto_rx/autorx/sondehub.py | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 879e4999..356fabca 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -17,7 +17,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.5.10-beta1" +__version__ = "1.5.10-beta2" # Global Variables diff --git a/auto_rx/autorx/decode.py b/auto_rx/autorx/decode.py index 818467ae..6f989970 100644 --- a/auto_rx/autorx/decode.py +++ b/auto_rx/autorx/decode.py @@ -1298,6 +1298,7 @@ def handle_decoder_line(self, data): _telemetry["subtype"] = "DFM" + # Check frame ID here to ensure we are on dfm09mod version with the frame number fixes (2020-12). if _telemetry["frame"] < 256: self.log_error( diff --git a/auto_rx/autorx/sondehub.py b/auto_rx/autorx/sondehub.py index 90501a1b..66f67ff1 100644 --- a/auto_rx/autorx/sondehub.py +++ b/auto_rx/autorx/sondehub.py @@ -385,6 +385,14 @@ def upload_telemetry(self, telem_list): _retries += 1 continue + elif _req.status_code == 201: + self.log_debug( + "Sondehub reported issue when adding packets to DB. Status Code: %d %s." + % (_req.status_code, _req.text) + ) + _upload_success = True + break + else: self.log_error( "Error uploading to Sondehub. Status Code: %d %s." From fe786b038b3d798209ae53b8a57fa15b2ab2f917 Mon Sep 17 00:00:00 2001 From: xss Date: Tue, 1 Mar 2022 07:45:28 +1100 Subject: [PATCH 4/8] Add APRS performance improvements --- auto_rx/auto_rx.py | 1 - auto_rx/autorx/aprs.py | 18 ++++++++++-------- auto_rx/autorx/config.py | 2 +- auto_rx/station.cfg.example | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/auto_rx/auto_rx.py b/auto_rx/auto_rx.py index f7c20013..a6542b4e 100644 --- a/auto_rx/auto_rx.py +++ b/auto_rx/auto_rx.py @@ -876,7 +876,6 @@ def main(): position_report=config["aprs_position_report"], aprsis_host=config["aprs_server"], aprsis_port=config["aprs_port"], - synchronous_upload_time=config["aprs_upload_rate"], callsign_validity_threshold=config["payload_id_valid"], station_beacon=config["station_beacon_enabled"], station_beacon_rate=config["station_beacon_rate"], diff --git a/auto_rx/autorx/aprs.py b/auto_rx/autorx/aprs.py index a5c0b2fe..398421d4 100644 --- a/auto_rx/autorx/aprs.py +++ b/auto_rx/autorx/aprs.py @@ -89,8 +89,8 @@ def telemetry_to_aprs_position( # TODO: RS41 Burst Timer - # Add on auto_rx version - _aprs_comment += " auto_rx v" + auto_rx_version + # Add on auto_rx at the end + _aprs_comment += " auto_rx" # Convert float latitude to APRS format (DDMM.MM) lat = float(sonde_data["lat"]) @@ -308,7 +308,7 @@ def __init__( station_beacon_position=(0.0, 0.0, 0.0), station_beacon_comment="radiosonde_auto_rx SondeGate v", station_beacon_icon="/r", - synchronous_upload_time=30, + upload_time=60, callsign_validity_threshold=5, upload_queue_size=16, upload_timeout=5, @@ -339,9 +339,7 @@ def __init__( station_beacon_comment (str): Comment field for the station beacon. will be replaced with the current auto_rx version. station_beacon_icon (str): The APRS icon to be used, as the two characters (symbol table, symbol index), as per http://www.aprs.org/symbols.html - synchronous_upload_time (int): Upload the most recent telemetry when time.time()%synchronous_upload_time == 0 - This is done in an attempt to get multiple stations uploading the same telemetry sentence simultaneously, - and also acts as decimation on the number of sentences uploaded to APRS-IS. + upload_time (int): Upload the most recent telemetry after this time is up. callsign_validity_threshold (int): Only upload telemetry data if the callsign has been observed more than N times. Default = 5 @@ -362,7 +360,8 @@ def __init__( self.aprsis_reconnect = aprsis_reconnect self.upload_timeout = upload_timeout self.upload_queue_size = upload_queue_size - self.synchronous_upload_time = synchronous_upload_time + self.upload_time = upload_time + self.next_upload = datetime.datetime.now() + datetime.timedelta(seconds=upload_time) self.callsign_validity_threshold = callsign_validity_threshold self.inhibit = inhibit @@ -653,7 +652,7 @@ def upload_timer(self): """ Add packets to the aprs upload queue if it is time for us to upload. """ while self.timer_thread_running: - if int(time.time()) % self.synchronous_upload_time == 0: + if datetime.datetime.now() > self.next_upload: # Time to upload! for _id in self.observed_payloads.keys(): # If no data, continue... @@ -677,6 +676,9 @@ def upload_timer(self): # Flush APRS-IS RX buffer self.flush_rx() + + # Reset upload timer + self.next_upload = datetime.datetime.now() + datetime.timedelta(seconds=self.upload_time) else: # Not yet time to upload, wait for a bit. time.sleep(0.1) diff --git a/auto_rx/autorx/config.py b/auto_rx/autorx/config.py index 3467440c..9f172bcf 100644 --- a/auto_rx/autorx/config.py +++ b/auto_rx/autorx/config.py @@ -36,7 +36,7 @@ # Fixed minimum update rates for APRS & Habitat # These are set to avoid congestion on the APRS-IS network, and on the Habitat server # Please respect other users of these networks and leave these settings as they are. -MINIMUM_APRS_UPDATE_RATE = 30 +MINIMUM_APRS_UPDATE_RATE = 60 MINIMUM_HABITAT_UPDATE_RATE = 30 diff --git a/auto_rx/station.cfg.example b/auto_rx/station.cfg.example index 79af8537..d190513e 100644 --- a/auto_rx/station.cfg.example +++ b/auto_rx/station.cfg.example @@ -176,7 +176,7 @@ aprs_pass = 00000 # This has a lower limit of 30 seconds, to avoid flooding the APRS-IS network. # Please be respectful of other uses of the APRS-IS network, and do not attempt # to upload faster than this. -upload_rate = 30 +upload_rate = 60 # APRS-IS server to upload to. # Default to radiosondy.info for now, to allow stats to show up on http://radiosondy.info From 668d4ea141cd13306e839d4aaf922af00d026d96 Mon Sep 17 00:00:00 2001 From: xss Date: Tue, 1 Mar 2022 10:01:15 +1100 Subject: [PATCH 5/8] Switch to monotonic time --- auto_rx/autorx/aprs.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/auto_rx/autorx/aprs.py b/auto_rx/autorx/aprs.py index 398421d4..9f4dbf9d 100644 --- a/auto_rx/autorx/aprs.py +++ b/auto_rx/autorx/aprs.py @@ -89,9 +89,6 @@ def telemetry_to_aprs_position( # TODO: RS41 Burst Timer - # Add on auto_rx at the end - _aprs_comment += " auto_rx" - # Convert float latitude to APRS format (DDMM.MM) lat = float(sonde_data["lat"]) lat_degree = abs(int(lat)) @@ -361,7 +358,7 @@ def __init__( self.upload_timeout = upload_timeout self.upload_queue_size = upload_queue_size self.upload_time = upload_time - self.next_upload = datetime.datetime.now() + datetime.timedelta(seconds=upload_time) + self.next_upload = time.monotonic() + upload_time self.callsign_validity_threshold = callsign_validity_threshold self.inhibit = inhibit @@ -652,7 +649,7 @@ def upload_timer(self): """ Add packets to the aprs upload queue if it is time for us to upload. """ while self.timer_thread_running: - if datetime.datetime.now() > self.next_upload: + if time.monotonic() > self.next_upload: # Time to upload! for _id in self.observed_payloads.keys(): # If no data, continue... @@ -678,7 +675,7 @@ def upload_timer(self): self.flush_rx() # Reset upload timer - self.next_upload = datetime.datetime.now() + datetime.timedelta(seconds=self.upload_time) + self.next_upload = time.monotonic() + self.upload_time else: # Not yet time to upload, wait for a bit. time.sleep(0.1) From 73e9e1afef3522def9e7024cef39a4cf19d8d1cf Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Tue, 1 Mar 2022 17:50:02 +1030 Subject: [PATCH 6/8] Restrict allowed APRS servers and ports. --- auto_rx/auto_rx.py | 1 + auto_rx/autorx/__init__.py | 2 +- auto_rx/autorx/config.py | 39 +++++++++++++++++++++++++++--- auto_rx/station.cfg.example | 47 +++++++++++++++++++++++++------------ 4 files changed, 70 insertions(+), 19 deletions(-) diff --git a/auto_rx/auto_rx.py b/auto_rx/auto_rx.py index a6542b4e..e2755fea 100644 --- a/auto_rx/auto_rx.py +++ b/auto_rx/auto_rx.py @@ -876,6 +876,7 @@ def main(): position_report=config["aprs_position_report"], aprsis_host=config["aprs_server"], aprsis_port=config["aprs_port"], + upload_time=config["aprs_upload_rate"], callsign_validity_threshold=config["payload_id_valid"], station_beacon=config["station_beacon_enabled"], station_beacon_rate=config["station_beacon_rate"], diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 356fabca..5b90a1e0 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -17,7 +17,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.5.10-beta2" +__version__ = "1.5.10-beta3" # Global Variables diff --git a/auto_rx/autorx/config.py b/auto_rx/autorx/config.py index 9f172bcf..62d78443 100644 --- a/auto_rx/autorx/config.py +++ b/auto_rx/autorx/config.py @@ -36,7 +36,7 @@ # Fixed minimum update rates for APRS & Habitat # These are set to avoid congestion on the APRS-IS network, and on the Habitat server # Please respect other users of these networks and leave these settings as they are. -MINIMUM_APRS_UPDATE_RATE = 60 +MINIMUM_APRS_UPDATE_RATE = 30 MINIMUM_HABITAT_UPDATE_RATE = 30 @@ -352,7 +352,7 @@ def read_auto_rx_config(filename, no_sdr_test=False): if auto_rx_config["aprs_upload_rate"] < MINIMUM_APRS_UPDATE_RATE: logging.warning( - "Config - APRS Update Rate clipped to minimum of %d seconds. Please be respectful of other users of APRS-IS." + "Config - APRS Update Rate clipped to minimum of %d seconds." % MINIMUM_APRS_UPDATE_RATE ) auto_rx_config["aprs_upload_rate"] = MINIMUM_APRS_UPDATE_RATE @@ -517,7 +517,7 @@ def read_auto_rx_config(filename, no_sdr_test=False): auto_rx_config["aprs_port"] = config.getint("aprs", "aprs_port") except: logging.warning( - "Config - Did not find aprs_port setting - using default of 14590. APRS packets might not be forwarded out to the wider APRS-IS network!" + "Config - Did not find aprs_port setting - using default of 14590." ) auto_rx_config["aprs_port"] = 14590 @@ -652,6 +652,39 @@ def read_auto_rx_config(filename, no_sdr_test=False): ) auto_rx_config["experimental_decoders"]["MK2LMS"] = False + + # As of auto_rx version 1.5.10, we are limiting APRS output to only radiosondy.info, + # and only on the non-forwarding port. + # This decision was not made lightly, and is a result of the considerable amount of + # non-amateur traffic that radiosonde flights are causing within the APRS-IS network. + # Until some form of common format can be agreed to amongst the developers of *all* + # radiosonde tracking software to enable radiosonde telemetry to be de-duped, + # I have decided to help reduce the impact on the wider APRS-IS network by restricting + # the allowed servers and ports. + # If you are using another APRS-IS server that *does not* forward to the wider APRS-IS + # network and want it allowed, then please raise an issue at + # https://github.com/projecthorus/radiosonde_auto_rx/issues + # + # You are of course free to fork and modify this codebase as you wish, but please be aware + # that this goes against the wishes of the radiosonde_auto_rx developers to not be part + # of the bigger problem of APRS-IS congestion. + + ALLOWED_APRS_SERVERS = ["radiosondy.info"] + ALLOWED_APRS_PORTS = [14590] + + if auto_rx_config["aprs_server"] not in ALLOWED_APRS_SERVERS: + logging.warning( + "Please do not upload to servers which forward to the wider APRS-IS network and cause network congestion. Switching to default server of radiosondy.info. If you believe this to be in error, please raise an issue at https://github.com/projecthorus/radiosonde_auto_rx/issues" + ) + auto_rx_config["aprs_server"] = "radiosondy.info" + + if auto_rx_config["aprs_port"] not in ALLOWED_APRS_PORTS: + logging.warning( + "Please do not use APRS ports which forward data out to the wider APRS-IS network and cause network congestion. Switching to default port of 14590. If you believe this to be in error, please raise an issue at https://github.com/projecthorus/radiosonde_auto_rx/issues" + ) + auto_rx_config["aprs_port"] = 14590 + + # If we are being called as part of a unit test, just return the config now. if no_sdr_test: return auto_rx_config diff --git a/auto_rx/station.cfg.example b/auto_rx/station.cfg.example index d190513e..d370db03 100644 --- a/auto_rx/station.cfg.example +++ b/auto_rx/station.cfg.example @@ -159,37 +159,54 @@ sondehub_contact_email = none@none.com ######################## # APRS UPLOAD SETTINGS # ######################## -# Settings for uploading to APRS-IS +# Settings for uploading to radiosondy.info +# +# IMPORTANT APRS NOTE +# +# As of auto_rx version 1.5.10, we are limiting APRS output to only radiosondy.info, +# and only on the non-forwarding port. +# This decision was not made lightly, and is a result of the considerable amount of +# non-amateur traffic that radiosonde flights are causing within the APRS-IS network. +# Until some form of common format can be agreed to amongst the developers of *all* +# radiosonde tracking software to enable radiosonde telemetry to be de-duped, +# I have decided to help reduce the impact on the wider APRS-IS network by restricting +# the allowed servers and ports. +# If you are using another APRS-IS server that *does not* forward to the wider APRS-IS +# network and want it allowed, then please raise an issue at +# https://github.com/projecthorus/radiosonde_auto_rx/issues +# +# You are of course free to fork and modify this codebase as you wish, but please be aware +# that this goes against the wishes of the radiosonde_auto_rx developers to not be part +# of the bigger problem of APRS-IS congestion. + [aprs] # Enable APRS upload (you will also need to change some options below!) aprs_enabled = False # APRS-IS Login Information # The aprs_user field can have an SSID on the end if desired, i.e. N0CALL-4 -# If you are a licensed amateur radio operator, you may want to change the aprs_port number below -# to 14580, so that your uploaded telemetry makes its way out to the wider APRS network. aprs_user = N0CALL # APRS-IS Passcode. You can generate one for your callsign here: https://apps.magicbug.co.uk/passcode/ aprs_pass = 00000 # APRS Upload Rate - Upload a packet every X seconds. -# This has a lower limit of 30 seconds, to avoid flooding the APRS-IS network. -# Please be respectful of other uses of the APRS-IS network, and do not attempt -# to upload faster than this. -upload_rate = 60 +# This has a lower limit of 30 seconds, to avoid flooding radiosondy.info +# Please be respectful, and do not attempt to upload faster than this. +upload_rate = 30 # APRS-IS server to upload to. -# Default to radiosondy.info for now, to allow stats to show up on http://radiosondy.info -# Packets are forwarded onto the rest of the APRS-IS network from radiosondy.info. -# If you wish to inject packets directly into the APRS-IS network, use rotate.aprs2.net +# Currently we only support uploading to radiosondy.info +# When using port 14580, packets are not forwarded to the wider APRS-IS network, and hence +# are help reduce the huge amount of non-amateur traffic that ends up in APRS-IS from +# radiosondes. aprs_server = radiosondy.info # APRS-IS Port Number to upload to. -# When using radiosondy.info: -# Port 14590 - Packets stay within radiosondy.info. Non-licensed operators can use this. -# Port 14580 - Packets are forwarded out to the wider APRS-IS network. Only licensed amateur radio operators should use this!! -# For all other APRS-IS servers (licensed amateur radio operators only!), use port 14580. -aprs_port = 14590 +# +# Port 14590 - Packets stay within radiosondy.info and do not congest the wider APRS-IS +# network. +# +aprs_port = 14580 # APRS Station Location Beaconing # If enabled, you will show up on APRS using the aprs_user callsign set above. From 983727e4e5be92c5049d22149d3eba262aac4d10 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Tue, 1 Mar 2022 18:04:45 +1030 Subject: [PATCH 7/8] additional thoughts --- auto_rx/station.cfg.example | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/auto_rx/station.cfg.example b/auto_rx/station.cfg.example index d370db03..7e350c8f 100644 --- a/auto_rx/station.cfg.example +++ b/auto_rx/station.cfg.example @@ -178,6 +178,13 @@ sondehub_contact_email = none@none.com # You are of course free to fork and modify this codebase as you wish, but please be aware # that this goes against the wishes of the radiosonde_auto_rx developers to not be part # of the bigger problem of APRS-IS congestion. +# As of 2022-03-01, radiosonde traffic has been filtered from aprs.fi, so even if you do +# modify the code, you still won't see sondes on that map. +# APRS-IS is a *shared resource*, intended for the use of all amateur radio operators, and +# for many years we have been treating it as a playground to dump large amounts of non-amateur +# traffic into, so we can see weather balloons on a map. +# Instead of congesting this shared resource with this non-amateur traffic, we should instead +# be moving to using databases and sites specialised for this purpose, for example sondehub.org [aprs] # Enable APRS upload (you will also need to change some options below!) From 1e3523857b1af74ac6f75aaa529c07240f3eece8 Mon Sep 17 00:00:00 2001 From: Mark Jessop Date: Sat, 5 Mar 2022 11:32:06 +1030 Subject: [PATCH 8/8] Bump version before merge --- auto_rx/autorx/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_rx/autorx/__init__.py b/auto_rx/autorx/__init__.py index 5b90a1e0..22257866 100644 --- a/auto_rx/autorx/__init__.py +++ b/auto_rx/autorx/__init__.py @@ -17,7 +17,7 @@ # MINOR - New sonde type support, other fairly big changes that may result in telemetry or config file incompatability issus. # PATCH - Small changes, or minor feature additions. -__version__ = "1.5.10-beta3" +__version__ = "1.5.10" # Global Variables