From 3c0b701af8d03274a3c704eddad46ff5e8b3ae13 Mon Sep 17 00:00:00 2001 From: Martin Panzner Date: Mon, 2 May 2022 11:59:01 +0200 Subject: [PATCH 1/8] fixed AarhusWorkbench input requirements --- libaarhusxyz/alc.py | 14 ++++++++++++++ libaarhusxyz/xyz.py | 15 ++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/libaarhusxyz/alc.py b/libaarhusxyz/alc.py index 76d37c3..f2457c2 100644 --- a/libaarhusxyz/alc.py +++ b/libaarhusxyz/alc.py @@ -17,6 +17,8 @@ def parse(nameorfile, xyz_columns=None): supported_fields = ["DateTime","Date","Time","Flight","Line","GdSpeed","Alt","DEM", "Magnetic","PowerLineMonitor", + "Misc1","Misc2","Misc3","Misc4", + "Current_Ch01","Current_Ch02", "RxPitch","RxRoll","Topography","TxAltitude", "TxOffTime","TxOnTime","TxPeakTime", "TxPitch","TxRoll","TxRxHoriSep","TxRxVertSep","UTMX","UTMY", @@ -33,8 +35,20 @@ def is_supported_field(fieldname): return False def _dump(xyz, f, columns=None): + print(columns) if columns is None: columns = xyz["file_meta"]["columns"] + else: + columns_=[] + for column_name in columns: + words=re.split('\[|\]', column_name) + if len(words)==1: + columns_.append(words[0]) + else: + newname="{0}_{1:02d}".format(words[0],int(words[1])+1) + columns_.append(newname) + columns=columns_ + print(columns) rows = [{"canonical_name": col, "position": idx + 1} for idx, col in enumerate(columns) if is_supported_field(col)] diff --git a/libaarhusxyz/xyz.py b/libaarhusxyz/xyz.py index 1de9c4a..bfdc524 100644 --- a/libaarhusxyz/xyz.py +++ b/libaarhusxyz/xyz.py @@ -142,13 +142,14 @@ def _un_split_layer_columns(data): def _dump(data, file, alcfile=None): df = _un_split_layer_columns(data) - for key, value in data['model_info'].items(): - if key != 'source': - file.write("/" + str(key) + "\n") - if isinstance(value, list): - file.write("/" + ' '.join(str(item) for item in value) + "\n") - else: - file.write("/" + str(value) + "\n") + if alcfile==None: + for key, value in data['model_info'].items(): + if key != 'source': + file.write("/" + str(key) + "\n") + if isinstance(value, list): + file.write("/" + ' '.join(str(item) for item in value) + "\n") + else: + file.write("/" + str(value) + "\n") file.write('/ ') df.to_csv(file, index=False, sep=' ', na_rep="*", encoding='utf-8') From 46d9335103792e9d0aa185522e9cb33bd7559d55 Mon Sep 17 00:00:00 2001 From: Martin Panzner Date: Mon, 2 May 2022 12:22:17 +0200 Subject: [PATCH 2/8] cleanup --- libaarhusxyz/alc.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/libaarhusxyz/alc.py b/libaarhusxyz/alc.py index f2457c2..dc84a0e 100644 --- a/libaarhusxyz/alc.py +++ b/libaarhusxyz/alc.py @@ -35,7 +35,6 @@ def is_supported_field(fieldname): return False def _dump(xyz, f, columns=None): - print(columns) if columns is None: columns = xyz["file_meta"]["columns"] else: @@ -48,11 +47,9 @@ def _dump(xyz, f, columns=None): newname="{0}_{1:02d}".format(words[0],int(words[1])+1) columns_.append(newname) columns=columns_ - print(columns) rows = [{"canonical_name": col, "position": idx + 1} for idx, col in enumerate(columns) if is_supported_field(col)] - channels = set([row["canonical_name"][len("Gate_Ch"):].split("_")[0] for row in rows if row["canonical_name"].startswith("Gate_Ch")]) From cede8a0072976516c3376b373c1b0cb881844f0b Mon Sep 17 00:00:00 2001 From: Martin Panzner Date: Mon, 16 May 2022 11:17:01 +0200 Subject: [PATCH 3/8] minor mod --- libaarhusxyz/xyz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libaarhusxyz/xyz.py b/libaarhusxyz/xyz.py index bfdc524..8b242bf 100644 --- a/libaarhusxyz/xyz.py +++ b/libaarhusxyz/xyz.py @@ -24,7 +24,7 @@ # with the naming convention used in Aaarhus Workbench / ALC files. _RE_LAYER_COL = re.compile(r"^(.*?)[(_\[]([0-9]+)[)\]]?$") -_NA_VALUES = ["", "#N/A", "#N/A N/A", "#NA", "-1.#IND", "-1.#QNAN", "-NaN", "-nan", "1.#IND", "1.#QNAN", "", +_NA_VALUES = [9999, 9999.9, "9999", "9999.9", "", "#N/A", "#N/A N/A", "#NA", "-1.#IND", "-1.#QNAN", "-NaN", "-nan", "1.#IND", "1.#QNAN", "", "N/A", "NA", "NULL", "NaN", "n/a", "nan", "null", "*"] From 457ee36e9c02033523c5d3aa4e95ab45d0f786e5 Mon Sep 17 00:00:00 2001 From: Martin Panzner Date: Mon, 16 May 2022 12:49:02 +0200 Subject: [PATCH 4/8] small bugfix --- libaarhusxyz/gex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libaarhusxyz/gex.py b/libaarhusxyz/gex.py index ad78e5e..844fe3a 100644 --- a/libaarhusxyz/gex.py +++ b/libaarhusxyz/gex.py @@ -28,7 +28,7 @@ def split_sections(text): sectionlineidx.append(len(text)+1) sections={"header":text[0]} for k in range(len(sectionheaders)): - sections[sectionheaders[k]]=text[sectionlineidx[k]+1:sectionlineidx[k+1]-1] + sections[sectionheaders[k]]=text[sectionlineidx[k]+1:sectionlineidx[k+1]] return sections, sectionheaders def parse_parameters(textlines): From 98f51c8e9c03d871b512e8d05cd94a340b22d46f Mon Sep 17 00:00:00 2001 From: Martin Panzner Date: Mon, 16 May 2022 15:02:24 +0200 Subject: [PATCH 5/8] minor mods --- libaarhusxyz/xyz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libaarhusxyz/xyz.py b/libaarhusxyz/xyz.py index 8b242bf..0e94db2 100644 --- a/libaarhusxyz/xyz.py +++ b/libaarhusxyz/xyz.py @@ -31,7 +31,7 @@ def _split_layer_columns(df): per_layer_cols = [col for col in df.columns if re.match(_RE_LAYER_COL, col)] per_sounding_cols = [col for col in df.columns if not col in per_layer_cols] - + colgroups = {} for col in per_layer_cols: group = re.match(_RE_LAYER_COL, col).groups()[0] From af8b80908a2e5f483e9527ad3c73c8a0269d1056 Mon Sep 17 00:00:00 2001 From: Martin Panzner Date: Wed, 18 May 2022 14:42:28 +0200 Subject: [PATCH 6/8] minor cleanup --- libaarhusxyz/gex.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libaarhusxyz/gex.py b/libaarhusxyz/gex.py index 844fe3a..a1e3769 100644 --- a/libaarhusxyz/gex.py +++ b/libaarhusxyz/gex.py @@ -64,7 +64,7 @@ def _parse(inputfile): gex={"header":sections["header"]} for header in sectionheaders: gex[header.strip("[").strip("]")]=parse_parameters(sections[header]) - print("header {} parsed".format(header)) + # print("header {} parsed".format(header)) return gex def parse(nameorfile, **kw): From ca59955bf586dcb5fea1fac098f33505cdc58e92 Mon Sep 17 00:00:00 2001 From: Martin Panzner Date: Thu, 1 Sep 2022 08:35:38 +0200 Subject: [PATCH 7/8] minor mods --- libaarhusxyz/xyz.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libaarhusxyz/xyz.py b/libaarhusxyz/xyz.py index 0e94db2..2c4cb9a 100644 --- a/libaarhusxyz/xyz.py +++ b/libaarhusxyz/xyz.py @@ -24,6 +24,7 @@ # with the naming convention used in Aaarhus Workbench / ALC files. _RE_LAYER_COL = re.compile(r"^(.*?)[(_\[]([0-9]+)[)\]]?$") + _NA_VALUES = [9999, 9999.9, "9999", "9999.9", "", "#N/A", "#N/A N/A", "#NA", "-1.#IND", "-1.#QNAN", "-NaN", "-nan", "1.#IND", "1.#QNAN", "", "N/A", "NA", "NULL", "NaN", "n/a", "nan", "null", "*"] From 5a5e4faaa4a463287766c46bfd13dcad837c3550 Mon Sep 17 00:00:00 2001 From: "Benjamin R. Bloss" Date: Tue, 13 Sep 2022 17:33:38 +0200 Subject: [PATCH 8/8] fixed header issue so that it's a list and not an index --- libaarhusxyz/xyz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libaarhusxyz/xyz.py b/libaarhusxyz/xyz.py index 0e94db2..c860033 100644 --- a/libaarhusxyz/xyz.py +++ b/libaarhusxyz/xyz.py @@ -115,7 +115,7 @@ def _parse(inputfile, source=None, alcfile=None, **kw): res = {"flightlines": df, "layer_data": layer_dfs, "model_info": headers, - "file_meta": {"columns": full_df.columns}} + "file_meta": {"columns": list(full_df.columns)}} if alcdata is not None: res["alc_info"] = alcdata["meta"]