diff --git a/qgepqwat2ili/qgep/export.py b/qgepqwat2ili/qgep/export.py index 07087053..88f21c5e 100644 --- a/qgepqwat2ili/qgep/export.py +++ b/qgepqwat2ili/qgep/export.py @@ -5,9 +5,16 @@ from sqlalchemy.orm import Session from sqlalchemy.sql import text +# 4.10.2024 +# from ..utils.ili2db import skip_wwtp_structure_ids +# 6.11.2024 replaced with from .. import utils from ..utils.basket_utils import BasketUtils -from ..utils.ili2db import skip_wwtp_structure_ids + +# 4.10.2024 +# from ..utils.ili2db import skip_wwtp_structure_ids +# 6.11.2024 replaced with +from ..utils.ili2db import add_to_selection, get_ws_wn_ids, remove_from_selection from ..utils.qgep_export_utils import QgepExportUtils from ..utils.various import logger from .model_abwasser import get_abwasser_model @@ -43,19 +50,46 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e current_basket = basket_utils.basket_topic_sia405_abwasser - # Filtering + # 1. Filtering - check if selection filtered = selection is not None subset_ids = selection if selection is not None else [] # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - wastewater_structure_id_sia405abwasser_list = None - wastewater_structure_id_sia405abwasser_list = skip_wwtp_structure_ids() + # 2. check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") - logger.info( - f"wastewater_structure_id_sia405abwasser_list : {wastewater_structure_id_sia405abwasser_list}", + # 3. Show ws_off_sia405abwasser_list + logger.debug( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", ) + # 4. check if filtered + if filtered: + if ws_off_sia405abwasser_list: + # take out ws_off_sia405abwasser_list from selection + subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) + # else do nothing + else: + if ws_off_sia405abwasser_list: + # add all data of wastewater_structures to selection + subset_ids = add_to_selection(subset_ids, get_ws_wn_ids("wastewater_structure")) + logger.debug( + f"subset_ids of all wws : {subset_ids}", + ) + # take out ws_off_sia405abwasser_list from selection + subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) + logger.debug( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_ids}", + ) + # add reach_ids + # subset_ids = add_to_selection(subset_ids, get_cl_re_ids("channel")) + # treat export as with a selection + filtered = True + + # else do nothing + # Orientation oriented = orientation is not None if oriented: @@ -63,6 +97,230 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e else: labelorientation = 0 + def get_tid(relation): + """ + Makes a tid for a relation + """ + if relation is None: + return None + return tid_maker.tid_for_row(relation) + + def get_vl(relation): + """ + Gets a literal value from a value list relation + """ + if relation is None: + return None + return relation.value_de + + def null_to_emptystr(val): + """ + Converts nulls to blank strings and raises a warning + """ + if val is None: + logger.warning( + "A mandatory value was null. It will be cast to a blank string, and probably cause validation errors", + ) + val = "" + return val + + def emptystr_to_null(val): + """ + Converts blank strings to nulls and raises a warning + + This is needed as is seems ili2pg 4.4.6 crashes with emptystrings under certain circumstances (see https://github.com/QGEP/qgepqwat2ili/issues/33) + """ + if val == "": + logger.warning( + "An empty string was converted to NULL, to workaround ili2pg issue. This should have no impact on output.", + ) + val = None + return val + + def truncate(val, max_length): + """ + Raises a warning if values gets truncated + """ + if val is None: + return None + if len(val) > max_length: + # _log() got an unexpected keyword argument 'stacklevel' + # logger.warning(f"Value '{val}' exceeds expected length ({max_length})", stacklevel=2) + logger.warning(f"Value '{val}' exceeds expected length ({max_length})") + return val[0:max_length] + + def modulo_angle(val): + """ + Returns an angle between 0 and 359.9 (for Orientierung in Base_d-20181005.ili) + """ + if val is None: + return None + + # add orientation + val = val + float(labelorientation) + + val = val % 360.0 + if val > 359.9: + val = 0 + + logger.info(f"modulo_angle - added orientation: {labelorientation}") + print("modulo_angle - added orientation: ", str(labelorientation)) + + return val + + def check_fk_in_subsetid(subset, relation): + """ + checks, whether foreignkey is in the subset_ids - if yes it return the tid of the foreignkey, if no it will return None + """ + # first check for None, as is get_tid + if relation is None: + return None + + logger.debug(f"check_fk_in_subsetid - Subset ID's '{subset}'") + # get the value of the fk_ attribute as str out of the relation to be able to check whether it is in the subset + fremdschluesselstr = getattr(relation, "obj_id") + logger.debug(f"check_fk_in_subsetid - fremdschluesselstr '{fremdschluesselstr}'") + + if fremdschluesselstr in subset: + logger.debug(f"check_fk_in_subsetid - '{fremdschluesselstr}' is in subset ") + logger.debug(f"check_fk_in_subsetid - tid = '{tid_maker.tid_for_row(relation)}' ") + return tid_maker.tid_for_row(relation) + else: + if filtered: + logger.warning( + f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in filtered subset - replaced with None instead!" + ) + return None + else: + logger.warning( + f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in datamodel - replaced with None instead!" + ) + return None + + def create_metaattributes(row): + metaattribute = ABWASSER.metaattribute( + # FIELDS TO MAP TO ABWASSER.metaattribute + # --- metaattribute --- + # 31.3.2023 identifier instead of name + # datenherr=getattr(row.fk_dataowner__REL, "name", "unknown"), # TODO : is unknown ok ? + # datenlieferant=getattr(row.fk_provider__REL, "name", "unknown"), # TODO : is unknown ok ? + # datenherr=getattr(row.fk_dataowner__REL, "identifier", "unknown"), # TODO : is unknown ok ? + # datenlieferant=getattr(row.fk_provider__REL, "identifier", "unknown"), # TODO : is unknown ok ? + # 31.3.2023 obj_id instead of name + datenherr=getattr( + row.fk_dataowner__REL, "obj_id", "unknown" + ), # TODO : is unknown ok ? + datenlieferant=getattr( + row.fk_provider__REL, "obj_id", "unknown" + ), # TODO : is unknown ok ? + letzte_aenderung=row.last_modification, + sia405_baseclass_metaattribute=get_tid(row), + # OD : is this OK ? Don't we need a different t_id from what inserted above in organisation ? if so, consider adding a "for_class" arg to tid_for_row + t_id=get_tid(row), + t_seq=0, + ) + abwasser_session.add(metaattribute) + + def base_common(row, type_name): + """ + Returns common attributes for base + """ + return { + "t_ili_tid": row.obj_id, + "t_type": type_name, + "obj_id": row.obj_id, + "t_id": get_tid(row), + } + + def wastewater_structure_common(row): + """ + Returns common attributes for wastewater_structure + ATTENTION : Mapping of 3D wastewater_structure->abwasserbauerk + is not fully implemented. + """ + return { + # --- abwasserbauwerk --- + "akten": row.records, + "astatus": get_vl(row.status__REL), + "baujahr": row.year_of_construction, + "baulicherzustand": get_vl(row.structure_condition__REL), + "baulos": row.contract_section, + "bemerkung": truncate(emptystr_to_null(row.remark), 80), + "betreiberref": get_tid(row.fk_operator__REL), + "bezeichnung": null_to_emptystr(row.identifier), + "bruttokosten": row.gross_costs, + "detailgeometrie": ST_Force2D(row.detail_geometry_geometry), + "eigentuemerref": get_tid(row.fk_owner__REL), + "ersatzjahr": row.year_of_replacement, + "finanzierung": get_vl(row.financing__REL), + "inspektionsintervall": row.inspection_interval, + "sanierungsbedarf": get_vl(row.renovation_necessity__REL), + "standortname": row.location_name, + "subventionen": row.subsidies, + "wbw_basisjahr": row.rv_base_year, + "wbw_bauart": get_vl(row.rv_construction_type__REL), + "wiederbeschaffungswert": row.replacement_value, + "zugaenglichkeit": get_vl(row.accessibility__REL), + } + + def wastewater_networkelement_common(row): + """ + Returns common attributes for network_element + """ + + return { + # "abwasserbauwerkref": get_tid(row.fk_wastewater_structure__REL), + # 6.11.2024 Besides wn_id and re_id we also need ws_obj_ids in a separate subset - call it ws_subset_id + # "abwasserbauwerkref": check_fk_in_subsetid( + # subset_ids, row.fk_wastewater_structure__REL + # ), + "abwasserbauwerkref": check_fk_in_subsetid( + subset_ids, row.fk_wastewater_structure__REL + ), + "bemerkung": truncate(emptystr_to_null(row.remark), 80), + "bezeichnung": null_to_emptystr(row.identifier), + } + + def structure_part_common(row): + """ + Returns common attributes for structure_part + """ + return { + # abwasserbauwerkref is MANDATORY, so it cannot be set to NULL + "abwasserbauwerkref": get_tid(row.fk_wastewater_structure__REL), + "bemerkung": truncate(emptystr_to_null(row.remark), 80), + "bezeichnung": null_to_emptystr(row.identifier), + "instandstellung": get_vl(row.renovation_demand__REL), + } + + def textpos_common(row, t_type, geojson_crs_def): + """ + Returns common attributes for textpos + """ + t_id = tid_maker.next_tid() + return { + "t_id": t_id, + "t_type": t_type, + "t_ili_tid": t_id, + # --- TextPos --- + "textpos": ST_GeomFromGeoJSON( + json.dumps( + { + "type": "Point", + "coordinates": row["geometry"]["coordinates"], + "crs": geojson_crs_def, + } + ) + ), + "textori": modulo_angle(row["properties"]["LabelRotation"]), + "texthali": "Left", # can be Left/Center/Right + "textvali": "Bottom", # can be Top,Cap,Half,Base,Bottom + # --- SIA405_TextPos --- + "plantyp": row["properties"]["scale"], + "textinhalt": row["properties"]["LabelText"], + "bemerkung": None, + } + qgep_export_utils = QgepExportUtils( tid_maker=tid_maker, current_basket=current_basket, @@ -239,21 +497,263 @@ def qgep_export_kek(selection=None, labels_file=None, orientation=None, basket_e logger.info( "Exporting QGEP.dryweather_downspout -> ABWASSER.trockenwetterfallrohr, ABWASSER.metaattribute" ) - qgep_export_utils.export_dryweather_downspout() + + query = qgep_session.query(QGEP.dryweather_downspout) + if filtered: + logger.info(f"filtered: subset_ids = {subset_ids}") + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.dryweather_downspout + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- dryweather_downspout --- + # diameter, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, renovation_demand__REL + + trockenwetterfallrohr = ABWASSER.trockenwetterfallrohr( + # FIELDS TO MAP TO ABWASSER.trockenwetterfallrohr + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "trockenwetterfallrohr"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- trockenwetterfallrohr --- + durchmesser=row.diameter, + ) + abwasser_session.add(trockenwetterfallrohr) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info("Exporting QGEP.access_aid -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute") - qgep_export_utils.export_access_aid() + query = qgep_session.query(QGEP.access_aid) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.access_aid + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- access_aid --- + # kind, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, kind__REL, renovation_demand__REL + + einstiegshilfe = ABWASSER.einstiegshilfe( + # FIELDS TO MAP TO ABWASSER.einstiegshilfe + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "einstiegshilfe"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- einstiegshilfe --- + art=get_vl(row.kind__REL), + ) + abwasser_session.add(einstiegshilfe) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info( "Exporting QGEP.dryweather_flume -> ABWASSER.trockenwetterrinne, ABWASSER.metaattribute" ) - qgep_export_utils.export_dryweather_flume() + + query = qgep_session.query(QGEP.dryweather_flume) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.dryweather_flume + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- dryweather_flume --- + # material, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, material__REL, renovation_demand__REL + + trockenwetterrinne = ABWASSER.trockenwetterrinne( + # FIELDS TO MAP TO ABWASSER.trockenwetterrinne + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "trockenwetterrinne"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- trockenwetterrinne --- + material=get_vl(row.material__REL), + ) + abwasser_session.add(trockenwetterrinne) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info("Exporting QGEP.cover -> ABWASSER.deckel, ABWASSER.metaattribute") - qgep_export_utils.export_cover() + query = qgep_session.query(QGEP.cover) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.cover + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- cover --- + # brand, cover_shape, diameter, fastening, level, material, obj_id, positional_accuracy, situation_geometry, sludge_bucket, venting + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id, wastewater_structure__BWREL_fk_main_cover + + # --- _rel_ --- + # cover_shape__REL, fastening__REL, fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, material__REL, positional_accuracy__REL, renovation_demand__REL, sludge_bucket__REL, venting__REL + + deckel = ABWASSER.deckel( + # FIELDS TO MAP TO ABWASSER.deckel + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "deckel"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- deckel --- + deckelform=get_vl(row.cover_shape__REL), + durchmesser=row.diameter, + entlueftung=get_vl(row.venting__REL), + fabrikat=row.brand, + kote=row.level, + lage=ST_Force2D(row.situation_geometry), + lagegenauigkeit=get_vl(row.positional_accuracy__REL), + material=get_vl(row.material__REL), + schlammeimer=get_vl(row.sludge_bucket__REL), + verschluss=get_vl(row.fastening__REL), + ) + abwasser_session.add(deckel) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info("Exporting QGEP.benching -> ABWASSER.bankett, ABWASSER.metaattribute") - qgep_export_utils.export_benching() + query = qgep_session.query(QGEP.benching) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.benching + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- benching --- + # kind, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, kind__REL, renovation_demand__REL + + bankett = ABWASSER.bankett( + # FIELDS TO MAP TO ABWASSER.bankett + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "bankett"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- bankett --- + art=get_vl(row.kind__REL), + ) + abwasser_session.add(bankett) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info("Exporting QGEP.examination -> ABWASSER.untersuchung, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.examination) diff --git a/qgepqwat2ili/qgepdss/export.py b/qgepqwat2ili/qgepdss/export.py index 6bcfe72e..57cc64da 100644 --- a/qgepqwat2ili/qgepdss/export.py +++ b/qgepqwat2ili/qgepdss/export.py @@ -7,12 +7,14 @@ from .. import utils from ..utils.basket_utils import BasketUtils -from ..utils.ili2db import skip_wwtp_structure_ids -from ..utils.qgep_export_utils import QgepExportUtils from ..utils.various import logger from .model_abwasser import get_abwasser_model from .model_qgep import get_qgep_model +# 4.10.2024 +# from ..utils.ili2db import skip_wwtp_structure_ids +# 6.11.2024 replaced with - to check if really necessary here (as no sia405 abwasser exceptions needed) + def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_enabled=False): """ @@ -36,12 +38,11 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e abwasser_session.execute(text("SET CONSTRAINTS ALL DEFERRED;")) basket_utils = None - current_basket = None if basket_enabled: basket_utils = BasketUtils(abwasser_model, abwasser_session) basket_utils.create_basket() - current_basket = basket_utils.basket_topic_sia405_abwasser + basket_utils.basket_topic_sia405_abwasser # Filtering filtered = selection is not None @@ -54,14 +55,7 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e # Logging for debugging logger.debug(f"print subset_ids '{subset_ids}'") - # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure - - wastewater_structure_id_sia405abwasser_list = None - wastewater_structure_id_sia405abwasser_list = skip_wwtp_structure_ids() - - logger.info( - f"wastewater_structure_id_sia405abwasser_list : {wastewater_structure_id_sia405abwasser_list}", - ) + # ws_off_sia405abwasser_list and selection process not needed for VSA-DSS Export # Orientation oriented = orientation is not None @@ -70,17 +64,138 @@ def qgep_export_dss(selection=None, labels_file=None, orientation=None, basket_e else: labelorientation = 0 - qgep_export_utils = QgepExportUtils( - tid_maker=tid_maker, - current_basket=current_basket, - abwasser_session=abwasser_session, - abwasser_model=abwasser_model, - qgep_session=qgep_session, - qgep_model=qgep_model, - labelorientation=labelorientation, - filtered=filtered, - subset_ids=subset_ids, - ) + def get_tid(relation): + """ + Makes a tid for a relation + """ + if relation is None: + return None + + return tid_maker.tid_for_row(relation) + + def get_vl(relation): + """ + Gets a literal value from a value list relation + """ + if relation is None: + return None + return relation.value_de + + def null_to_emptystr(val): + """ + Converts nulls to blank strings and raises a warning + """ + if val is None: + logger.warning( + "A mandatory value was null. It will be cast to a blank string, and probably cause validation errors", + ) + val = "" + return val + + def emptystr_to_null(val): + """ + Converts blank strings to nulls and raises a warning + + This is needed as is seems ili2pg 4.4.6 crashes with emptystrings under certain circumstances (see https://github.com/QGEP/qgepqwat2ili/issues/33) + """ + if val == "": + logger.warning( + "An empty string was converted to NULL, to workaround ili2pg issue. This should have no impact on output.", + ) + val = None + return val + + def truncate(val, max_length): + """ + Raises a warning if values gets truncated + """ + if val is None: + return None + if len(val) > max_length: + # _log() got an unexpected keyword argument 'stacklevel' + # logger.warning(f"Value '{val}' exceeds expected length ({max_length})", stacklevel=2) + logger.warning(f"Value '{val}' exceeds expected length ({max_length})") + return val[0:max_length] + + def modulo_angle(val): + """ + Returns an angle between 0 and 359.9 (for Orientierung in Base_d-20181005.ili) + """ + if val is None: + return None + + # add orientation + val = val + float(labelorientation) + + val = val % 360.0 + if val > 359.9: + val = 0 + + logger.info(f"modulo_angle - added orientation: {labelorientation}") + print("modulo_angle - added orientation: ", str(labelorientation)) + + return val + + def check_fk_in_subsetid(subset, relation): + """ + checks, whether foreignkey is in the subset_ids - if yes it return the tid of the foreignkey, if no it will return None + """ + # first check for None, as is get_tid + if relation is None: + return None + + logger.debug(f"check_fk_in_subsetid - Subset ID's '{subset}'") + # get the value of the fk_ attribute as str out of the relation to be able to check whether it is in the subset + fremdschluesselstr = getattr(relation, "obj_id") + logger.debug(f"check_fk_in_subsetid - fremdschluesselstr '{fremdschluesselstr}'") + + if fremdschluesselstr in subset: + logger.debug(f"check_fk_in_subsetid - '{fremdschluesselstr}' is in subset ") + logger.debug(f"check_fk_in_subsetid - tid = '{tid_maker.tid_for_row(relation)}' ") + return tid_maker.tid_for_row(relation) + else: + if filtered: + logger.warning( + f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in filtered subset - replaced with None instead!" + ) + return None + else: + logger.warning( + f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in datamodel - replaced with None instead!" + ) + return None + + def create_metaattributes(row): + metaattribute = ABWASSER.metaattribute( + # FIELDS TO MAP TO ABWASSER.metaattribute + # --- metaattribute --- + # datenherr=getattr(row.fk_dataowner__REL, "name", "unknown"), # TODO : is unknown ok ? + # datenlieferant=getattr(row.fk_provider__REL, "name", "unknown"), # TODO : is unknown ok ? + # obj_id instead of name + datenherr=getattr( + row.fk_dataowner__REL, "obj_id", "unknown" + ), # TODO : is unknown ok ? + datenlieferant=getattr( + row.fk_provider__REL, "obj_id", "unknown" + ), # TODO : is unknown ok ? + letzte_aenderung=row.last_modification, + sia405_baseclass_metaattribute=get_tid(row), + # OD : is this OK ? Don't we need a different t_id from what inserted above in organisation ? if so, consider adding a "for_class" arg to tid_for_row + t_id=get_tid(row), + t_seq=0, + ) + abwasser_session.add(metaattribute) + + def base_common(row, type_name): + """ + Returns common attributes for base + """ + return { + "t_ili_tid": row.obj_id, + "t_type": type_name, + "obj_id": row.obj_id, + "t_id": get_tid(row), + } def organisation_common(row): """ @@ -133,6 +248,31 @@ def water_control_structure_common(row): "lage": ST_Force2D(row.situation_geometry), } + def wastewater_networkelement_common(row): + """ + Returns common attributes for wastewater_networkelement + """ + return { + # "abwasserbauwerkref": get_tid(row.fk_wastewater_structure__REL), + "abwasserbauwerkref": check_fk_in_subsetid( + subset_ids, row.fk_wastewater_structure__REL + ), + "bemerkung": truncate(emptystr_to_null(row.remark), 80), + "bezeichnung": null_to_emptystr(row.identifier), + } + + def structure_part_common(row): + """ + Returns common attributes for structure_part + """ + return { + # abwasserbauwerkref is MANDATORY, so it cannot be set to NULL + "abwasserbauwerkref": get_tid(row.fk_wastewater_structure__REL), + "bemerkung": truncate(emptystr_to_null(row.remark), 80), + "bezeichnung": null_to_emptystr(row.identifier), + "instandstellung": get_vl(row.renovation_demand__REL), + } + def connection_object_common(row): """ Returns common attributes for connection_object @@ -1166,7 +1306,37 @@ def overflow_common(row): abwasser_session.flush() logger.info("Exporting QGEP.pipe_profile -> ABWASSER.rohrprofil, ABWASSER.metaattribute") - qgep_export_utils.export_pipe_profile() + + query = qgep_session.query(QGEP.pipe_profile) + + for row in query: + + # AVAILABLE FIELDS IN QGEP.pipe_profile + + # --- pipe_profile --- + # to do e.g. fk_dataowner, fk_provider, height_width_ratio, identifier, last_modification, obj_id, profile_type, remark + # --- _bwrel_ --- + # to do add superclassrelations e.g. profile_geometry__BWREL_fk_pipe_profile, reach__BWREL_fk_pipe_profile + + # --- _rel_ --- + # to do add relations fk_dataowner__REL, fk_provider__REL, profile_type__REL + + rohrprofil = ABWASSER.rohrprofil( + # FIELDS TO MAP TO ABWASSER.rohrprofil + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "rohrprofil"), + # --- rohrprofil --- + bemerkung=truncate(emptystr_to_null(row.remark), 80), + bezeichnung=null_to_emptystr(row.identifier), + hoehenbreitenverhaeltnis=row.height_width_ratio, + profiltyp=get_vl(row.profile_type__REL), + ) + abwasser_session.add(rohrprofil) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info( "Exporting QGEP.wwtp_energy_use -> ABWASSER.araenergienutzung, ABWASSER.metaattribute" @@ -1295,11 +1465,10 @@ def overflow_common(row): logger.info( "Exporting QGEP.control_center -> ABWASSER.steuerungszentrale, ABWASSER.metaattribute" ) - query = qgep_session.query(qgep_model.control_center) - if filtered: - query = query.join(qgep_model.throttle_shut_off_unit, qgep_model.wastewater_node).filter( - qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) - ) + + query = qgep_session.query(QGEP.control_center) + # Always export all, no filtering + for row in query: # AVAILABLE FIELDS IN QGEP.control_center @@ -1629,11 +1798,9 @@ def overflow_common(row): abwasser_session.flush() logger.info("Exporting QGEP.hydr_geometry -> ABWASSER.hydr_geometrie, ABWASSER.metaattribute") - query = qgep_session.query(qgep_model.hydr_geometry) - if filtered: - query = query.join(qgep_model.wastewater_node).filter( - qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) - ) + + query = qgep_session.query(QGEP.hydr_geometry) + for row in query: # AVAILABLE FIELDS IN QGEP.hydr_geometry @@ -1716,11 +1883,9 @@ def overflow_common(row): logger.info( "Exporting QGEP.profile_geometry -> ABWASSER.rohrprofil_geometrie, ABWASSER.metaattribute" ) - query = qgep_session.query(qgep_model.profile_geometry) - if filtered: - query = query.join(qgep_model.pipe_profile, qgep_model.reach).filter( - qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) - ) + + query = qgep_session.query(QGEP.profile_geometry) + for row in query: # AVAILABLE FIELDS IN QGEP.profile_geometry @@ -1753,11 +1918,9 @@ def overflow_common(row): logger.info( "Exporting QGEP.hydr_geom_relation -> ABWASSER.hydr_geomrelation, ABWASSER.metaattribute" ) - query = qgep_session.query(qgep_model.hydr_geom_relation) - if filtered: - query = query.join(qgep_model.hydr_geometry, qgep_model.wastewater_node).filter( - qgep_model.wastewater_networkelement.obj_id.in_(subset_ids) - ) + + query = qgep_session.query(QGEP.hydr_geom_relation) + for row in query: # AVAILABLE FIELDS IN QGEP.hydr_geom_relation @@ -1792,9 +1955,19 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.mechanical_pretreatment) if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.mechanical_pretreatment.fk_wastewater_structure + == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.mechanical_pretreatment @@ -1813,15 +1986,12 @@ def overflow_common(row): # --- sia405_baseclass --- **qgep_export_utils.base_common(row, "mechanischevorreinigung"), # --- mechanischevorreinigung --- - abwasserbauwerkref=qgep_export_utils.get_tid(row.fk_wastewater_structure__REL), - art=qgep_export_utils.get_vl(row.kind__REL), - bemerkung=qgep_export_utils.truncate( - qgep_export_utils.emptystr_to_null(row.remark), 80 - ), - bezeichnung=qgep_export_utils.null_to_emptystr(row.identifier), - versickerungsanlageref=qgep_export_utils.get_tid( - row.fk_infiltration_installation__REL - ), + # abwasserbauwerkref=get_tid(row.fk_wastewater_structure__REL), + abwasserbauwerkref=check_fk_in_subsetid(subset_ids, row.fk_wastewater_structure__REL), + art=get_vl(row.kind__REL), + bemerkung=truncate(emptystr_to_null(row.remark), 80), + bezeichnung=null_to_emptystr(row.identifier), + versickerungsanlageref=get_tid(row.fk_infiltration_installation__REL), ) abwasser_session.add(mechanischevorreinigung) qgep_export_utils.create_metaattributes(row) @@ -1834,9 +2004,21 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.retention_body) if filtered: - query = query.join( - qgep_model.infiltration_installation, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + + # explicit join on added + query = ( + query.join( + QGEP.infiltration_installation, + QGEP.retention_body.fk_infiltration_installation + == QGEP.infiltration_installation.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.retention_body @@ -1879,7 +2061,7 @@ def overflow_common(row): if filtered: # add sql statement to logger statement = query.statement - logger.info(f" always export all overflow_char datasets query = {statement}") + logger.debug(f" always export all overflow_char datasets query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.overflow_char @@ -1920,7 +2102,7 @@ def overflow_common(row): ) # add sql statement to logger statement = query.statement - logger.info(f" selection query = {statement}") + logger.debug(f" selection query = {statement}") for row in query: # AVAILABLE FIELDS IN QGEP.hq_relation @@ -1953,27 +2135,233 @@ def overflow_common(row): logger.info( "Exporting QGEP.dryweather_downspout -> ABWASSER.trockenwetterfallrohr, ABWASSER.metaattribute" ) - qgep_export_utils.export_dryweather_downspout() + + query = qgep_session.query(QGEP.dryweather_downspout) + if filtered: + logger.info(f"filtered: subset_ids = {subset_ids}") + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.dryweather_downspout + + # --- structure_part --- + # to do attributeslist of superclass + # --- dryweather_downspout --- + # to do attributeslist of subclass + # to do extra funktion schreiben wo alle englischen attribute erzeugt werden + + # --- _bwrel_ --- + # to do extra funktion schreiben wo alle fk auf diese superklasse erzeugt werden z.B. # measuring_point__BWREL_fk_wastewater_structure, + + # --- _rel_ --- + # to do extra funktion schreiben wo alle fk auf diese klasse erzeugt werden z.B. # accessibility__REL, bedding_encasement__REL, + + trockenwetterfallrohr = ABWASSER.trockenwetterfallrohr( + # FIELDS TO MAP TO ABWASSER.trockenwetterfallrohr + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "trockenwetterfallrohr"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- trockenwetterfallrohr --- + durchmesser=row.diameter, + ) + abwasser_session.add(trockenwetterfallrohr) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info("Exporting QGEP.access_aid -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute") - qgep_export_utils.export_access_aid() + query = qgep_session.query(QGEP.access_aid) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.access_aid + + # --- structure_part --- + # to do attributeslist of superclass + # --- access_aid --- + # to do attributeslist of subclass + # to do extra funktion schreiben wo alle englischen attribute erzeugt werden + + # --- _bwrel_ --- + # to do extra funktion schreiben wo alle fk auf diese superklasse erzeugt werden z.B. # measuring_point__BWREL_fk_wastewater_structure, + + # --- _rel_ --- + # to do extra funktion schreiben wo alle fk auf diese klasse erzeugt werden z.B. # accessibility__REL, bedding_encasement__REL, + + einstiegshilfe = ABWASSER.einstiegshilfe( + # FIELDS TO MAP TO ABWASSER.einstiegshilfe + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "einstiegshilfe"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- einstiegshilfe --- + art=get_vl(row.kind__REL), + ) + abwasser_session.add(einstiegshilfe) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info( "Exporting QGEP.dryweather_flume -> ABWASSER.trockenwetterrinne, ABWASSER.metaattribute" ) - qgep_export_utils.export_dryweather_flume() + + query = qgep_session.query(QGEP.dryweather_flume) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.dryweather_flume + + # --- structure_part --- + # to do attributeslist of superclass + # --- dryweather_flume --- + # to do attributeslist of subclass + # to do extra funktion schreiben wo alle englischen attribute erzeugt werden + + # --- _bwrel_ --- + # to do extra funktion schreiben wo alle fk auf diese superklasse erzeugt werden z.B. # measuring_point__BWREL_fk_wastewater_structure, + + # --- _rel_ --- + # to do extra funktion schreiben wo alle fk auf diese klasse erzeugt werden z.B. # accessibility__REL, bedding_encasement__REL, + + trockenwetterrinne = ABWASSER.trockenwetterrinne( + # FIELDS TO MAP TO ABWASSER.trockenwetterrinne + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "trockenwetterrinne"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- trockenwetterrinne --- + material=get_vl(row.material__REL), + ) + abwasser_session.add(trockenwetterrinne) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info("Exporting QGEP.cover -> ABWASSER.deckel, ABWASSER.metaattribute") - qgep_export_utils.export_cover() + query = qgep_session.query(QGEP.cover) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.cover + + # --- structure_part --- + # to do attributeslist of superclass + # --- cover --- + # to do attributeslist of subclass + # to do extra funktion schreiben wo alle englischen attribute erzeugt werden + + # --- _bwrel_ --- + # to do extra funktion schreiben wo alle fk auf diese superklasse erzeugt werden z.B. # measuring_point__BWREL_fk_wastewater_structure, + + # --- _rel_ --- + # to do extra funktion schreiben wo alle fk auf diese klasse erzeugt werden z.B. # accessibility__REL, bedding_encasement__REL, + + deckel = ABWASSER.deckel( + # FIELDS TO MAP TO ABWASSER.deckel + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "deckel"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- deckel --- + deckelform=get_vl(row.cover_shape__REL), + durchmesser=row.diameter, + entlueftung=get_vl(row.venting__REL), + fabrikat=row.brand, + kote=row.level, + lage=ST_Force2D(row.situation_geometry), + lagegenauigkeit=get_vl(row.positional_accuracy__REL), + # -- attribute 3D --- + # maechtigkeit=row.depth, + material=get_vl(row.material__REL), + schlammeimer=get_vl(row.sludge_bucket__REL), + verschluss=get_vl(row.fastening__REL), + ) + abwasser_session.add(deckel) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info( "Exporting QGEP.electric_equipment -> ABWASSER.elektrischeeinrichtung, ABWASSER.metaattribute" ) query = qgep_session.query(qgep_model.electric_equipment) if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.electric_equipment @@ -2012,9 +2400,22 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.electromechanical_equipment) if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.electromechanical_equipment @@ -2049,7 +2450,53 @@ def overflow_common(row): abwasser_session.flush() logger.info("Exporting QGEP.benching -> ABWASSER.bankett, ABWASSER.metaattribute") - qgep_export_utils.export_benching() + + query = qgep_session.query(QGEP.benching) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + # AVAILABLE FIELDS IN QGEP.benching + + # --- structure_part --- + # to do attributeslist of superclass + # --- benching --- + # to do attributeslist of subclass + # to do extra funktion schreiben wo alle englischen attribute erzeugt werden + + # --- _bwrel_ --- + # to do extra funktion schreiben wo alle fk auf diese superklasse erzeugt werden z.B. # measuring_point__BWREL_fk_wastewater_structure, + + # --- _rel_ --- + # to do extra funktion schreiben wo alle fk auf diese klasse erzeugt werden z.B. # accessibility__REL, bedding_encasement__REL, + + bankett = ABWASSER.bankett( + # FIELDS TO MAP TO ABWASSER.bankett + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "bankett"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- bankett --- + art=get_vl(row.kind__REL), + ) + abwasser_session.add(bankett) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info("Exporting QGEP.building -> ABWASSER.gebaeude, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.building) @@ -2212,9 +2659,19 @@ def overflow_common(row): logger.info("Exporting QGEP.hazard_source -> ABWASSER.gefahrenquelle, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.hazard_source) if filtered: - query = query.join( - qgep_model.connection_object, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + + query = ( + query.join( + QGEP.connection_object, + QGEP.hazard_source.fk_connection_object == QGEP.connection_object.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.hazard_source @@ -2250,11 +2707,19 @@ def overflow_common(row): logger.info("Exporting QGEP.accident -> ABWASSER.unfall, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.accident) if filtered: - query = query.join( - qgep_model.hazard_source, - qgep_model.connection_object, - qgep_model.wastewater_networkelement, - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + + query = ( + query.join(QGEP.hazard_source) + .join( + QGEP.connection_object, + QGEP.hazard_source.fk_connection_object == QGEP.connection_object.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ).filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.accident @@ -2292,11 +2757,16 @@ def overflow_common(row): logger.info("Exporting QGEP.substance -> ABWASSER.stoff, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.substance) if filtered: - query = query.join( - qgep_model.hazard_source, - qgep_model.connection_object, - qgep_model.wastewater_networkelement, - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + + query = ( + query.join(QGEP.hazard_source) + .join( + QGEP.connection_object, + QGEP.hazard_source.fk_connection_object == QGEP.connection_object.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ).filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + for row in query: # AVAILABLE FIELDS IN QGEP.substance @@ -2428,9 +2898,10 @@ def overflow_common(row): query = qgep_session.query(qgep_model.measuring_point) if filtered: query1 = query.join( - qgep_model.wastewater_structure, - qgep_model.wastewater_networkelement, - ) + QGEP.wastewater_structure, + QGEP.measuring_point.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ).join(QGEP.wastewater_networkelement) + # needs to add QGEP.wastewater_structure as waste_water_treatment_plant is a subclass of organisation that has a relation to wastewater_structure and then wastewater_networkelement # variant1 for query2 # query2=query.join( @@ -2448,23 +2919,48 @@ def overflow_common(row): # QGEP.wastewater_networkelement, # ) - # query2 via waste_water_treatment_plant TODO : Fix Mapping - query2 = query.join( - qgep_model.model_classes_tww_od.waste_water_treatment_plant, - qgep_model.model_classes_tww_od.wwtp_structure, - qgep_model.model_classes_tww_od.wastewater_networkelement, + + # query2 via waste_water_treatment_plant Release 2015 where waste_water_treatment_plant is subclass of organisation + query2 = ( + query.join( + QGEP.waste_water_treatment_plant, + QGEP.measuring_point.fk_waste_water_treatment_plant + == QGEP.waste_water_treatment_plant.obj_id, + ) + .join( + QGEP.wastewater_structure, + QGEP.wastewater_structure.fk_owner == QGEP.waste_water_treatment_plant.obj_id, + ) + .join(QGEP.wastewater_networkelement) ) # only until VSA-DSS Release 2015 - query3 = query.join( - qgep_model.water_course_segment, - qgep_model.river, - qgep_model.sector_water_body, - qgep_model.discharge_point, - qgep_model.wastewater_networkelement, + query3 = ( + query.join( + QGEP.water_course_segment, + QGEP.measuring_point.fk_water_course_segment == QGEP.water_course_segment.obj_id, + ) + .join( + QGEP.river, + # Fehler im Datenmodell fk_watercourse should be name fk_surface_water_bodies (resp. fk_surface_water_body - class should be renamed to single) + QGEP.water_course_segment.fk_watercourse == QGEP.river.obj_id, + ) + .join( + QGEP.sector_water_body, + QGEP.sector_water_body.fk_surface_water_bodies == QGEP.sector_water_body.obj_id, + ) + .join( + QGEP.discharge_point, + QGEP.discharge_point.fk_sector_water_body == QGEP.sector_water_body.obj_id, + ) + .join(QGEP.wastewater_networkelement) ) query = query.union(query1, query2, query3) # query = query.union(query1, query3) - query = query.filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = query.filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.measuring_point @@ -2483,10 +2979,9 @@ def overflow_common(row): # --- sia405_baseclass --- **qgep_export_utils.base_common(row, "messstelle"), # --- messstelle --- - abwasserbauwerkref=qgep_export_utils.get_tid(row.fk_wastewater_structure__REL), - abwasserreinigungsanlageref=qgep_export_utils.get_tid( - row.fk_waste_water_treatment_plant__REL - ), + # abwasserbauwerkref=get_tid(row.fk_wastewater_structure__REL), + abwasserbauwerkref=check_fk_in_subsetid(subset_ids, row.fk_wastewater_structure__REL), + abwasserreinigungsanlageref=get_tid(row.fk_waste_water_treatment_plant__REL), art=row.kind, bemerkung=qgep_export_utils.truncate( qgep_export_utils.emptystr_to_null(row.remark), 80 @@ -2509,18 +3004,70 @@ def overflow_common(row): logger.info("Exporting QGEP.measuring_device -> ABWASSER.messgeraet, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.measuring_device) if filtered: - query = query.join( - qgep_model.measuring_point, - qgep_model.wastewater_structure, - qgep_model.wastewater_networkelement, - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - # or does not work with this - currently do not support - # QGEP.wastewater_networkelement, - # or_( - # (QGEP.measuring_point, QGEP.waste_water_treatment_plant, QGEP.wastewater_networkelement), - # (QGEP.measuring_point, QGEP.wastewater_structure, QGEP.wastewater_networkelement), - # (QGEP.measuring_point, QGEP.water_course_segment, QGEP.river, QGEP.sector_water_body, QGEP.discharge_point, QGEP.wastewater_networkelement), - # ) + + # query = query.join( + # QGEP.measuring_point, QGEP.wastewater_structure, QGEP.wastewater_networkelement + # ).filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + + query1 = ( + query.join( + QGEP.measuring_point, + QGEP.measuring_device.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.wastewater_structure, + QGEP.measuring_point.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + # query2 via waste_water_treatment_plant Release 2015 where waste_water_treatment_plant is subclass of organisation + query2 = ( + query.join( + QGEP.measuring_point, + QGEP.measuring_device.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.waste_water_treatment_plant, + QGEP.measuring_point.fk_waste_water_treatment_plant + == QGEP.waste_water_treatment_plant.obj_id, + ) + .join( + QGEP.wastewater_structure, + QGEP.wastewater_structure.fk_owner == QGEP.waste_water_treatment_plant.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + # only until VSA-DSS Release 2015 + query3 = ( + query.join( + QGEP.measuring_point, + QGEP.measuring_device.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.water_course_segment, + QGEP.measuring_point.fk_water_course_segment == QGEP.water_course_segment.obj_id, + ) + .join( + QGEP.river, + # Fehler im Datenmodell fk_watercourse should be name fk_surface_water_bodies (resp. fk_surface_water_body - class should be renamed to single) + QGEP.water_course_segment.fk_watercourse == QGEP.river.obj_id, + ) + .join( + QGEP.sector_water_body, + QGEP.sector_water_body.fk_surface_water_bodies == QGEP.sector_water_body.obj_id, + ) + .join( + QGEP.discharge_point, + QGEP.discharge_point.fk_sector_water_body == QGEP.sector_water_body.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + query = query.union(query1, query2, query3) + # query = query.union(query1, query3) + query = query.filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: @@ -2558,17 +3105,70 @@ def overflow_common(row): logger.info("Exporting QGEP.measurement_series -> ABWASSER.messreihe, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.measurement_series) if filtered: - query = query.join( - qgep_model.measuring_point, - qgep_model.wastewater_structure, - qgep_model.wastewater_networkelement, - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - # QGEP.wastewater_networkelement, - # or_( - # (QGEP.measuring_point, QGEP.waste_water_treatment_plant, QGEP.wastewater_networkelement), - # (QGEP.measuring_point, QGEP.wastewater_structure, QGEP.wastewater_networkelement), - # (QGEP.measuring_point, QGEP.water_course_segment, QGEP.river, QGEP.sector_water_body, QGEP.discharge_point, QGEP.wastewater_networkelement), - # ) + + # query = query.join( + # QGEP.measuring_point, QGEP.wastewater_structure, QGEP.wastewater_networkelement + # ).filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + query1 = ( + query.join( + QGEP.measuring_point, + QGEP.measurement_series.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.wastewater_structure, + QGEP.measuring_point.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + # query2 via waste_water_treatment_plant Release 2015 where waste_water_treatment_plant is subclass of organisation + query2 = ( + query.join( + QGEP.measuring_point, + QGEP.measurement_series.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.waste_water_treatment_plant, + QGEP.measuring_point.fk_waste_water_treatment_plant + == QGEP.waste_water_treatment_plant.obj_id, + ) + .join( + QGEP.wastewater_structure, + QGEP.wastewater_structure.fk_owner == QGEP.waste_water_treatment_plant.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + # only until VSA-DSS Release 2015 + query3 = ( + query.join( + QGEP.measuring_point, + QGEP.measurement_series.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.water_course_segment, + QGEP.measuring_point.fk_water_course_segment == QGEP.water_course_segment.obj_id, + ) + .join( + QGEP.river, + # Fehler im Datenmodell fk_watercourse should be name fk_surface_water_bodies (resp. fk_surface_water_body - class should be renamed to single) + QGEP.water_course_segment.fk_watercourse == QGEP.river.obj_id, + ) + .join( + QGEP.sector_water_body, + QGEP.sector_water_body.fk_surface_water_bodies == QGEP.sector_water_body.obj_id, + ) + .join( + QGEP.discharge_point, + QGEP.discharge_point.fk_sector_water_body == QGEP.sector_water_body.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + query = query.union(query1, query2, query3) + # query = query.union(query1, query3) + query = query.filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.measurement_series @@ -2608,21 +3208,85 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.measurement_result) if filtered: - query = query.join( - qgep_model.measurement_series, - qgep_model.measuring_point, - qgep_model.wastewater_structure, - qgep_model.wastewater_networkelement, - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) - # or_( - # (QGEP.measurement_series, QGEP.measuring_point, QGEP.waste_water_treatment_plant, QGEP.wastewater_networkelement), - # (QGEP.measurement_series, QGEP.measuring_point, QGEP.wastewater_structure, QGEP.wastewater_networkelement), - # (QGEP.measurement_series, QGEP.measuring_point, QGEP.water_course_segment, QGEP.river, QGEP.sector_water_body, QGEP.discharge_point, QGEP.wastewater_networkelement), - # (QGEP.measuring_device, QGEP.measuring_point, QGEP.waste_water_treatment_plant, QGEP.wastewater_networkelement), - # (QGEP.measuring_device, QGEP.measuring_point, QGEP.wastewater_structure, QGEP.wastewater_networkelement), - # (QGEP.measuring_device, QGEP.measuring_point, QGEP.water_course_segment, QGEP.river, QGEP.sector_water_body, QGEP.discharge_point, QGEP.wastewater_networkelement), - # ) + # query = query.join( + # QGEP.measurement_series, + # QGEP.measuring_point, + # QGEP.wastewater_structure, + # QGEP.wastewater_networkelement, + # ).filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + query1 = ( + query.join( + QGEP.measurement_series, + QGEP.measurement_result.fk_measurement_series == QGEP.measurement_series.obj_id, + ) + .join( + QGEP.measuring_point, + QGEP.measurement_series.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.wastewater_structure, + QGEP.measuring_point.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + # query2 via waste_water_treatment_plant Release 2015 where waste_water_treatment_plant is subclass of organisation + query2 = ( + query.join( + QGEP.measurement_series, + QGEP.measurement_result.fk_measurement_series == QGEP.measurement_series.obj_id, + ) + .join( + QGEP.measuring_point, + QGEP.measurement_series.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.waste_water_treatment_plant, + QGEP.measuring_point.fk_waste_water_treatment_plant + == QGEP.waste_water_treatment_plant.obj_id, + ) + .join( + QGEP.wastewater_structure, + QGEP.wastewater_structure.fk_owner == QGEP.waste_water_treatment_plant.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + # only until VSA-DSS Release 2015 + query3 = ( + query.join( + QGEP.measurement_series, + QGEP.measurement_result.fk_measurement_series == QGEP.measurement_series.obj_id, + ) + .join( + QGEP.measuring_point, + QGEP.measurement_series.fk_measuring_point == QGEP.measuring_point.obj_id, + ) + .join( + QGEP.water_course_segment, + QGEP.measuring_point.fk_water_course_segment == QGEP.water_course_segment.obj_id, + ) + .join( + QGEP.river, + # Fehler im Datenmodell fk_watercourse should be name fk_surface_water_bodies (resp. fk_surface_water_body - class should be renamed to single) + QGEP.water_course_segment.fk_watercourse == QGEP.river.obj_id, + ) + .join( + QGEP.sector_water_body, + QGEP.sector_water_body.fk_surface_water_bodies == QGEP.sector_water_body.obj_id, + ) + .join( + QGEP.discharge_point, + QGEP.discharge_point.fk_sector_water_body == QGEP.sector_water_body.obj_id, + ) + .join(QGEP.wastewater_networkelement) + ) + # query4 not implemented via measuring_device + query = query.union(query1, query2, query3) + # query = query.union(query1, query3) + query = query.filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") for row in query: @@ -2925,9 +3589,22 @@ def overflow_common(row): query = qgep_session.query(qgep_model.backflow_prevention) # side fk_throttle_shut_off_unit and fk_overflow not considered in filter query - they are usually added only for log_cards and then the corresponding nodes exist anyway thru the direct relation. if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.backflow_prevention @@ -2968,9 +3645,22 @@ def overflow_common(row): ) query = qgep_session.query(qgep_model.solids_retention) if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.solids_retention @@ -3009,9 +3699,22 @@ def overflow_common(row): logger.info("Exporting QGEP.tank_cleaning -> ABWASSER.beckenreinigung, ABWASSER.metaattribute") query = qgep_session.query(qgep_model.tank_cleaning) if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.tank_cleaning @@ -3051,9 +3754,22 @@ def overflow_common(row): query = qgep_session.query(qgep_model.tank_emptying) # side fk_throttle_shut_off_unit and fk_overflow not considered in filter query - they are usually added only for log_cards and then the corresponding nodes exist anyway thru the direct relation. if filtered: - query = query.join( - qgep_model.wastewater_structure, qgep_model.wastewater_networkelement - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.tank_emptying @@ -3181,14 +3897,28 @@ def overflow_common(row): logger.info( "Exporting QGEP.maintenance_event -> ABWASSER.maintenance_event, ABWASSER.metaattribute" ) - query = qgep_session.query(qgep_model.maintenance_event) - # to check if join is correct like this n:m re_maintenance_event_wastewater_structure + + query = qgep_session.query(QGEP.maintenance_event) + # explicit join for n:m re_maintenance_event_wastewater_structure if filtered: - query = query.join( - qgep_model.re_maintenance_event_wastewater_structure, - qgep_model.wastewater_structure, - qgep_model.wastewater_networkelement, - ).filter(qgep_model.wastewater_networkelement.obj_id.in_(subset_ids)) + query = ( + query.join( + QGEP.re_maintenance_event_wastewater_structure, + QGEP.re_maintenance_event_wastewater_structure.fk_maintenance_event + == QGEP.maintenance_event.obj_id, + ) + .join( + QGEP.wastewater_structure, + QGEP.re_maintenance_event_wastewater_structure.fk_wastewater_structure + == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: # AVAILABLE FIELDS IN QGEP.maintenance_event @@ -3363,10 +4093,9 @@ def overflow_common(row): # --- baseclass --- # --- sia405_baseclass --- # --- erhaltungsereignis_abwasserbauwerk --- - abwasserbauwerkref=qgep_export_utils.get_tid(row.fk_wastewater_structure__REL), - erhaltungsereignis_abwasserbauwerkassocref=qgep_export_utils.get_tid( - row.fk_maintenance_event__REL - ), + # abwasserbauwerkref=get_tid(row.fk_wastewater_structure__REL), + abwasserbauwerkref=check_fk_in_subsetid(subset_ids, row.fk_wastewater_structure__REL), + erhaltungsereignis_abwasserbauwerkassocref=get_tid(row.fk_maintenance_event__REL), ) abwasser_session2.add(erhaltungsereignis_abwasserbauwerk) diff --git a/qgepqwat2ili/qgepsia405/export.py b/qgepqwat2ili/qgepsia405/export.py index e1e93382..1c92fdd2 100644 --- a/qgepqwat2ili/qgepsia405/export.py +++ b/qgepqwat2ili/qgepsia405/export.py @@ -6,7 +6,19 @@ from .. import utils from ..utils.basket_utils import BasketUtils -from ..utils.ili2db import skip_wwtp_structure_ids + +# 4.10.2024 +# 6.11.2024 replaced with / 15.11.2024 get_ws_selected_ww_networkelements added +from ..utils.ili2db import ( + add_to_selection, + filter_reaches, + get_connected_overflow_to_wn_ids, + get_connected_we_from_re, + get_connected_we_to_re, + get_ws_selected_ww_networkelements, + get_ws_wn_ids, + remove_from_selection, +) from ..utils.qgep_export_utils import QgepExportUtils from ..utils.various import logger from .model_abwasser import get_abwasser_model @@ -42,18 +54,104 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske current_basket = basket_utils.basket_topic_sia405_abwasser - # Filtering + # 1. Filtering - check if selection + filtered = selection is not None subset_ids = selection if selection is not None else [] - # get list of id's of class wwtp_structure (ARABauwerk) to be able to check if fk_wastewater_structure references to wwtp_structure + flag_approach_urs = True - wastewater_structure_id_sia405abwasser_list = None - wastewater_structure_id_sia405abwasser_list = skip_wwtp_structure_ids() + if flag_approach_urs: + # 2. Get all connected from wastewater_nodes of selected reaches + connected_from_wn_ids = get_connected_we_from_re(subset_ids) + # 3. Get all connected to wastewater_nodes of selected reaches + connected_to_wn_ids = get_connected_we_to_re(subset_ids) + # 4. Get all connected wastewater_nodes from overflows.fk_overflow_to + connected_overflow_to_wn_ids = get_connected_overflow_to_wn_ids(subset_ids) + # 5. Add results from 2., 3. and 4. to subset_ids -> adapted_subset_ids + adapted_subset_ids = add_to_selection(subset_ids, connected_from_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_to_wn_ids) + adapted_subset_ids = add_to_selection(adapted_subset_ids, connected_overflow_to_wn_ids) + # 6. check blind connections - are there reaches in adapted_subset_ids that have not been in subset_ids + subset_ids_reaches = filter_reaches(subset_ids) + adapted_subset_ids_reaches = filter_reaches(adapted_subset_ids) - logger.info( - f"wastewater_structure_id_sia405abwasser_list : {wastewater_structure_id_sia405abwasser_list}", - ) + # https://www.w3schools.com/python/ref_set_difference.asp + # x = {"apple", "banana", "cherry"} + # y = {"google", "microsoft", "apple"} + # z = x.difference(y) + extra_reaches_ids = subset_ids_reaches.difference(adapted_subset_ids_reaches) + # 7. If extra_reaches then remove from adapted_subset_ids + if not extra_reaches_ids: + # list is empty - no need for adaption + logger.debug( + "no extra reaches - so nothing to remove from adapted_subset_ids", + ) + else: + # if len(extra_reaches_ids) > 0: + adapted_subset_ids = remove_from_selection(adapted_subset_ids, extra_reaches_ids) + # 8. get all id's of connected wastewater_structures + subset_wws_ids = get_ws_selected_ww_networkelements(adapted_subset_ids) + logger.debug( + f"subset_wws_ids: {subset_wws_ids}", + ) + # 9. if sia405 export: check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") + + # 10. Show ws_off_sia405abwasser_list + logger.debug( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + ) + # 11. take out ws_off_sia405abwasser_list from subset_wws_ids + subset_wws_ids = remove_from_selection(subset_wws_ids, ws_off_sia405abwasser_list) + logger.debug( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_wws_ids}", + ) + else: + # 2. check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") + + # 3. Show ws_off_sia405abwasser_list + logger.debug( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + ) + + # 4. check if filtered + if filtered: + if ws_off_sia405abwasser_list: + # take out ws_off_sia405abwasser_list from selection + subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) + # else do nothing + else: + if ws_off_sia405abwasser_list: + # add all data of wastewater_structures to selection + subset_ids = add_to_selection(subset_ids, get_ws_wn_ids("wastewater_structure")) + logger.debug( + f"subset_ids of all wws : {subset_ids}", + ) + # take out ws_off_sia405abwasser_list from selection + subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) + logger.debug( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_ids}", + ) + # add reach_ids + # subset_ids = add_to_selection(subset_ids, get_cl_re_ids("channel")) + # treat export as with a selection + filtered = True + + # else do nothing + + # 5. get and add all id's of connected wastewater_structures (not only of wastewater_network_element (reach, wwn) + subset_wws_ids = get_ws_selected_ww_networkelements(subset_ids) + logger.debug( + f"subset_wws_ids: {subset_wws_ids}", + ) + subset_ids = add_to_selection(subset_ids, subset_wws_ids) + logger.debug( + f"subset_ids with wws : {subset_ids}", + ) # Orientation oriented = orientation is not None @@ -62,6 +160,228 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske else: labelorientation = 0 + def get_tid(relation): + """ + Makes a tid for a relation + """ + if relation is None: + return None + + return tid_maker.tid_for_row(relation) + + def get_vl(relation): + """ + Gets a literal value from a value list relation + """ + if relation is None: + return None + return relation.value_de + + def null_to_emptystr(val): + """ + Converts nulls to blank strings and raises a warning + """ + if val is None: + logger.warning( + "A mandatory value was null. It will be cast to a blank string, and probably cause validation errors", + ) + val = "" + return val + + def emptystr_to_null(val): + """ + Converts blank strings to nulls and raises a warning + + This is needed as is seems ili2pg 4.4.6 crashes with emptystrings under certain circumstances (see https://github.com/QGEP/qgepqwat2ili/issues/33) + """ + if val == "": + logger.warning( + "An empty string was converted to NULL, to workaround ili2pg issue. This should have no impact on output.", + ) + val = None + return val + + def truncate(val, max_length): + """ + Raises a warning if values gets truncated + """ + if val is None: + return None + if len(val) > max_length: + # _log() got an unexpected keyword argument 'stacklevel' + # logger.warning(f"Value '{val}' exceeds expected length ({max_length})", stacklevel=2) + logger.warning(f"Value '{val}' exceeds expected length ({max_length})") + return val[0:max_length] + + def modulo_angle(val): + """ + Returns an angle between 0 and 359.9 (for Orientierung in Base_d-20181005.ili) + """ + if val is None: + return None + + # add orientation + val = val + float(labelorientation) + + val = val % 360.0 + if val > 359.9: + val = 0 + + logger.info(f"modulo_angle - added orientation: {labelorientation}") + print("modulo_angle - added orientation: ", str(labelorientation)) + + return val + + def check_fk_in_subsetid(subset, relation): + """ + checks, whether foreignkey is in the subset_ids - if yes it return the tid of the foreignkey, if no it will return None + """ + # first check for None, as is get_tid + if relation is None: + return None + + logger.debug(f"check_fk_in_subsetid - Subset ID's '{subset}'") + # get the value of the fk_ attribute as str out of the relation to be able to check whether it is in the subset + fremdschluesselstr = getattr(relation, "obj_id") + logger.debug(f"check_fk_in_subsetid - fremdschluesselstr '{fremdschluesselstr}'") + + if fremdschluesselstr in subset: + logger.debug(f"check_fk_in_subsetid - '{fremdschluesselstr}' is in subset ") + logger.debug(f"check_fk_in_subsetid - tid = '{tid_maker.tid_for_row(relation)}' ") + return tid_maker.tid_for_row(relation) + else: + if filtered: + logger.warning( + f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in filtered subset - replaced with None instead!" + ) + return None + else: + logger.warning( + f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in datamodel - replaced with None instead!" + ) + return None + + def create_metaattributes(row): + metaattribute = ABWASSER.metaattribute( + # FIELDS TO MAP TO ABWASSER.metaattribute + # --- metaattribute --- + # datenherr=getattr(row.fk_dataowner__REL, "name", "unknown"), # TODO : is unknown ok ? + # datenlieferant=getattr(row.fk_provider__REL, "name", "unknown"), # TODO : is unknown ok ? + # obj_id instead of name + datenherr=getattr( + row.fk_dataowner__REL, "obj_id", "unknown" + ), # TODO : is unknown ok ? + datenlieferant=getattr( + row.fk_provider__REL, "obj_id", "unknown" + ), # TODO : is unknown ok ? + letzte_aenderung=row.last_modification, + sia405_baseclass_metaattribute=get_tid(row), + # OD : is this OK ? Don't we need a different t_id from what inserted above in organisation ? if so, consider adding a "for_class" arg to tid_for_row + t_id=get_tid(row), + t_seq=0, + ) + abwasser_session.add(metaattribute) + + def base_common(row, type_name): + """ + Returns common attributes for base + """ + return { + "t_ili_tid": row.obj_id, + "t_type": type_name, + "obj_id": row.obj_id, + "t_id": get_tid(row), + } + + def wastewater_structure_common(row): + """ + Returns common attributes for wastewater_structure + ATTENTION : Mapping of 3D wastewater_structure->abwasserbauerk + is not fully implemented. + """ + return { + # --- abwasserbauwerk --- + "akten": row.records, + "astatus": get_vl(row.status__REL), + "baujahr": row.year_of_construction, + "baulicherzustand": get_vl(row.structure_condition__REL), + "baulos": row.contract_section, + "bemerkung": truncate(emptystr_to_null(row.remark), 80), + "betreiberref": get_tid(row.fk_operator__REL), + "bezeichnung": null_to_emptystr(row.identifier), + "bruttokosten": row.gross_costs, + "detailgeometrie": ST_Force2D(row.detail_geometry_geometry), + "eigentuemerref": get_tid(row.fk_owner__REL), + "ersatzjahr": row.year_of_replacement, + "finanzierung": get_vl(row.financing__REL), + "inspektionsintervall": row.inspection_interval, + "sanierungsbedarf": get_vl(row.renovation_necessity__REL), + "standortname": row.location_name, + "subventionen": row.subsidies, + "wbw_basisjahr": row.rv_base_year, + "wbw_bauart": get_vl(row.rv_construction_type__REL), + "wiederbeschaffungswert": row.replacement_value, + "zugaenglichkeit": get_vl(row.accessibility__REL), + } + + def wastewater_networkelement_common(row): + """ + Returns common attributes for network_element + """ + + return { + # "abwasserbauwerkref": get_tid(row.fk_wastewater_structure__REL), + # 6.11.2024 Besides wn_id and re_id we also need ws_obj_ids in a separate subset - call it ws_subset_id / 15.11.2024 integrated wws in subset_ids + "abwasserbauwerkref": check_fk_in_subsetid( + subset_ids, row.fk_wastewater_structure__REL + ), + "bemerkung": truncate(emptystr_to_null(row.remark), 80), + "bezeichnung": null_to_emptystr(row.identifier), + } + + def structure_part_common(row): + """ + Returns common attributes for structure_part + """ + return { + # abwasserbauwerkref is MANDATORY, so it cannot be set to NULL + "abwasserbauwerkref": get_tid(row.fk_wastewater_structure__REL), + # "abwasserbauwerkref": check_fk_in_subsetid( + # subset_ids, row.fk_wastewater_structure__REL + # ), + "bemerkung": truncate(emptystr_to_null(row.remark), 80), + "bezeichnung": null_to_emptystr(row.identifier), + "instandstellung": get_vl(row.renovation_demand__REL), + } + + def textpos_common(row, t_type, geojson_crs_def): + """ + Returns common attributes for textpos + """ + t_id = tid_maker.next_tid() + return { + "t_id": t_id, + "t_type": t_type, + "t_ili_tid": t_id, + # --- TextPos --- + "textpos": ST_GeomFromGeoJSON( + json.dumps( + { + "type": "Point", + "coordinates": row["geometry"]["coordinates"], + "crs": geojson_crs_def, + } + ) + ), + "textori": modulo_angle(row["properties"]["LabelRotation"]), + "texthali": "Left", # can be Left/Center/Right + "textvali": "Bottom", # can be Top,Cap,Half,Base,Bottom + # --- SIA405_TextPos --- + "plantyp": row["properties"]["scale"], + "textinhalt": row["properties"]["LabelText"], + "bemerkung": None, + } + qgep_export_utils = QgepExportUtils( tid_maker=tid_maker, current_basket=current_basket, @@ -189,7 +509,55 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske qgep_export_utils.export_pipe_profile() logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") - qgep_export_utils.export_reach_point() + + query = qgep_session.query(QGEP.reach_point) + if filtered: + query = query.join( + QGEP.reach, + or_( + QGEP.reach_point.obj_id == QGEP.reach.fk_reach_point_from, + QGEP.reach_point.obj_id == QGEP.reach.fk_reach_point_to, + ), + ).filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + for row in query: + + # AVAILABLE FIELDS IN QGEP.reach_point + + # --- reach_point --- + # elevation_accuracy, fk_dataowner, fk_provider, fk_wastewater_networkelement, identifier, last_modification, level, obj_id, outlet_shape, position_of_connection, remark, situation_geometry + + # --- _bwrel_ --- + # examination__BWREL_fk_reach_point, reach__BWREL_fk_reach_point_from, reach__BWREL_fk_reach_point_to + + # --- _rel_ --- + # elevation_accuracy__REL, fk_dataowner__REL, fk_provider__REL, fk_wastewater_networkelement__REL, outlet_shape__REL + + haltungspunkt = ABWASSER.haltungspunkt( + # FIELDS TO MAP TO ABWASSER.haltungspunkt + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "haltungspunkt"), + # --- haltungspunkt --- + # changed call from get_tid to check_fk_in_subsetid so it does not write foreignkeys on elements that do not exist + # abwassernetzelementref=get_tid(row.fk_wastewater_networkelement__REL), + abwassernetzelementref=check_fk_in_subsetid( + subset_ids, row.fk_wastewater_networkelement__REL + ), + auslaufform=get_vl(row.outlet_shape__REL), + bemerkung=truncate(emptystr_to_null(row.remark), 80), + bezeichnung=null_to_emptystr(row.identifier), + hoehengenauigkeit=get_vl(row.elevation_accuracy__REL), + kote=row.level, + lage=ST_Force2D(row.situation_geometry), + lage_anschluss=row.position_of_connection, + ) + abwasser_session.add(haltungspunkt) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + # qgep_export_utils.export_reach_point() logger.info( "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" @@ -238,21 +606,522 @@ def qgep_export_sia405(selection=None, labels_file=None, orientation=None, baske logger.info( "Exporting QGEP.dryweather_downspout -> ABWASSER.trockenwetterfallrohr, ABWASSER.metaattribute" ) - qgep_export_utils.export_dryweather_downspout() + + query = qgep_session.query(QGEP.dryweather_downspout) + if filtered: + logger.info(f"filtered: subset_ids = {subset_ids}") + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.dryweather_downspout + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- dryweather_downspout --- + # diameter, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, renovation_demand__REL + + trockenwetterfallrohr = ABWASSER.trockenwetterfallrohr( + # FIELDS TO MAP TO ABWASSER.trockenwetterfallrohr + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "trockenwetterfallrohr"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- trockenwetterfallrohr --- + durchmesser=row.diameter, + ) + abwasser_session.add(trockenwetterfallrohr) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info("Exporting QGEP.access_aid -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute") - qgep_export_utils.export_access_aid() + query = qgep_session.query(QGEP.access_aid) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.access_aid + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- access_aid --- + # kind, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, kind__REL, renovation_demand__REL + + einstiegshilfe = ABWASSER.einstiegshilfe( + # FIELDS TO MAP TO ABWASSER.einstiegshilfe + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "einstiegshilfe"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- einstiegshilfe --- + art=get_vl(row.kind__REL), + ) + abwasser_session.add(einstiegshilfe) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + # qgep_export_utils.export_dryweather_downspout() + + logger.info("Exporting QGEP.access_aid -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute") + # qgep_export_utils.export_access_aid() logger.info( "Exporting QGEP.dryweather_flume -> ABWASSER.trockenwetterrinne, ABWASSER.metaattribute" ) - qgep_export_utils.export_dryweather_flume() + + query = qgep_session.query(QGEP.dryweather_flume) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.dryweather_flume + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- dryweather_flume --- + # material, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, material__REL, renovation_demand__REL + + trockenwetterrinne = ABWASSER.trockenwetterrinne( + # FIELDS TO MAP TO ABWASSER.trockenwetterrinne + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "trockenwetterrinne"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- trockenwetterrinne --- + material=get_vl(row.material__REL), + ) + abwasser_session.add(trockenwetterrinne) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info("Exporting QGEP.cover -> ABWASSER.deckel, ABWASSER.metaattribute") - qgep_export_utils.export_cover() + query = qgep_session.query(QGEP.cover) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.cover + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- cover --- + # brand, cover_shape, diameter, fastening, level, material, obj_id, positional_accuracy, situation_geometry, sludge_bucket, venting + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id, wastewater_structure__BWREL_fk_main_cover + + # --- _rel_ --- + # cover_shape__REL, fastening__REL, fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, material__REL, positional_accuracy__REL, renovation_demand__REL, sludge_bucket__REL, venting__REL + + deckel = ABWASSER.deckel( + # FIELDS TO MAP TO ABWASSER.deckel + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "deckel"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- deckel --- + deckelform=get_vl(row.cover_shape__REL), + durchmesser=row.diameter, + entlueftung=get_vl(row.venting__REL), + fabrikat=row.brand, + kote=row.level, + lage=ST_Force2D(row.situation_geometry), + lagegenauigkeit=get_vl(row.positional_accuracy__REL), + material=get_vl(row.material__REL), + schlammeimer=get_vl(row.sludge_bucket__REL), + verschluss=get_vl(row.fastening__REL), + ) + abwasser_session.add(deckel) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() logger.info("Exporting QGEP.benching -> ABWASSER.bankett, ABWASSER.metaattribute") - qgep_export_utils.export_benching() + query = qgep_session.query(QGEP.benching) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.debug(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.benching + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- benching --- + # kind, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, kind__REL, renovation_demand__REL + + bankett = ABWASSER.bankett( + # FIELDS TO MAP TO ABWASSER.bankett + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "bankett"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- bankett --- + art=get_vl(row.kind__REL), + ) + abwasser_session.add(bankett) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + # VSA -KEK commented out + + # logger.info("Exporting QGEP.examination -> ABWASSER.untersuchung, ABWASSER.metaattribute") + # query = qgep_session.query(QGEP.examination) + # if filtered: + # query = ( + # query.join(QGEP.re_maintenance_event_wastewater_structure) + # .join(QGEP.wastewater_structure) + # .join(QGEP.wastewater_networkelement) + # .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + + # for row in query: + + # # AVAILABLE FIELDS IN QGEP.examination + + # # --- maintenance_event --- + # # --- examination --- + # # equipment, fk_reach_point, from_point_identifier, inspected_length, obj_id, recording_type, to_point_identifier, vehicle, videonumber, weather + + # # --- _bwrel_ --- + # # damage__BWREL_fk_examination, re_maintenance_event_wastewater_structure__BWREL_fk_maintenance_event + + # # --- _rel_ --- + # # fk_dataowner__REL, fk_operating_company__REL, fk_provider__REL, fk_reach_point__REL, kind__REL, recording_type__REL, status__REL, weather__REL + # logger.warning( + # f"QGEP field maintenance_event.active_zone has no equivalent in the interlis model. It will be ignored." + # ) + + # untersuchung = ABWASSER.untersuchung( + # # FIELDS TO MAP TO ABWASSER.untersuchung + # # --- baseclass --- + # # --- sia405_baseclass --- + # **base_common(row, "untersuchung"), + # # --- erhaltungsereignis --- + # # abwasserbauwerkref=row.REPLACE_ME, # TODO : convert this to M2N relation through re_maintenance_event_wastewater_structure + # art=get_vl(row.kind__REL), + # astatus=get_vl(row.status__REL), + # ausfuehrende_firmaref=get_tid(row.fk_operating_company__REL), + # ausfuehrender=row.operator, + # bemerkung=truncate(emptystr_to_null(row.remark), 80), + # bezeichnung=null_to_emptystr(row.identifier), + # datengrundlage=row.base_data, + # dauer=row.duration, + # detaildaten=row.data_details, + # ergebnis=row.result, + # grund=row.reason, + # kosten=row.cost, + # zeitpunkt=row.time_point, + # # --- untersuchung --- + # bispunktbezeichnung=row.to_point_identifier, + # erfassungsart=get_vl(row.recording_type__REL), + # fahrzeug=row.vehicle, + # geraet=row.equipment, + # haltungspunktref=get_tid(row.fk_reach_point__REL), + # inspizierte_laenge=row.inspected_length, + # videonummer=row.videonumber, + # vonpunktbezeichnung=row.from_point_identifier, + # witterung=get_vl(row.weather__REL), + # ) + # abwasser_session.add(untersuchung) + # create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # abwasser_session.flush() + + # logger.info("Exporting QGEP.damage_manhole -> ABWASSER.normschachtschaden, ABWASSER.metaattribute") + # query = qgep_session.query(QGEP.damage_manhole) + # if filtered: + # query = ( + # query.join(QGEP.examination) + # .join(QGEP.re_maintenance_event_wastewater_structure) + # .join(QGEP.wastewater_structure) + # .join(QGEP.wastewater_networkelement) + # .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + # for row in query: + + # # AVAILABLE FIELDS IN QGEP.damage_manhole + + # # --- damage --- + + # # --- damage_manhole --- + # # manhole_damage_code, manhole_shaft_area, obj_id + + # # --- _bwrel_ --- + # # damage_channel_channel_damage_code__BWREL_obj_id + + # # --- _rel_ --- + # # connection__REL, fk_dataowner__REL, fk_examination__REL, fk_provider__REL, manhole_damage_code__REL, manhole_shaft_area__REL, single_damage_class__REL + + # normschachtschaden = ABWASSER.normschachtschaden( + # # FIELDS TO MAP TO ABWASSER.normschachtschaden + # # --- baseclass --- + # # --- sia405_baseclass --- + # **base_common(row, "normschachtschaden"), + # # --- schaden --- + # anmerkung=row.comments, + # ansichtsparameter=row.view_parameters, + # einzelschadenklasse=get_vl(row.single_damage_class__REL), + # streckenschaden=row.damage_reach, + # untersuchungref=get_tid(row.fk_examination__REL), + # verbindung=get_vl(row.connection__REL), + # videozaehlerstand=row.video_counter, + # # --- normschachtschaden --- + # distanz=row.distance, + # quantifizierung1=row.quantification1, + # quantifizierung2=row.quantification2, + # schachtbereich=get_vl(row.manhole_shaft_area__REL), + # schachtschadencode=get_vl(row.manhole_damage_code__REL), + # schadenlageanfang=row.damage_begin, + # schadenlageende=row.damage_end, + # ) + # abwasser_session.add(normschachtschaden) + # create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # abwasser_session.flush() + + # logger.info("Exporting QGEP.damage_channel -> ABWASSER.kanalschaden, ABWASSER.metaattribute") + # query = qgep_session.query(QGEP.damage_channel) + # if filtered: + # query = ( + # query.join(QGEP.examination) + # .join(QGEP.re_maintenance_event_wastewater_structure) + # .join(QGEP.wastewater_structure) + # .join(QGEP.wastewater_networkelement) + # .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + # for row in query: + + # # AVAILABLE FIELDS IN QGEP.damage_channel + + # # --- damage --- + # # comments, connection, damage_begin, damage_end, damage_reach, distance, fk_dataowner, fk_examination, fk_provider, last_modification, quantification1, quantification2, single_damage_class, video_counter, view_parameters + + # # --- damage_channel --- + # # , obj_id + + # # --- _bwrel_ --- + # # damage_channel_channel_damage_code__BWREL_obj_id + + # # --- _rel_ --- + # # channel_damage_code__REL, connection__REL, fk_dataowner__REL, fk_examination__REL, fk_provider__REL, single_damage_class__REL + + # kanalschaden = ABWASSER.kanalschaden( + # # FIELDS TO MAP TO ABWASSER.kanalschaden + # # --- baseclass --- + # # --- sia405_baseclass --- + # **base_common(row, "kanalschaden"), + # # --- schaden --- + # anmerkung=row.comments, + # ansichtsparameter=row.view_parameters, + # einzelschadenklasse=get_vl(row.single_damage_class__REL), + # streckenschaden=row.damage_reach, + # untersuchungref=get_tid(row.fk_examination__REL), + # verbindung=get_vl(row.connection__REL), + # videozaehlerstand=row.video_counter, + # # --- kanalschaden --- + # distanz=row.distance, + # kanalschadencode=get_vl(row.channel_damage_code__REL), + # quantifizierung1=row.quantification1, + # quantifizierung2=row.quantification2, + # schadenlageanfang=row.damage_begin, + # schadenlageende=row.damage_end, + # ) + # abwasser_session.add(kanalschaden) + # create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # abwasser_session.flush() + + # logger.info("Exporting QGEP.data_media -> ABWASSER.datentraeger, ABWASSER.metaattribute") + # query = qgep_session.query(QGEP.data_media) + # for row in query: + + # # AVAILABLE FIELDS IN QGEP.data_media + + # # --- data_media --- + # # fk_dataowner, fk_provider, identifier, kind, last_modification, location, obj_id, path, remark + + # # --- _rel_ --- + # # fk_dataowner__REL, fk_provider__REL, kind__REL + + # datentraeger = ABWASSER.datentraeger( + # # FIELDS TO MAP TO ABWASSER.datentraeger + # # --- baseclass --- + # # --- sia405_baseclass --- + # **base_common(row, "datentraeger"), + # # --- datentraeger --- + # art=get_vl(row.kind__REL), + # bemerkung=truncate(emptystr_to_null(row.remark), 80), + # bezeichnung=null_to_emptystr(row.identifier), + # pfad=row.path, + # standort=row.location, + # ) + # abwasser_session.add(datentraeger) + # create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # abwasser_session.flush() + + # logger.info("Exporting QGEP.file -> ABWASSER.datei, ABWASSER.metaattribute") + # query = qgep_session.query(QGEP.file) + # if filtered: + # query = ( + # query.outerjoin(QGEP.damage, QGEP.file.object == QGEP.damage.obj_id) + # .join( + # QGEP.examination, + # or_(QGEP.file.object == QGEP.damage.obj_id, QGEP.file.object == QGEP.examination.obj_id), + # ) + # .join(QGEP.re_maintenance_event_wastewater_structure) + # .join(QGEP.wastewater_structure) + # .join(QGEP.wastewater_networkelement) + # .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + # for row in query: + + # # AVAILABLE FIELDS IN QGEP.file + + # # --- file --- + # # class, fk_data_media, fk_dataowner, fk_provider, identifier, kind, last_modification, obj_id, object, path_relative, remark + + # # --- _rel_ --- + # # class__REL, fk_dataowner__REL, fk_provider__REL, kind__REL + + # datei = ABWASSER.datei( + # # FIELDS TO MAP TO ABWASSER.datei + # # --- baseclass --- + # # --- sia405_baseclass --- + # **base_common(row, "datei"), + # # --- datei --- + # art=get_vl(row.kind__REL) or "andere", + # bemerkung=truncate(emptystr_to_null(row.remark), 80), + # bezeichnung=null_to_emptystr(row.identifier), + # datentraegerref=get_tid(row.fk_data_media__REL), + # klasse=get_vl(row.class__REL), + # objekt=null_to_emptystr(row.object), + # relativpfad=row.path_relative, + # ) + # abwasser_session.add(datei) + # create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # abwasser_session.flush() + + # qgep_export_utils.export_dryweather_flume() + + # logger.info("Exporting QGEP.cover -> ABWASSER.deckel, ABWASSER.metaattribute") + # qgep_export_utils.export_cover() + + # logger.info("Exporting QGEP.benching -> ABWASSER.bankett, ABWASSER.metaattribute") + # qgep_export_utils.export_benching() # Labels # Note: these are extracted from the optional labels file (not exported from the QGEP database) diff --git a/qgepqwat2ili/qgepsia405/export.py.old b/qgepqwat2ili/qgepsia405/export.py.old new file mode 100644 index 00000000..5395fab1 --- /dev/null +++ b/qgepqwat2ili/qgepsia405/export.py.old @@ -0,0 +1,1299 @@ +import json + +from geoalchemy2.functions import ST_Force2D, ST_GeomFromGeoJSON +from sqlalchemy import or_ +from sqlalchemy.orm import Session +from sqlalchemy.sql import text + +from .. import utils + +# 4.10.2024 +# 6.11.2024 replaced with / 15.11.2024 get_ws_selected_ww_networkelements added +from ..utils.ili2db import ( + add_to_selection, + get_ws_selected_ww_networkelements, + get_ws_wn_ids, + remove_from_selection, +) +from ..utils.various import logger +from .model_abwasser import get_abwasser_model +from .model_qgep import get_qgep_model + + +def qgep_export(selection=None, labels_file=None, orientation=None): + """ + Export data from the QGEP model into the ili2pg model. + + Args: + selection: if provided, limits the export to networkelements that are provided in the selection + """ + + QGEP = get_qgep_model() + ABWASSER = get_abwasser_model() + + # Logging disabled (very slow) + # qgep_session = Session(utils.sqlalchemy.create_engine(logger_name="qgep"), autocommit=False, autoflush=False) + # abwasser_session = Session(utils.sqlalchemy.create_engine(logger_name="abwasser"), autocommit=False, autoflush=False) + qgep_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False) + abwasser_session = Session(utils.sqlalchemy.create_engine(), autocommit=False, autoflush=False) + tid_maker = utils.ili2db.TidMaker(id_attribute="obj_id") + + # backport from tww https://github.com/teksi/wastewater/blob/3acfba249866d299f8a22e249d9f1e475fe7b88d/plugin/teksi_wastewater/interlis/interlis_model_mapping/interlis_exporter_to_intermediate_schema.py#L83 + abwasser_session.execute(text("SET CONSTRAINTS ALL DEFERRED;")) + + # 1. Filtering - check if selection + filtered = selection is not None + subset_ids = selection if selection is not None else [] + + # 2. check if wastewater_structures exist that are not part of SIA 405 Abwasser (in Release 2015 this is the class wwtp_structures, in Release 2020 it will be more - to be extended in tww) + ws_off_sia405abwasser_list = None + ws_off_sia405abwasser_list = get_ws_wn_ids("wwtp_structure") + + # 3. Show ws_off_sia405abwasser_list + logger.debug( + f"ws_off_sia405abwasser_list : {ws_off_sia405abwasser_list}", + ) + + # 4. check if filtered + if filtered: + if ws_off_sia405abwasser_list: + # take out ws_off_sia405abwasser_list from selection + subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) + # else do nothing + else: + if ws_off_sia405abwasser_list: + # add all data of wastewater_structures to selection + subset_ids = add_to_selection(subset_ids, get_ws_wn_ids("wastewater_structure")) + logger.debug( + f"subset_ids of all wws : {subset_ids}", + ) + # take out ws_off_sia405abwasser_list from selection + subset_ids = remove_from_selection(subset_ids, ws_off_sia405abwasser_list) + logger.debug( + f"subset_ids of all wws minus ws_off_sia405abwasser_list: {subset_ids}", + ) + # add reach_ids + # subset_ids = add_to_selection(subset_ids, get_cl_re_ids("channel")) + # treat export as with a selection + filtered = True + + # else do nothing + + # 5. get and add all id's of connected wastewater_structures (not only of wastewater_network_element (reach, wwn) + subset_wws_ids = get_ws_selected_ww_networkelements(subset_ids) + subset_ids = add_to_selection(subset_ids, subset_wws_ids) + + logger.debug( + f"subset_ids with wws : {subset_ids}", + ) + + # Orientation + oriented = orientation is not None + if oriented: + labelorientation = orientation + else: + labelorientation = 0 + + def get_tid(relation): + """ + Makes a tid for a relation + """ + if relation is None: + return None + + return tid_maker.tid_for_row(relation) + + def get_vl(relation): + """ + Gets a literal value from a value list relation + """ + if relation is None: + return None + return relation.value_de + + def null_to_emptystr(val): + """ + Converts nulls to blank strings and raises a warning + """ + if val is None: + logger.warning( + "A mandatory value was null. It will be cast to a blank string, and probably cause validation errors", + ) + val = "" + return val + + def emptystr_to_null(val): + """ + Converts blank strings to nulls and raises a warning + + This is needed as is seems ili2pg 4.4.6 crashes with emptystrings under certain circumstances (see https://github.com/QGEP/qgepqwat2ili/issues/33) + """ + if val == "": + logger.warning( + "An empty string was converted to NULL, to workaround ili2pg issue. This should have no impact on output.", + ) + val = None + return val + + def truncate(val, max_length): + """ + Raises a warning if values gets truncated + """ + if val is None: + return None + if len(val) > max_length: + # _log() got an unexpected keyword argument 'stacklevel' + # logger.warning(f"Value '{val}' exceeds expected length ({max_length})", stacklevel=2) + logger.warning(f"Value '{val}' exceeds expected length ({max_length})") + return val[0:max_length] + + def modulo_angle(val): + """ + Returns an angle between 0 and 359.9 (for Orientierung in Base_d-20181005.ili) + """ + if val is None: + return None + + # add orientation + val = val + float(labelorientation) + + val = val % 360.0 + if val > 359.9: + val = 0 + + logger.info(f"modulo_angle - added orientation: {labelorientation}") + print("modulo_angle - added orientation: ", str(labelorientation)) + + return val + + def check_fk_in_subsetid(subset, relation): + """ + checks, whether foreignkey is in the subset_ids - if yes it return the tid of the foreignkey, if no it will return None + """ + # first check for None, as is get_tid + if relation is None: + return None + + logger.debug(f"check_fk_in_subsetid - Subset ID's '{subset}'") + # get the value of the fk_ attribute as str out of the relation to be able to check whether it is in the subset + fremdschluesselstr = getattr(relation, "obj_id") + logger.debug(f"check_fk_in_subsetid - fremdschluesselstr '{fremdschluesselstr}'") + + if fremdschluesselstr in subset: + logger.debug(f"check_fk_in_subsetid - '{fremdschluesselstr}' is in subset ") + logger.debug(f"check_fk_in_subsetid - tid = '{tid_maker.tid_for_row(relation)}' ") + return tid_maker.tid_for_row(relation) + else: + if filtered: + logger.warning( + f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in filtered subset - replaced with None instead!" + ) + return None + else: + logger.warning( + f"check_fk_in_subsetid - '{fremdschluesselstr}' is not in datamodel - replaced with None instead!" + ) + return None + + def create_metaattributes(row): + metaattribute = ABWASSER.metaattribute( + # FIELDS TO MAP TO ABWASSER.metaattribute + # --- metaattribute --- + # datenherr=getattr(row.fk_dataowner__REL, "name", "unknown"), # TODO : is unknown ok ? + # datenlieferant=getattr(row.fk_provider__REL, "name", "unknown"), # TODO : is unknown ok ? + # obj_id instead of name + datenherr=getattr( + row.fk_dataowner__REL, "obj_id", "unknown" + ), # TODO : is unknown ok ? + datenlieferant=getattr( + row.fk_provider__REL, "obj_id", "unknown" + ), # TODO : is unknown ok ? + letzte_aenderung=row.last_modification, + sia405_baseclass_metaattribute=get_tid(row), + # OD : is this OK ? Don't we need a different t_id from what inserted above in organisation ? if so, consider adding a "for_class" arg to tid_for_row + t_id=get_tid(row), + t_seq=0, + ) + abwasser_session.add(metaattribute) + + def base_common(row, type_name): + """ + Returns common attributes for base + """ + return { + "t_ili_tid": row.obj_id, + "t_type": type_name, + "obj_id": row.obj_id, + "t_id": get_tid(row), + } + + def wastewater_structure_common(row): + """ + Returns common attributes for wastewater_structure + ATTENTION : Mapping of 3D wastewater_structure->abwasserbauerk + is not fully implemented. + """ + return { + # --- abwasserbauwerk --- + "akten": row.records, + "astatus": get_vl(row.status__REL), + "baujahr": row.year_of_construction, + "baulicherzustand": get_vl(row.structure_condition__REL), + "baulos": row.contract_section, + "bemerkung": truncate(emptystr_to_null(row.remark), 80), + "betreiberref": get_tid(row.fk_operator__REL), + "bezeichnung": null_to_emptystr(row.identifier), + "bruttokosten": row.gross_costs, + "detailgeometrie": ST_Force2D(row.detail_geometry_geometry), + "eigentuemerref": get_tid(row.fk_owner__REL), + "ersatzjahr": row.year_of_replacement, + "finanzierung": get_vl(row.financing__REL), + "inspektionsintervall": row.inspection_interval, + "sanierungsbedarf": get_vl(row.renovation_necessity__REL), + "standortname": row.location_name, + "subventionen": row.subsidies, + "wbw_basisjahr": row.rv_base_year, + "wbw_bauart": get_vl(row.rv_construction_type__REL), + "wiederbeschaffungswert": row.replacement_value, + "zugaenglichkeit": get_vl(row.accessibility__REL), + } + + def wastewater_networkelement_common(row): + """ + Returns common attributes for network_element + """ + + return { + # "abwasserbauwerkref": get_tid(row.fk_wastewater_structure__REL), + # 6.11.2024 Besides wn_id and re_id we also need ws_obj_ids in a separate subset - call it ws_subset_id / 15.11.2024 integrated wws in subset_ids + "abwasserbauwerkref": check_fk_in_subsetid( + subset_ids, row.fk_wastewater_structure__REL + ), + "bemerkung": truncate(emptystr_to_null(row.remark), 80), + "bezeichnung": null_to_emptystr(row.identifier), + } + + def structure_part_common(row): + """ + Returns common attributes for structure_part + """ + return { + #"abwasserbauwerkref": get_tid(row.fk_wastewater_structure__REL), + "abwasserbauwerkref": check_fk_in_subsetid( + subset_ids, row.fk_wastewater_structure__REL + ), + "bemerkung": truncate(emptystr_to_null(row.remark), 80), + "bezeichnung": null_to_emptystr(row.identifier), + "instandstellung": get_vl(row.renovation_demand__REL), + } + + def textpos_common(row, t_type, geojson_crs_def): + """ + Returns common attributes for textpos + """ + t_id = tid_maker.next_tid() + return { + "t_id": t_id, + "t_type": t_type, + "t_ili_tid": t_id, + # --- TextPos --- + "textpos": ST_GeomFromGeoJSON( + json.dumps( + { + "type": "Point", + "coordinates": row["geometry"]["coordinates"], + "crs": geojson_crs_def, + } + ) + ), + "textori": modulo_angle(row["properties"]["LabelRotation"]), + "texthali": "Left", # can be Left/Center/Right + "textvali": "Bottom", # can be Top,Cap,Half,Base,Bottom + # --- SIA405_TextPos --- + "plantyp": row["properties"]["scale"], + "textinhalt": row["properties"]["LabelText"], + "bemerkung": None, + } + + # ADAPTED FROM 052a_sia405_abwasser_2015_2_d_interlisexport2.sql + logger.info("Exporting QGEP.organisation -> ABWASSER.organisation, ABWASSER.metaattribute") + query = qgep_session.query(QGEP.organisation) + for row in query: + + # AVAILABLE FIELDS IN QGEP.organisation + + # --- organisation --- + # fk_dataowner, fk_provider, identifier, last_modification, obj_id, remark, uid + + # --- _bwrel_ --- + # accident__BWREL_fk_dataowner, accident__BWREL_fk_provider, administrative_office__BWREL_obj_id, aquifier__BWREL_fk_dataowner, aquifier__BWREL_fk_provider, bathing_area__BWREL_fk_dataowner, bathing_area__BWREL_fk_provider, canton__BWREL_obj_id, catchment_area__BWREL_fk_dataowner, catchment_area__BWREL_fk_provider, connection_object__BWREL_fk_dataowner, connection_object__BWREL_fk_operator, connection_object__BWREL_fk_owner, connection_object__BWREL_fk_provider, control_center__BWREL_fk_dataowner, control_center__BWREL_fk_provider, cooperative__BWREL_obj_id, damage__BWREL_fk_dataowner, damage__BWREL_fk_provider, data_media__BWREL_fk_dataowner, data_media__BWREL_fk_provider, file__BWREL_fk_dataowner, file__BWREL_fk_provider, fish_pass__BWREL_fk_dataowner, fish_pass__BWREL_fk_provider, hazard_source__BWREL_fk_dataowner, hazard_source__BWREL_fk_owner, hazard_source__BWREL_fk_provider, hq_relation__BWREL_fk_dataowner, hq_relation__BWREL_fk_provider, hydr_geom_relation__BWREL_fk_dataowner, hydr_geom_relation__BWREL_fk_provider, hydr_geometry__BWREL_fk_dataowner, hydr_geometry__BWREL_fk_provider, hydraulic_char_data__BWREL_fk_dataowner, hydraulic_char_data__BWREL_fk_provider, maintenance_event__BWREL_fk_dataowner, maintenance_event__BWREL_fk_operating_company, maintenance_event__BWREL_fk_provider, measurement_result__BWREL_fk_dataowner, measurement_result__BWREL_fk_provider, measurement_series__BWREL_fk_dataowner, measurement_series__BWREL_fk_provider, measuring_device__BWREL_fk_dataowner, measuring_device__BWREL_fk_provider, measuring_point__BWREL_fk_dataowner, measuring_point__BWREL_fk_operator, measuring_point__BWREL_fk_provider, mechanical_pretreatment__BWREL_fk_dataowner, mechanical_pretreatment__BWREL_fk_provider, municipality__BWREL_obj_id, mutation__BWREL_fk_dataowner, mutation__BWREL_fk_provider, organisation__BWREL_fk_dataowner, organisation__BWREL_fk_provider, overflow__BWREL_fk_dataowner, overflow__BWREL_fk_provider, overflow_char__BWREL_fk_dataowner, overflow_char__BWREL_fk_provider, pipe_profile__BWREL_fk_dataowner, pipe_profile__BWREL_fk_provider, private__BWREL_obj_id, profile_geometry__BWREL_fk_dataowner, profile_geometry__BWREL_fk_provider, reach_point__BWREL_fk_dataowner, reach_point__BWREL_fk_provider, retention_body__BWREL_fk_dataowner, retention_body__BWREL_fk_provider, river_bank__BWREL_fk_dataowner, river_bank__BWREL_fk_provider, river_bed__BWREL_fk_dataowner, river_bed__BWREL_fk_provider, sector_water_body__BWREL_fk_dataowner, sector_water_body__BWREL_fk_provider, sludge_treatment__BWREL_fk_dataowner, sludge_treatment__BWREL_fk_provider, structure_part__BWREL_fk_dataowner, structure_part__BWREL_fk_provider, substance__BWREL_fk_dataowner, substance__BWREL_fk_provider, surface_runoff_parameters__BWREL_fk_dataowner, surface_runoff_parameters__BWREL_fk_provider, surface_water_bodies__BWREL_fk_dataowner, surface_water_bodies__BWREL_fk_provider, throttle_shut_off_unit__BWREL_fk_dataowner, throttle_shut_off_unit__BWREL_fk_provider, txt_symbol__BWREL_fk_dataowner, txt_symbol__BWREL_fk_provider, waste_water_association__BWREL_obj_id, waste_water_treatment__BWREL_fk_dataowner, waste_water_treatment__BWREL_fk_provider, waste_water_treatment_plant__BWREL_obj_id, wastewater_networkelement__BWREL_fk_dataowner, wastewater_networkelement__BWREL_fk_provider, wastewater_structure__BWREL_fk_dataowner, wastewater_structure__BWREL_fk_operator, wastewater_structure__BWREL_fk_owner, wastewater_structure__BWREL_fk_provider, wastewater_structure_symbol__BWREL_fk_dataowner, wastewater_structure_symbol__BWREL_fk_provider, water_catchment__BWREL_fk_dataowner, water_catchment__BWREL_fk_provider, water_control_structure__BWREL_fk_dataowner, water_control_structure__BWREL_fk_provider, water_course_segment__BWREL_fk_dataowner, water_course_segment__BWREL_fk_provider, wwtp_energy_use__BWREL_fk_dataowner, wwtp_energy_use__BWREL_fk_provider, zone__BWREL_fk_dataowner, zone__BWREL_fk_provider + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL + + organisation = ABWASSER.organisation( + # FIELDS TO MAP TO ABWASSER.organisation + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "organisation"), + # --- organisation --- + auid=row.uid, + bemerkung=truncate(emptystr_to_null(row.remark), 80), + bezeichnung=null_to_emptystr(row.identifier), + ) + abwasser_session.add(organisation) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info("Exporting QGEP.channel -> ABWASSER.kanal, ABWASSER.metaattribute") + query = qgep_session.query(QGEP.channel) + if filtered: + query = query.join(QGEP.wastewater_networkelement).filter( + QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + ) + for row in query: + + # AVAILABLE FIELDS IN QGEP.channel + + # --- wastewater_structure --- + # _bottom_label, _cover_label, _depth, _function_hierarchic, _input_label, _label, _output_label, _usage_current, accessibility, contract_section, detail_geometry_geometry, financing, fk_dataowner, fk_main_cover, fk_main_wastewater_node, fk_operator, fk_owner, fk_provider, gross_costs, identifier, inspection_interval, last_modification, location_name, records, remark, renovation_necessity, replacement_value, rv_base_year, rv_construction_type, status, structure_condition, subsidies, year_of_construction, year_of_replacement + + # --- _bwrel_ --- + # measuring_point__BWREL_fk_wastewater_structure, mechanical_pretreatment__BWREL_fk_wastewater_structure, re_maintenance_event_wastewater_structure__BWREL_fk_wastewater_structure, structure_part__BWREL_fk_wastewater_structure, txt_symbol__BWREL_fk_wastewater_structure, txt_text__BWREL_fk_wastewater_structure, wastewater_networkelement__BWREL_fk_wastewater_structure, wastewater_structure_symbol__BWREL_fk_wastewater_structure, wastewater_structure_text__BWREL_fk_wastewater_structure, wwtp_structure_kind__BWREL_obj_id + + # --- _rel_ --- + # accessibility__REL, bedding_encasement__REL, connection_type__REL, financing__REL, fk_dataowner__REL, fk_main_cover__REL, fk_main_wastewater_node__REL, fk_operator__REL, fk_owner__REL, fk_provider__REL, function_hierarchic__REL, function_hydraulic__REL, renovation_necessity__REL, rv_construction_type__REL, status__REL, structure_condition__REL, usage_current__REL, usage_planned__REL + + kanal = ABWASSER.kanal( + # FIELDS TO MAP TO ABWASSER.kanal + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "kanal"), + # --- abwasserbauwerk --- + **wastewater_structure_common(row), + # --- kanal --- + bettung_umhuellung=get_vl(row.bedding_encasement__REL), + funktionhierarchisch=get_vl(row.function_hierarchic__REL), + funktionhydraulisch=get_vl(row.function_hydraulic__REL), + nutzungsart_geplant=get_vl(row.usage_planned__REL), + nutzungsart_ist=get_vl(row.usage_current__REL), + rohrlaenge=row.pipe_length, + spuelintervall=row.jetting_interval, + verbindungsart=get_vl(row.connection_type__REL), + ) + abwasser_session.add(kanal) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info("Exporting QGEP.manhole -> ABWASSER.normschacht, ABWASSER.metaattribute") + query = qgep_session.query(QGEP.manhole) + if filtered: + query = query.join(QGEP.wastewater_networkelement).filter( + QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + ) + for row in query: + normschacht = ABWASSER.normschacht( + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "normschacht"), + # --- abwasserbauwerk --- + **wastewater_structure_common(row), + # --- normschacht --- + dimension1=row.dimension1, + dimension2=row.dimension2, + funktion=get_vl(row.function__REL), + material=get_vl(row.material__REL), + oberflaechenzulauf=get_vl(row.surface_inflow__REL), + ) + abwasser_session.add(normschacht) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info("Exporting QGEP.discharge_point -> ABWASSER.einleitstelle, ABWASSER.metaattribute") + query = qgep_session.query(QGEP.discharge_point) + if filtered: + query = query.join(QGEP.wastewater_networkelement).filter( + QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + ) + for row in query: + einleitstelle = ABWASSER.einleitstelle( + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "einleitstelle"), + # --- abwasserbauwerk --- + **wastewater_structure_common(row), + # --- einleitstelle --- + hochwasserkote=row.highwater_level, + relevanz=get_vl(row.relevance__REL), + terrainkote=row.terrain_level, + wasserspiegel_hydraulik=row.waterlevel_hydraulic, + ) + abwasser_session.add(einleitstelle) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info( + "Exporting QGEP.special_structure -> ABWASSER.spezialbauwerk, ABWASSER.metaattribute" + ) + query = qgep_session.query(QGEP.special_structure) + if filtered: + query = query.join(QGEP.wastewater_networkelement).filter( + QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + ) + for row in query: + + # AVAILABLE FIELDS IN QGEP.special_structure + + # --- wastewater_structure --- + # _bottom_label, _cover_label, _depth, _function_hierarchic, _input_label, _label, _output_label, _usage_current, accessibility, contract_section, detail_geometry_geometry, financing, fk_dataowner, fk_main_cover, fk_main_wastewater_node, fk_operator, fk_owner, fk_provider, gross_costs, identifier, inspection_interval, last_modification, location_name, records, remark, renovation_necessity, replacement_value, rv_base_year, rv_construction_type, status, structure_condition, subsidies, year_of_construction, year_of_replacement + + # --- special_structure --- + # bypass, emergency_spillway, function, obj_id, stormwater_tank_arrangement, upper_elevation + + # --- _bwrel_ --- + # measuring_point__BWREL_fk_wastewater_structure, mechanical_pretreatment__BWREL_fk_wastewater_structure, re_maintenance_event_wastewater_structure__BWREL_fk_wastewater_structure, structure_part__BWREL_fk_wastewater_structure, txt_symbol__BWREL_fk_wastewater_structure, txt_text__BWREL_fk_wastewater_structure, wastewater_networkelement__BWREL_fk_wastewater_structure, wastewater_structure_symbol__BWREL_fk_wastewater_structure, wastewater_structure_text__BWREL_fk_wastewater_structure, wwtp_structure_kind__BWREL_obj_id + + # --- _rel_ --- + # accessibility__REL, bypass__REL, emergency_spillway__REL, financing__REL, fk_dataowner__REL, fk_main_cover__REL, fk_main_wastewater_node__REL, fk_operator__REL, fk_owner__REL, fk_provider__REL, function__REL, renovation_necessity__REL, rv_construction_type__REL, status__REL, stormwater_tank_arrangement__REL, structure_condition__REL + + # QGEP field special_structure.upper_elevation is a 3D attribute and has no equivalent in the INTERLIS 2D model release used. It will be ignored for now and not supported with QGEP. + + spezialbauwerk = ABWASSER.spezialbauwerk( + # FIELDS TO MAP TO ABWASSER.spezialbauwerk + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "spezialbauwerk"), + # --- abwasserbauwerk --- + **wastewater_structure_common(row), + # --- spezialbauwerk --- + # TODO : WARNING : upper_elevation is not mapped + bypass=get_vl(row.bypass__REL), + funktion=get_vl(row.function__REL), + notueberlauf=get_vl(row.emergency_spillway__REL), + regenbecken_anordnung=get_vl(row.stormwater_tank_arrangement__REL), + ) + abwasser_session.add(spezialbauwerk) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info( + "Exporting QGEP.infiltration_installation -> ABWASSER.versickerungsanlage, ABWASSER.metaattribute" + ) + query = qgep_session.query(QGEP.infiltration_installation) + if filtered: + query = query.join(QGEP.wastewater_networkelement).filter( + QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + ) + for row in query: + + # AVAILABLE FIELDS IN QGEP.infiltration_installation + + # --- wastewater_structure --- + # _bottom_label, _cover_label, _depth, _function_hierarchic, _input_label, _label, _output_label, _usage_current, accessibility, contract_section, detail_geometry_geometry, financing, fk_dataowner, fk_main_cover, fk_main_wastewater_node, fk_operator, fk_owner, fk_provider, gross_costs, identifier, inspection_interval, last_modification, location_name, records, remark, renovation_necessity, replacement_value, rv_base_year, rv_construction_type, status, structure_condition, subsidies, year_of_construction, year_of_replacement + + # --- infiltration_installation --- + # absorption_capacity, defects, dimension1, dimension2, distance_to_aquifer, effective_area, emergency_spillway, fk_aquifier, kind, labeling, obj_id, seepage_utilization, upper_elevation, vehicle_access, watertightness + + # --- _bwrel_ --- + # measuring_point__BWREL_fk_wastewater_structure, mechanical_pretreatment__BWREL_fk_infiltration_installation, mechanical_pretreatment__BWREL_fk_wastewater_structure, re_maintenance_event_wastewater_structure__BWREL_fk_wastewater_structure, retention_body__BWREL_fk_infiltration_installation, structure_part__BWREL_fk_wastewater_structure, txt_symbol__BWREL_fk_wastewater_structure, txt_text__BWREL_fk_wastewater_structure, wastewater_networkelement__BWREL_fk_wastewater_structure, wastewater_structure_symbol__BWREL_fk_wastewater_structure, wastewater_structure_text__BWREL_fk_wastewater_structure, wwtp_structure_kind__BWREL_obj_id + + # --- _rel_ --- + # accessibility__REL, defects__REL, emergency_spillway__REL, financing__REL, fk_aquifier__REL, fk_dataowner__REL, fk_main_cover__REL, fk_main_wastewater_node__REL, fk_operator__REL, fk_owner__REL, fk_provider__REL, kind__REL, labeling__REL, renovation_necessity__REL, rv_construction_type__REL, seepage_utilization__REL, status__REL, structure_condition__REL, vehicle_access__REL, watertightness__REL + + logger.warning( + "QGEP field infiltration_installation.upper_elevation has no equivalent in the interlis model. It will be ignored." + ) + versickerungsanlage = ABWASSER.versickerungsanlage( + # FIELDS TO MAP TO ABWASSER.versickerungsanlage + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "versickerungsanlage"), + # --- abwasserbauwerk --- + **wastewater_structure_common(row), + # --- versickerungsanlage --- + # TODO : NOT MAPPED : upper_elevation + art=get_vl(row.kind__REL), + beschriftung=get_vl(row.labeling__REL), + dimension1=row.dimension1, + dimension2=row.dimension2, + gwdistanz=row.distance_to_aquifer, + maengel=get_vl(row.defects__REL), + notueberlauf=get_vl(row.emergency_spillway__REL), + saugwagen=get_vl(row.vehicle_access__REL), + schluckvermoegen=row.absorption_capacity, + versickerungswasser=get_vl(row.seepage_utilization__REL), + wasserdichtheit=get_vl(row.watertightness__REL), + wirksameflaeche=row.effective_area, + ) + abwasser_session.add(versickerungsanlage) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info("Exporting QGEP.pipe_profile -> ABWASSER.rohrprofil, ABWASSER.metaattribute") + query = qgep_session.query(QGEP.pipe_profile) + if filtered: + query = query.join(QGEP.reach).filter( + QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + ) + for row in query: + + # AVAILABLE FIELDS IN QGEP.pipe_profile + + # --- pipe_profile --- + # fk_dataowner, fk_provider, height_width_ratio, identifier, last_modification, obj_id, profile_type, remark + + # --- _bwrel_ --- + # profile_geometry__BWREL_fk_pipe_profile, reach__BWREL_fk_pipe_profile + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, profile_type__REL + + rohrprofil = ABWASSER.rohrprofil( + # FIELDS TO MAP TO ABWASSER.rohrprofil + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "rohrprofil"), + # --- rohrprofil --- + bemerkung=truncate(emptystr_to_null(row.remark), 80), + bezeichnung=null_to_emptystr(row.identifier), + hoehenbreitenverhaeltnis=row.height_width_ratio, + profiltyp=get_vl(row.profile_type__REL), + ) + abwasser_session.add(rohrprofil) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info("Exporting QGEP.reach_point -> ABWASSER.haltungspunkt, ABWASSER.metaattribute") + query = qgep_session.query(QGEP.reach_point) + if filtered: + query = query.join( + QGEP.reach, + or_( + QGEP.reach_point.obj_id == QGEP.reach.fk_reach_point_from, + QGEP.reach_point.obj_id == QGEP.reach.fk_reach_point_to, + ), + ).filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + for row in query: + + # AVAILABLE FIELDS IN QGEP.reach_point + + # --- reach_point --- + # elevation_accuracy, fk_dataowner, fk_provider, fk_wastewater_networkelement, identifier, last_modification, level, obj_id, outlet_shape, position_of_connection, remark, situation_geometry + + # --- _bwrel_ --- + # examination__BWREL_fk_reach_point, reach__BWREL_fk_reach_point_from, reach__BWREL_fk_reach_point_to + + # --- _rel_ --- + # elevation_accuracy__REL, fk_dataowner__REL, fk_provider__REL, fk_wastewater_networkelement__REL, outlet_shape__REL + + haltungspunkt = ABWASSER.haltungspunkt( + # FIELDS TO MAP TO ABWASSER.haltungspunkt + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "haltungspunkt"), + # --- haltungspunkt --- + # changed call from get_tid to check_fk_in_subsetid so it does not write foreignkeys on elements that do not exist + # abwassernetzelementref=get_tid(row.fk_wastewater_networkelement__REL), + abwassernetzelementref=check_fk_in_subsetid( + subset_ids, row.fk_wastewater_networkelement__REL + ), + auslaufform=get_vl(row.outlet_shape__REL), + bemerkung=truncate(emptystr_to_null(row.remark), 80), + bezeichnung=null_to_emptystr(row.identifier), + hoehengenauigkeit=get_vl(row.elevation_accuracy__REL), + kote=row.level, + lage=ST_Force2D(row.situation_geometry), + lage_anschluss=row.position_of_connection, + ) + abwasser_session.add(haltungspunkt) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info( + "Exporting QGEP.wastewater_node -> ABWASSER.abwasserknoten, ABWASSER.metaattribute" + ) + query = qgep_session.query(QGEP.wastewater_node) + if filtered: + query = query.filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + for row in query: + + # AVAILABLE FIELDS IN QGEP.wastewater_node + + # --- wastewater_networkelement --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # --- wastewater_node --- + + # --- _bwrel_ --- + # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, hydraulic_char_data__BWREL_fk_wastewater_node, overflow__BWREL_fk_overflow_to, overflow__BWREL_fk_wastewater_node, reach_point__BWREL_fk_wastewater_networkelement, throttle_shut_off_unit__BWREL_fk_wastewater_node, wastewater_structure__BWREL_fk_main_wastewater_node + + # --- _rel_ --- + # fk_dataowner__REL, fk_hydr_geometry__REL, fk_provider__REL, fk_wastewater_structure__REL + + # QGEP field wastewater_node.fk_hydr_geometry has no equivalent in the interlis model. It will be ignored. + + abwasserknoten = ABWASSER.abwasserknoten( + # FIELDS TO MAP TO ABWASSER.abwasserknoten + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "abwasserknoten"), + # --- abwassernetzelement --- + **wastewater_networkelement_common(row), + # --- abwasserknoten --- + # TODO : WARNING : fk_hydr_geometry is not mapped + lage=ST_Force2D(row.situation_geometry), + rueckstaukote=row.backflow_level, + sohlenkote=row.bottom_level, + ) + abwasser_session.add(abwasserknoten) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info("Exporting QGEP.reach -> ABWASSER.haltung, ABWASSER.metaattribute") + query = qgep_session.query(QGEP.reach) + if filtered: + query = query.filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + for row in query: + + # AVAILABLE FIELDS IN QGEP.reach + + # --- wastewater_networkelement --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark + + # --- reach --- + # clear_height, coefficient_of_friction, elevation_determination, fk_pipe_profile, fk_reach_point_from, fk_reach_point_to, horizontal_positioning, inside_coating, length_effective, material, obj_id, progression_geometry, reliner_material, reliner_nominal_size, relining_construction, relining_kind, ring_stiffness, slope_building_plan, wall_roughness + + # --- _bwrel_ --- + # catchment_area__BWREL_fk_wastewater_networkelement_rw_current, catchment_area__BWREL_fk_wastewater_networkelement_rw_planned, catchment_area__BWREL_fk_wastewater_networkelement_ww_current, catchment_area__BWREL_fk_wastewater_networkelement_ww_planned, connection_object__BWREL_fk_wastewater_networkelement, reach_point__BWREL_fk_wastewater_networkelement, reach_text__BWREL_fk_reach, txt_text__BWREL_fk_reach + + # --- _rel_ --- + # elevation_determination__REL, fk_dataowner__REL, fk_pipe_profile__REL, fk_provider__REL, fk_reach_point_from__REL, fk_reach_point_to__REL, fk_wastewater_structure__REL, horizontal_positioning__REL, inside_coating__REL, material__REL, reliner_material__REL, relining_construction__REL, relining_kind__REL + + # QGEP field reach.elevation_determination has no equivalent in the interlis model. It will be ignored. + + haltung = ABWASSER.haltung( + # FIELDS TO MAP TO ABWASSER.haltung + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "haltung"), + # --- abwassernetzelement --- + **wastewater_networkelement_common(row), + # --- haltung --- + # NOT MAPPED : elevation_determination + innenschutz=get_vl(row.inside_coating__REL), + laengeeffektiv=row.length_effective, + lagebestimmung=get_vl(row.horizontal_positioning__REL), + lichte_hoehe=row.clear_height, + material=get_vl(row.material__REL), + nachhaltungspunktref=get_tid(row.fk_reach_point_to__REL), + plangefaelle=row.slope_building_plan, # TODO : check, does this need conversion ? + reibungsbeiwert=row.coefficient_of_friction, + reliner_art=get_vl(row.relining_kind__REL), + reliner_bautechnik=get_vl(row.relining_construction__REL), + reliner_material=get_vl(row.reliner_material__REL), + reliner_nennweite=row.reliner_nominal_size, + ringsteifigkeit=row.ring_stiffness, + rohrprofilref=get_tid(row.fk_pipe_profile__REL), + verlauf=ST_Force2D(row.progression_geometry), + vonhaltungspunktref=get_tid(row.fk_reach_point_from__REL), + wandrauhigkeit=row.wall_roughness, + ) + abwasser_session.add(haltung) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info( + "Exporting QGEP.dryweather_downspout -> ABWASSER.trockenwetterfallrohr, ABWASSER.metaattribute" + ) + + query = qgep_session.query(QGEP.dryweather_downspout) + if filtered: + logger.info(f"filtered: subset_ids = {subset_ids}") + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.info(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.dryweather_downspout + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- dryweather_downspout --- + # diameter, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, renovation_demand__REL + + trockenwetterfallrohr = ABWASSER.trockenwetterfallrohr( + # FIELDS TO MAP TO ABWASSER.trockenwetterfallrohr + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "trockenwetterfallrohr"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- trockenwetterfallrohr --- + durchmesser=row.diameter, + ) + abwasser_session.add(trockenwetterfallrohr) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info("Exporting QGEP.access_aid -> ABWASSER.einstiegshilfe, ABWASSER.metaattribute") + query = qgep_session.query(QGEP.access_aid) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.info(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.access_aid + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- access_aid --- + # kind, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, kind__REL, renovation_demand__REL + + einstiegshilfe = ABWASSER.einstiegshilfe( + # FIELDS TO MAP TO ABWASSER.einstiegshilfe + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "einstiegshilfe"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- einstiegshilfe --- + art=get_vl(row.kind__REL), + ) + abwasser_session.add(einstiegshilfe) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info( + "Exporting QGEP.dryweather_flume -> ABWASSER.trockenwetterrinne, ABWASSER.metaattribute" + ) + query = qgep_session.query(QGEP.dryweather_flume) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.info(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.dryweather_flume + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- dryweather_flume --- + # material, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, material__REL, renovation_demand__REL + + trockenwetterrinne = ABWASSER.trockenwetterrinne( + # FIELDS TO MAP TO ABWASSER.trockenwetterrinne + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "trockenwetterrinne"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- trockenwetterrinne --- + material=get_vl(row.material__REL), + ) + abwasser_session.add(trockenwetterrinne) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info("Exporting QGEP.cover -> ABWASSER.deckel, ABWASSER.metaattribute") + query = qgep_session.query(QGEP.cover) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.info(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.cover + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- cover --- + # brand, cover_shape, diameter, fastening, level, material, obj_id, positional_accuracy, situation_geometry, sludge_bucket, venting + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id, wastewater_structure__BWREL_fk_main_cover + + # --- _rel_ --- + # cover_shape__REL, fastening__REL, fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, material__REL, positional_accuracy__REL, renovation_demand__REL, sludge_bucket__REL, venting__REL + + deckel = ABWASSER.deckel( + # FIELDS TO MAP TO ABWASSER.deckel + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "deckel"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- deckel --- + deckelform=get_vl(row.cover_shape__REL), + durchmesser=row.diameter, + entlueftung=get_vl(row.venting__REL), + fabrikat=row.brand, + kote=row.level, + lage=ST_Force2D(row.situation_geometry), + lagegenauigkeit=get_vl(row.positional_accuracy__REL), + material=get_vl(row.material__REL), + schlammeimer=get_vl(row.sludge_bucket__REL), + verschluss=get_vl(row.fastening__REL), + ) + abwasser_session.add(deckel) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + logger.info("Exporting QGEP.benching -> ABWASSER.bankett, ABWASSER.metaattribute") + query = qgep_session.query(QGEP.benching) + if filtered: + # query = query.join(QGEP.wastewater_structure, QGEP.wastewater_networkelement).filter( + # QGEP.wastewater_networkelement.obj_id.in_(subset_ids) + # ) + query = ( + query.join( + QGEP.wastewater_structure, + QGEP.structure_part.fk_wastewater_structure == QGEP.wastewater_structure.obj_id, + ) + .join(QGEP.wastewater_networkelement) + .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + ) + # add sql statement to logger + statement = query.statement + logger.info(f" selection query = {statement}") + for row in query: + + # AVAILABLE FIELDS IN QGEP.benching + + # --- structure_part --- + # fk_dataowner, fk_provider, fk_wastewater_structure, identifier, last_modification, remark, renovation_demand + + # --- benching --- + # kind, obj_id + + # --- _bwrel_ --- + # access_aid_kind__BWREL_obj_id, backflow_prevention__BWREL_obj_id, benching_kind__BWREL_obj_id, dryweather_flume_material__BWREL_obj_id, electric_equipment__BWREL_obj_id, electromechanical_equipment__BWREL_obj_id, solids_retention__BWREL_obj_id, tank_cleaning__BWREL_obj_id, tank_emptying__BWREL_obj_id + + # --- _rel_ --- + # fk_dataowner__REL, fk_provider__REL, fk_wastewater_structure__REL, kind__REL, renovation_demand__REL + + bankett = ABWASSER.bankett( + # FIELDS TO MAP TO ABWASSER.bankett + # --- baseclass --- + # --- sia405_baseclass --- + **base_common(row, "bankett"), + # --- bauwerksteil --- + **structure_part_common(row), + # --- bankett --- + art=get_vl(row.kind__REL), + ) + abwasser_session.add(bankett) + create_metaattributes(row) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + # VSA -KEK commented out + + # logger.info("Exporting QGEP.examination -> ABWASSER.untersuchung, ABWASSER.metaattribute") + # query = qgep_session.query(QGEP.examination) + # if filtered: + # query = ( + # query.join(QGEP.re_maintenance_event_wastewater_structure) + # .join(QGEP.wastewater_structure) + # .join(QGEP.wastewater_networkelement) + # .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + + # for row in query: + + # # AVAILABLE FIELDS IN QGEP.examination + + # # --- maintenance_event --- + # # --- examination --- + # # equipment, fk_reach_point, from_point_identifier, inspected_length, obj_id, recording_type, to_point_identifier, vehicle, videonumber, weather + + # # --- _bwrel_ --- + # # damage__BWREL_fk_examination, re_maintenance_event_wastewater_structure__BWREL_fk_maintenance_event + + # # --- _rel_ --- + # # fk_dataowner__REL, fk_operating_company__REL, fk_provider__REL, fk_reach_point__REL, kind__REL, recording_type__REL, status__REL, weather__REL + # logger.warning( + # f"QGEP field maintenance_event.active_zone has no equivalent in the interlis model. It will be ignored." + # ) + + # untersuchung = ABWASSER.untersuchung( + # # FIELDS TO MAP TO ABWASSER.untersuchung + # # --- baseclass --- + # # --- sia405_baseclass --- + # **base_common(row, "untersuchung"), + # # --- erhaltungsereignis --- + # # abwasserbauwerkref=row.REPLACE_ME, # TODO : convert this to M2N relation through re_maintenance_event_wastewater_structure + # art=get_vl(row.kind__REL), + # astatus=get_vl(row.status__REL), + # ausfuehrende_firmaref=get_tid(row.fk_operating_company__REL), + # ausfuehrender=row.operator, + # bemerkung=truncate(emptystr_to_null(row.remark), 80), + # bezeichnung=null_to_emptystr(row.identifier), + # datengrundlage=row.base_data, + # dauer=row.duration, + # detaildaten=row.data_details, + # ergebnis=row.result, + # grund=row.reason, + # kosten=row.cost, + # zeitpunkt=row.time_point, + # # --- untersuchung --- + # bispunktbezeichnung=row.to_point_identifier, + # erfassungsart=get_vl(row.recording_type__REL), + # fahrzeug=row.vehicle, + # geraet=row.equipment, + # haltungspunktref=get_tid(row.fk_reach_point__REL), + # inspizierte_laenge=row.inspected_length, + # videonummer=row.videonumber, + # vonpunktbezeichnung=row.from_point_identifier, + # witterung=get_vl(row.weather__REL), + # ) + # abwasser_session.add(untersuchung) + # create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # abwasser_session.flush() + + # logger.info("Exporting QGEP.damage_manhole -> ABWASSER.normschachtschaden, ABWASSER.metaattribute") + # query = qgep_session.query(QGEP.damage_manhole) + # if filtered: + # query = ( + # query.join(QGEP.examination) + # .join(QGEP.re_maintenance_event_wastewater_structure) + # .join(QGEP.wastewater_structure) + # .join(QGEP.wastewater_networkelement) + # .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + # for row in query: + + # # AVAILABLE FIELDS IN QGEP.damage_manhole + + # # --- damage --- + + # # --- damage_manhole --- + # # manhole_damage_code, manhole_shaft_area, obj_id + + # # --- _bwrel_ --- + # # damage_channel_channel_damage_code__BWREL_obj_id + + # # --- _rel_ --- + # # connection__REL, fk_dataowner__REL, fk_examination__REL, fk_provider__REL, manhole_damage_code__REL, manhole_shaft_area__REL, single_damage_class__REL + + # normschachtschaden = ABWASSER.normschachtschaden( + # # FIELDS TO MAP TO ABWASSER.normschachtschaden + # # --- baseclass --- + # # --- sia405_baseclass --- + # **base_common(row, "normschachtschaden"), + # # --- schaden --- + # anmerkung=row.comments, + # ansichtsparameter=row.view_parameters, + # einzelschadenklasse=get_vl(row.single_damage_class__REL), + # streckenschaden=row.damage_reach, + # untersuchungref=get_tid(row.fk_examination__REL), + # verbindung=get_vl(row.connection__REL), + # videozaehlerstand=row.video_counter, + # # --- normschachtschaden --- + # distanz=row.distance, + # quantifizierung1=row.quantification1, + # quantifizierung2=row.quantification2, + # schachtbereich=get_vl(row.manhole_shaft_area__REL), + # schachtschadencode=get_vl(row.manhole_damage_code__REL), + # schadenlageanfang=row.damage_begin, + # schadenlageende=row.damage_end, + # ) + # abwasser_session.add(normschachtschaden) + # create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # abwasser_session.flush() + + # logger.info("Exporting QGEP.damage_channel -> ABWASSER.kanalschaden, ABWASSER.metaattribute") + # query = qgep_session.query(QGEP.damage_channel) + # if filtered: + # query = ( + # query.join(QGEP.examination) + # .join(QGEP.re_maintenance_event_wastewater_structure) + # .join(QGEP.wastewater_structure) + # .join(QGEP.wastewater_networkelement) + # .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + # for row in query: + + # # AVAILABLE FIELDS IN QGEP.damage_channel + + # # --- damage --- + # # comments, connection, damage_begin, damage_end, damage_reach, distance, fk_dataowner, fk_examination, fk_provider, last_modification, quantification1, quantification2, single_damage_class, video_counter, view_parameters + + # # --- damage_channel --- + # # , obj_id + + # # --- _bwrel_ --- + # # damage_channel_channel_damage_code__BWREL_obj_id + + # # --- _rel_ --- + # # channel_damage_code__REL, connection__REL, fk_dataowner__REL, fk_examination__REL, fk_provider__REL, single_damage_class__REL + + # kanalschaden = ABWASSER.kanalschaden( + # # FIELDS TO MAP TO ABWASSER.kanalschaden + # # --- baseclass --- + # # --- sia405_baseclass --- + # **base_common(row, "kanalschaden"), + # # --- schaden --- + # anmerkung=row.comments, + # ansichtsparameter=row.view_parameters, + # einzelschadenklasse=get_vl(row.single_damage_class__REL), + # streckenschaden=row.damage_reach, + # untersuchungref=get_tid(row.fk_examination__REL), + # verbindung=get_vl(row.connection__REL), + # videozaehlerstand=row.video_counter, + # # --- kanalschaden --- + # distanz=row.distance, + # kanalschadencode=get_vl(row.channel_damage_code__REL), + # quantifizierung1=row.quantification1, + # quantifizierung2=row.quantification2, + # schadenlageanfang=row.damage_begin, + # schadenlageende=row.damage_end, + # ) + # abwasser_session.add(kanalschaden) + # create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # abwasser_session.flush() + + # logger.info("Exporting QGEP.data_media -> ABWASSER.datentraeger, ABWASSER.metaattribute") + # query = qgep_session.query(QGEP.data_media) + # for row in query: + + # # AVAILABLE FIELDS IN QGEP.data_media + + # # --- data_media --- + # # fk_dataowner, fk_provider, identifier, kind, last_modification, location, obj_id, path, remark + + # # --- _rel_ --- + # # fk_dataowner__REL, fk_provider__REL, kind__REL + + # datentraeger = ABWASSER.datentraeger( + # # FIELDS TO MAP TO ABWASSER.datentraeger + # # --- baseclass --- + # # --- sia405_baseclass --- + # **base_common(row, "datentraeger"), + # # --- datentraeger --- + # art=get_vl(row.kind__REL), + # bemerkung=truncate(emptystr_to_null(row.remark), 80), + # bezeichnung=null_to_emptystr(row.identifier), + # pfad=row.path, + # standort=row.location, + # ) + # abwasser_session.add(datentraeger) + # create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # abwasser_session.flush() + + # logger.info("Exporting QGEP.file -> ABWASSER.datei, ABWASSER.metaattribute") + # query = qgep_session.query(QGEP.file) + # if filtered: + # query = ( + # query.outerjoin(QGEP.damage, QGEP.file.object == QGEP.damage.obj_id) + # .join( + # QGEP.examination, + # or_(QGEP.file.object == QGEP.damage.obj_id, QGEP.file.object == QGEP.examination.obj_id), + # ) + # .join(QGEP.re_maintenance_event_wastewater_structure) + # .join(QGEP.wastewater_structure) + # .join(QGEP.wastewater_networkelement) + # .filter(QGEP.wastewater_networkelement.obj_id.in_(subset_ids)) + # ) + # for row in query: + + # # AVAILABLE FIELDS IN QGEP.file + + # # --- file --- + # # class, fk_data_media, fk_dataowner, fk_provider, identifier, kind, last_modification, obj_id, object, path_relative, remark + + # # --- _rel_ --- + # # class__REL, fk_dataowner__REL, fk_provider__REL, kind__REL + + # datei = ABWASSER.datei( + # # FIELDS TO MAP TO ABWASSER.datei + # # --- baseclass --- + # # --- sia405_baseclass --- + # **base_common(row, "datei"), + # # --- datei --- + # art=get_vl(row.kind__REL) or "andere", + # bemerkung=truncate(emptystr_to_null(row.remark), 80), + # bezeichnung=null_to_emptystr(row.identifier), + # datentraegerref=get_tid(row.fk_data_media__REL), + # klasse=get_vl(row.class__REL), + # objekt=null_to_emptystr(row.object), + # relativpfad=row.path_relative, + # ) + # abwasser_session.add(datei) + # create_metaattributes(row) + # print(".", end="") + # logger.info("done") + # abwasser_session.flush() + + # Labels + # Note: these are extracted from the optional labels file (not exported from the QGEP database) + if labels_file: + logger.info(f"Exporting label positions from {labels_file}") + + # Get t_id by obj_name to create the reference on the labels below + tid_for_obj_id = { + "haltung": {}, + "abwasserbauwerk": {}, + } + for row in abwasser_session.query(ABWASSER.haltung): + tid_for_obj_id["haltung"][row.obj_id] = row.t_id + for row in abwasser_session.query(ABWASSER.abwasserbauwerk): + tid_for_obj_id["abwasserbauwerk"][row.obj_id] = row.t_id + + with open(labels_file) as labels_file_handle: + labels = json.load(labels_file_handle) + + geojson_crs_def = labels["crs"] + + for label in labels["features"]: + layer_name = label["properties"]["Layer"] + obj_id = label["properties"]["qgep_obj_id"] + + print(f"label[properties]: {label['properties']}") + + if not label["properties"]["LabelText"]: + logger.warning( + f"Label of object '{obj_id}' from layer '{layer_name}' is empty and will not be exported" + ) + continue + + if layer_name == "vw_qgep_reach": + if obj_id not in tid_for_obj_id["haltung"]: + logger.warning( + f"Label for haltung `{obj_id}` exists, but that object is not part of the export" + ) + continue + ili_label = ABWASSER.haltung_text( + **textpos_common(label, "haltung_text", geojson_crs_def), + haltungref=tid_for_obj_id["haltung"][obj_id], + ) + + elif layer_name == "vw_qgep_wastewater_structure": + if obj_id not in tid_for_obj_id["abwasserbauwerk"]: + logger.warning( + f"Label for abwasserbauwerk `{obj_id}` exists, but that object is not part of the export" + ) + continue + ili_label = ABWASSER.abwasserbauwerk_text( + **textpos_common(label, "abwasserbauwerk_text", geojson_crs_def), + abwasserbauwerkref=tid_for_obj_id["abwasserbauwerk"][obj_id], + ) + + else: + logger.warning( + f"Unknown layer for label `{layer_name}`. Label will be ignored", + ) + continue + + abwasser_session.add(ili_label) + print(".", end="") + logger.info("done") + abwasser_session.flush() + + abwasser_session.commit() + + qgep_session.close() + abwasser_session.close() diff --git a/qgepqwat2ili/utils/ili2db.py b/qgepqwat2ili/utils/ili2db.py index b4e0f7f4..7f244dbb 100644 --- a/qgepqwat2ili/utils/ili2db.py +++ b/qgepqwat2ili/utils/ili2db.py @@ -448,7 +448,7 @@ def check_fk_provider_null(): return check_fk_provider_null -def skip_wwtp_structure_ids(): +def skip_wwtp_structure_ids_old(): """ Get list of id's of class wastewater_structure without wwtp_structure (ARABauwerk) """ @@ -464,10 +464,6 @@ def skip_wwtp_structure_ids(): cursor.execute( "SELECT * FROM qgep_od.wastewater_structure WHERE obj_id NOT IN (SELECT obj_id FROM qgep_od.wwtp_structure);" ) - # remove - only for testing - # cursor.execute( - # f"SELECT * FROM qgep_od.organisation WHERE obj_id NOT IN (SELECT obj_id FROM qgep_od.private);" - # ) # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ # wwtp_structure_count = int(cursor.fetchone()[0]) @@ -489,6 +485,384 @@ def skip_wwtp_structure_ids(): return not_wwtp_structure_ids +# 10.12.2024 +def get_selection_text_for_in_statement(selection_list): + """ + convert selection_list to selection_text to fit SQL IN statement + """ + selection_text = "" + + for list_item in selection_list: + selection_text += "'" + selection_text += list_item + selection_text += "'," + + # remove last komma to make it a correct IN statement + selection_text = selection_text[:-1] + + logger.debug(f"selection_text = {selection_text} ...") + return selection_text + + +# 12.11.2024 to clean up - get_ws_wn_ids kann das auch +def get_cl_re_ids(classname): + """ + Get list of id's of reaches of the channels provided + """ + + # define classes that this is allowed to use - adapt for TWW to include model changes + if classname == "channel": + logger.info(f"get list of id's of wastewater_nodes of {classname} ...") + + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + cl_re_ids = [] + + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + "SELECT wn.obj_id FROM qgep_od.channel LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = channel.obj_id WHERE wn.obj_id is not NULL;" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # cl_re_ids_count = int(cursor.fetchone()[0]) + # if cl_re_ids_count == 0: + if cursor.fetchone() is None: + cl_re_ids = None + else: + records = cursor.fetchall() + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + cl_re_ids.append(strrow) + logger.debug(f" building up '{cl_re_ids}' ...") + + return cl_re_ids + else: + logger.warning(f"Do not use this function with {classname} !") + return None + + +# 10.12.2024 +def get_connected_we_from_re(subset_reaches): + """ + Get connected wastewater_networkelements (wastewater_nodes and reaches) from subset of reaches + """ + + logger.info( + f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + connected_wn_from_re_ids = [] + + subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + + # select all connected from wastewater_nodes from provided subset of reaches + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_wn_from_re_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_from LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_wn_from_re_ids.append(strrow) + logger.debug(f" building up '{connected_wn_from_re_ids}' ...") + + return connected_wn_from_re_ids + + +# 10.12.2024 +def get_connected_overflow_to_wn_ids(selected_ids): + """ + Get all connected wastewater_nodes from overflows.fk_overflow_to + """ + + logger.info( + f"Get all connected wastewater_nodes from overflows.fk_overflow_to {selected_ids} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + connected_overflow_to_wn_ids = [] + + subset_text = get_selection_text_for_in_statement(selected_ids) + + # select all connected to wastewater_nodes from provided subset of reaches + cursor.execute( + f"SELECT ov.fk_overflow_to FROM tww_od.wastewater_node wn LEFT JOIN tww_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_overflow_to_wn_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT ov.fk_overflow_to FROM tww_od.wastewater_node wn LEFT JOIN tww_od.overflow ov ON wn.obj_id = ov.fk_wastewater_node WHERE wn.obj_id IN ({subset_text});" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_overflow_to_wn_ids.append(strrow) + logger.debug(f" building up '{connected_overflow_to_wn_ids}' ...") + + return connected_overflow_to_wn_ids + + +# 10.12.2024 +def get_connected_we_to_re(subset_reaches): + """ + Get connected wastewater_networkelements (wastewater_nodes and reaches) to subset of reaches + """ + + logger.info( + f"get list of id's of connected wastewater_nodes of provides subset of reaches {subset_reaches} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + connected_wn_to_re_ids = [] + + subset_reaches_text = get_selection_text_for_in_statement(subset_reaches) + + # select all connected to wastewater_nodes from provided subset of reaches + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + connected_wn_to_re_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wef.obj_id as wef_obj_id FROM qgep_od.reach re LEFT JOIN qgep_od.reach_point rpf ON rpf.obj_id = re.fk_reach_point_to LEFT JOIN qgep_od.wastewater_networkelement wef ON wef.obj_id = rpf.fk_wastewater_networkelement WHERE re.obj_id IN ({subset_reaches_text}) AND NOT wef.obj_id isNull;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + connected_wn_to_re_ids.append(strrow) + logger.debug(f" building up '{connected_wn_to_re_ids}' ...") + + return connected_wn_to_re_ids + + +def get_ws_wn_ids(classname): + """ + Get list of id's of wastewater_nodes of the wastewater_structure (sub)class provided, eg. wwtp_structure (ARABauwerk, does also work for channel (give reaches then) + """ + + logger.info(f"get list of id's of wastewater_nodes of {classname} ...") + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + ws_wn_ids = [] + + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + ws_wn_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute( + f"SELECT wn.obj_id FROM qgep_od.{classname} LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = {classname}.obj_id WHERE wn.obj_id is not NULL;" + ) + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + ws_wn_ids.append(strrow) + # logger.debug(f" building up '{ws_wn_ids}' ...") + + return ws_wn_ids + + +def get_ws_selected_ww_networkelements(selected_wwn): + """ + Get list of id's of wastewater_structure from selected wastewater_network_elements + """ + + logger.debug( + f"get list of id's of wastewater_structure of selected wastewater_network_elements {selected_wwn} ..." + ) + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + selection_text = "" + + for list_item in selected_wwn: + selection_text += "'" + selection_text += list_item + selection_text += "'," + + # remove last komma to make it a correct IN statement + selection_text = selection_text[:-1] + + logger.debug(f"selection_text = {selection_text} ...") + + ws_ids = [] + + # select all obj_id of the wastewater_nodes of wwtp_structure + cursor.execute( + f"SELECT ws.obj_id FROM qgep_od.wastewater_structure ws LEFT JOIN qgep_od.wastewater_networkelement wn ON wn.fk_wastewater_structure = ws.obj_id WHERE wn.obj_id IN ({selection_text});" + ) + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + ws_ids = None + else: + records = cursor.fetchall() + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + ws_ids.append(strrow) + # logger.debug(f" building up '{ws_wn_ids}' ...") + + return ws_ids + + +# 10.1.2024 +def filter_reaches(selected_ids): + """ + Filter out reaches from selected_ids + """ + + logger.info(f"Filter out reaches from selected_ids {selected_ids} ...") + + if selected_ids is None: + subset_reaches_ids = None + else: + connection = psycopg2.connect(get_pgconf_as_psycopg2_dsn()) + connection.set_session(autocommit=True) + cursor = connection.cursor() + + subset_reaches_ids = [] + + get_selection_text_for_in_statement(selected_ids) + + # select all reaches + cursor.execute(f"SELECT obj_id FROM qgep_od.reach;") + + # cursor.fetchall() - see https://pynative.com/python-cursor-fetchall-fetchmany-fetchone-to-read-rows-from-table/ + # ws_wn_ids_count = int(cursor.fetchone()[0]) + # if ws_wn_ids_count == 0: + if cursor.fetchone() is None: + all_reaches_ids = None + else: + # added cursor.execute again to see if with this all records will be available + # 15.11.2024 added - see https://stackoverflow.com/questions/58101874/cursor-fetchall-or-other-method-fetchone-is-not-working + cursor.execute(f"SELECT obj_id FROM qgep_od.reach;") + records = cursor.fetchall() + + # 15.11.2024 - does not get all records, but only n-1 + for row in records: + logger.debug(f" row[0] = {row[0]}") + # https://www.pythontutorial.net/python-string-methods/python-string-concatenation/ + strrow = str(row[0]) + if strrow is not None: + all_reaches_ids.append(strrow) + logger.debug(f" building up '{all_reaches_ids}' ...") + + for list_item in selected_ids: + if list_item in all_reaches_ids: + subset_reaches_ids.append(list_item) + logger.debug( + f"'filter_reaches: {list_item}' is a reach id - added to subset_reaches_ids" + ) + else: + logger.debug(f"'filter_reaches: {list_item}' is not a reach id") + + return subset_reaches_ids + + +def remove_from_selection(selected_ids, remove_ids): + """ + Remove ids from selected_ids if they are in selected_ids + """ + + for list_item in remove_ids: + # selected_ids = selected_ids.remove(list_item) + try: + selected_ids.remove(list_item) + except Exception: + logger.debug( + f" remove_from_selection: '{list_item}' not in selected_ids - could not be removed!" + ) + + return selected_ids + + +def add_to_selection(selected_ids, add_ids): + """ + Append ids to selected_ids + """ + + if selected_ids is None: + selected_ids = [] + + for list_item in add_ids: + # selected_ids = selected_ids.append(list_item) + selected_ids.append(list_item) + + return selected_ids + + def create_ili_schema(schema, model, log_path, recreate_schema=False, create_basket_col=False): """ Create schema for INTERLIS import