Improve Performance by changing availability behaviour (#1540)

* change availability behaviour

* track down legal provision problem

* fix references to SQLAlchemy classes

* fix lint and tests

dev/config/pyramid_oereb.yml.mako

@@ -571,6 +571,23 @@ pyramid_oereb:
     # (e.g. "{egrid}") to parameterize the URL.
     redirect: https://geoview.bl.ch/oereb/?egrid={egrid}
 
+  # The processor of the oereb project needs access to availability data. In the standard configuration this
+  # is assumed to be read from a database. Hint: If you want to read the availability out of an existing database
+  # table to avoid imports of this data every time it gets updates, you only need to change the model bound to
+  # the source. The model must implement the same field names and information as the default model does.
+  availability:
+    # The availability must have a property source.
+    source:
+      # The source must have a class which represents the accessor to the source. In this example, it is an
+      # already implemented source which reads data from a database.
+      class: pyramid_oereb.contrib.data_sources.standard.sources.availability.DatabaseSource
+      # The necessary parameters to use this class
+      params:
+        # The connection path where the database can be found
+        db_connection: *main_db_connection
+        # The model which maps the map layering database table.
+        model: pyramid_oereb.contrib.data_sources.standard.models.main.Availability
+
   # All PLRs which are provided by this application. This is related to all application behaviour, especially
   # the extract creation process which loops over this list.
   plrs:

dev/database/load_sample_data.py

@@ -104,7 +104,7 @@ def load(self):
 
         from pyramid_oereb.contrib.data_sources.standard.models.main import Theme, Logo, \
             DocumentTypeText, RealEstate, Address, Municipality, Glossary, Disclaimer, \
-            GeneralInformation, RealEstateType, LawStatus, Document, Office, ThemeDocument
+            GeneralInformation, RealEstateType, LawStatus, Document, Office, ThemeDocument, Availability
 
         # Fill tables with sample data
         for class_, file_name in [
@@ -126,7 +126,8 @@ def load(self):
             (Document, 'ch.laws.json'),
             (Document, 'dev.laws.json'),
             (ThemeDocument, 'ch.themes_docs.json'),
-            (ThemeDocument, 'dev.themes_docs.json')
+            (ThemeDocument, 'dev.themes_docs.json'),
+            (Availability, 'dev.availabilities.json')
         ]:
             self._load_sample(class_, file_name)
 
@@ -141,7 +142,6 @@ def load(self):
             print("Import theme {}.".format(folder))
 
             for class_, file_name in [
-                (schema.Availability, 'availabilities.json'),
                 (schema.Office, 'office.json'),
                 (schema.DataIntegration, 'data_integration.json'),
                 (schema.ViewService, 'view_service.json'),

dev/sample_data/dev.availabilities.json

@@ -0,0 +1,32 @@
+[
+  {
+    "theme_code": "ch.BelasteteStandorteMilitaer",
+    "municipality_fosnr": 2771,
+    "available": false
+  },
+  {
+    "theme_code": "ch.BelasteteStandorteOeffentlicherVerkehr",
+    "municipality_fosnr": 2771,
+    "available": true
+  },
+  {
+    "theme_code": "ch.StatischeWaldgrenzen",
+    "municipality_fosnr": 2771,
+    "available": true
+  },
+  {
+    "theme_code": "ch.Grundwasserschutzzonen",
+    "municipality_fosnr": 2771,
+    "available": true
+  },
+  {
+    "theme_code": "ch.Nutzungsplanung",
+    "municipality_fosnr": 2771,
+    "available": true
+  },
+  {
+    "theme_code": "ch.BaulinienNationalstrassen",
+    "municipality_fosnr": 2771,
+    "available": false
+  }
+]

pyramid_oereb/contrib/data_sources/create_tables.py

@@ -12,7 +12,7 @@
 log = logging.getLogger(__name__)
 
 
-def create_theme_tables_(theme_config, tables_only=False, sql_file=None, if_not_exists=False):
+def create_theme_tables_(theme_config, source_class, tables_only=False, sql_file=None, if_not_exists=False):
     """
     Create the tables for a specific theme.
 
@@ -21,7 +21,7 @@ def create_theme_tables_(theme_config, tables_only=False, sql_file=None, if_not_
         tables_only (bool): True to skip creation of schema. Default is False.
         sql_file (file): The file to generate. Default is None (in the database).
     """
-    if theme_config.get('standard') == True:  # noqa: E712
+    if theme_config['source']['class'] == source_class:
         config_parser = StandardThemeConfigParser(**theme_config)
         models = config_parser.get_models()
         theme_schema_name = models.schema_name
@@ -68,9 +68,13 @@ def create_tables_from_standard_configuration(
     sql_file.write(sql)
 
     for theme_config in Config.get('plrs'):
-        if theme_config.get('standard'):
-            create_theme_tables_(theme_config, tables_only=tables_only, sql_file=sql_file,
-                                 if_not_exists=if_not_exists)
+        create_theme_tables_(
+            theme_config,
+            'pyramid_oereb.contrib.data_sources.standard.sources.plr.DatabaseSource',
+            tables_only=tables_only,
+            sql_file=sql_file,
+            if_not_exists=if_not_exists
+        )
 
 
 def create_standard_tables():

pyramid_oereb/contrib/data_sources/interlis_2_3/models/theme.py

@@ -1,19 +1,18 @@
 from sqlalchemy import Column, ForeignKey
-from sqlalchemy import LargeBinary, Boolean, String, Integer, DateTime, Date, Text
+from sqlalchemy import LargeBinary, String, Integer, DateTime, Date, Text
 from sqlalchemy.ext.declarative import declarative_base
 from geoalchemy2.types import Geometry as GeoAlchemyGeometry
 from sqlalchemy.orm import relationship
 
 
 class Models(object):
 
-    def __init__(self, availability, data_integration, office, document, view_service,
+    def __init__(self, data_integration, office, document, view_service,
                  legend_entry, public_law_restriction, geometry,
                  public_law_restriction_document,
                  localised_blob, localised_uri, multilingual_blob, multilingual_uri,
                  base, db_connection, schema_name):
 
-        self.Availability = availability
         self.DataIntegration = data_integration
         self.Office = office
         self.Document = document
@@ -191,23 +190,6 @@ def model_factory(schema_name, pk_type, srid, db_connection):
 
     Office, Document = generic_models(Base, schema_name, pk_type)
 
-    class Availability(Base):
-        """
-        A simple bucket for achieving a switch per municipality. Here you can configure via the
-        imported data if a public law restriction is available or not. You need to fill it with
-        the data you provided in the app schemas municipality table (fosnr).
-
-        Attributes:
-            fosnr (int): The identifier of the municipality in your system (id_bfs = fosnr)
-            available (bool): The switch field to configure if this plr is available for the
-                municipality or not.  This field has direct influence on the applications
-                behaviour. See documentation for more info.
-        """
-        __table_args__ = {'schema': schema_name}
-        __tablename__ = 'verfuegbarkeit'
-        fosnr = Column('bfsnr', pk_type, primary_key=True, autoincrement=False)
-        available = Column('verfuegbar', Boolean, nullable=False, default=False)
-
     class DataIntegration(Base):
         """
         The bucket to fill in the date when this whole schema was updated. It has a relation to the
@@ -556,7 +538,7 @@ class PublicLawRestrictionDocument(Base):
         )
 
     return Models(
-        Availability, DataIntegration,
+        DataIntegration,
         Office, Document, ViewService,
         LegendEntry, PublicLawRestriction, Geometry, PublicLawRestrictionDocument,
         LocalisedBlob, LocalisedUri, MultilingualBlob, MultilingualUri,

pyramid_oereb/contrib/data_sources/interlis_2_3/sources/plr.py

@@ -10,7 +10,6 @@
 
 from pyramid_oereb import Config
 from pyramid_oereb.core import b64
-from pyramid_oereb.core.records.availability import AvailabilityRecord
 from pyramid_oereb.core.records.image import ImageRecord
 from pyramid_oereb.core.records.plr import EmptyPlrRecord
 from pyramid_oereb.core.sources import BaseDatabaseSource
@@ -104,34 +103,18 @@ def __init__(self, **kwargs):
                 as key and text as value.
         """
         config_parser = StandardThemeConfigParser(**kwargs)
-        models = config_parser.get_models()
+        self.models = config_parser.get_models()
         bds_kwargs = {
-            'model': models.Geometry,
+            'model': self.models.Geometry,
             'db_connection': kwargs.get('source').get('params').get('db_connection')
         }
 
         BaseDatabaseSource.__init__(self, **bds_kwargs)
         PlrBaseSource.__init__(self, **kwargs)
 
-        self.legend_entry_model = models.LegendEntry
-        availability_model = models.Availability
-
-        self.availabilities = []
+        self.legend_entry_model = self.models.LegendEntry
         self.datasource = []
 
-        session = self._adapter_.get_session(self._key_)
-
-        try:
-
-            availabilities_from_db = session.query(availability_model).all()
-            for availability in availabilities_from_db:
-                self.availabilities.append(
-                    AvailabilityRecord(availability.fosnr, available=availability.available)
-                )
-
-        finally:
-            session.close()
-
     def from_db_to_legend_entry_record(self, legend_entry_from_db):
         theme = Config.get_theme_by_code_sub_code(legend_entry_from_db.theme, legend_entry_from_db.sub_theme)
         legend_entry_record = self._legend_entry_record_class(
@@ -480,7 +463,7 @@ def read(self, params, real_estate, bbox):
         """
 
         # Check if the plr is marked as available
-        if self._is_available(real_estate):
+        if Config.availability_by_theme_code_municipality_fosnr(self._plr_info['code'], real_estate.fosnr):
             session = self._adapter_.get_session(self._key_)
             try:
                 if session.query(self._model_).count() == 0:
@@ -524,19 +507,3 @@ def read(self, params, real_estate, bbox):
                 Config.get_theme_by_code_sub_code(self._plr_info['code']),
                 has_data=False
             )]
-
-    def _is_available(self, real_estate):
-        """
-        Checks if the topic is available for the specified real estate.
-
-        Args:
-            real_estate (pyramid_oereb.lib.records.real_estate.RealEstateRecord): The real
-                estate in its record representation.
-
-        Returns:
-             bool: True if the topic is available, false otherwise.
-        """
-        for availability in self.availabilities:
-            if int(real_estate.fosnr) == int(availability.fosnr) and not availability.available:
-                return False
-        return True

pyramid_oereb/contrib/data_sources/oereblex/models/theme.py

@@ -5,7 +5,6 @@
 
 from pyramid_oereb.contrib.data_sources.standard.models import (
     get_office,
-    get_availability,
     get_data_integration,
     get_view_service,
     get_legend_entry,
@@ -15,11 +14,10 @@
 
 class Models(object):
 
-    def __init__(self, availability, office, data_integration, view_service,
+    def __init__(self, office, data_integration, view_service,
                  legend_entry, public_law_restriction, geometry, base,
                  db_connection, schema_name):
 
-        self.Availability = availability
         self.Office = office
         self.DataIntegration = data_integration
         self.ViewService = view_service
@@ -45,7 +43,6 @@ def model_factory(schema_name, pk_type, geometry_type, srid, db_connection):
     """
     Base = declarative_base()
 
-    Availability = get_availability(Base, schema_name, pk_type)
     Office = get_office(Base, schema_name, pk_type)
     DataIntegration = get_data_integration(Base, schema_name, pk_type, Office)
     ViewService = get_view_service(Base, schema_name, pk_type)
@@ -107,7 +104,7 @@ class PublicLawRestriction(Base):
     Geometry = get_geometry(Base, schema_name, pk_type, geometry_type, srid, PublicLawRestriction)
 
     return Models(
-        Availability, Office, DataIntegration, ViewService,
+        Office, DataIntegration, ViewService,
         LegendEntry, PublicLawRestriction, Geometry, Base,
         db_connection, schema_name
     )

pyramid_oereb/contrib/data_sources/standard/models/__init__.py

@@ -6,7 +6,7 @@
 
 
 from sqlalchemy import Column, ForeignKey
-from sqlalchemy import Boolean, String, Integer, Float, DateTime, Date
+from sqlalchemy import String, Integer, Float, DateTime, Date
 from geoalchemy2.types import Geometry as GeoAlchemyGeometry
 from sqlalchemy.orm import relationship
 from sqlalchemy_utils import JSONType
@@ -139,41 +139,6 @@ class Document(base):
     return Document
 
 
-def get_availability(base, schema_name, pk_type):
-    """
-    Factory to produce a generic availability model.
-
-    Args:
-        base (sqlalchemy.orm.decl_api.DeclarativeMeta): The SQLAlchemy base which is assigned to the models.
-        schema_name (str): The name of the database schema where this models belong to.
-        pk_type (sqlalchemy.sql.type_api.TypeEngine): The type of the primary column. E.g.
-            sqlalchemy.String or sqlalchemy.Integer or another one fitting the underlying DB
-            needs
-
-    Returns:
-        sqlalchemy.orm.decl_api.DeclarativeMeta: The generated office model.
-    """
-
-    class Availability(base):
-        """
-        A simple bucket for achieving a switch per municipality. Here you can configure via the
-        imported data if a public law restriction is available or not. You need to fill it with
-        the data you provided in the app schemas municipality table (fosnr).
-
-        Attributes:
-            fosnr (int): The identifier of the municipality in your system (id_bfs = fosnr)
-            available (bool): The switch field to configure if this plr is available for the
-                municipality or not.  This field has direct influence on the applications
-                behaviour. See documentation for more info.
-        """
-        __table_args__ = {'schema': schema_name}
-        __tablename__ = 'availability'
-        fosnr = Column(pk_type, primary_key=True, autoincrement=False)
-        available = Column(Boolean, nullable=False, default=False)
-
-    return Availability
-
-
 def get_data_integration(base, schema_name, pk_type, Office):
     """
     Factory to produce a generic data integration model.