db: separate tables

* Creates for every resource a separate metadata and id table in the database. With this change export and import of one resource should be easier and the access to seperate smaler tables are hopefully faster for big datasets. Co-Authored-by: Peter Weber <[email protected]>
rero · May 5, 2020 · f79ebc8 · f79ebc8
1 parent 58e7dff
commit f79ebc8
Show file tree

Hide file tree

Showing 38 changed files with 169 additions and 25 deletions.
diff --git a/rero_ils/modules/acq_accounts/api.py b/rero_ils/modules/acq_accounts/api.py
@@ -21,7 +21,7 @@
 
 from flask import current_app
 
-from .models import AcqAccountIdentifier
+from .models import AcqAccountIdentifier, AcqAccountMetadata
 from ..acq_order_lines.api import AcqOrderLinesSearch
 from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch
 from ..fetchers import id_fetcher
@@ -57,6 +57,7 @@ class AcqAccount(IlsRecord):
     minter = acq_account_id_minter
     fetcher = acq_account_id_fetcher
     provider = AcqAccountProvider
+    model_cls = AcqAccountMetadata
 
     @classmethod
     def create(cls, data, id_=None, delete_pid=False,

diff --git a/rero_ils/modules/acq_accounts/models.py b/rero_ils/modules/acq_accounts/models.py
@@ -21,6 +21,7 @@
 
 from invenio_db import db
 from invenio_pidstore.models import RecordIdentifier
+from invenio_records.models import RecordMetadataBase
 
 
 class AcqAccountIdentifier(RecordIdentifier):
@@ -33,3 +34,9 @@ class AcqAccountIdentifier(RecordIdentifier):
         db.BigInteger().with_variant(db.Integer, 'sqlite'),
         primary_key=True, autoincrement=True,
     )
+
+
+class AcqAccountMetadata(db.Model, RecordMetadataBase):
+    """AcqAccount record metadata."""
+
+    __tablename__ = 'acq_account_metadata'
diff --git a/rero_ils/modules/acq_invoices/api.py b/rero_ils/modules/acq_invoices/api.py
@@ -21,7 +21,7 @@
 
 from flask import current_app
 
-from .models import AcquisitionInvoiceIdentifier
+from .models import AcquisitionInvoiceIdentifier, AcquisitionInvoiceMetadata
 from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch
 from ..fetchers import id_fetcher
 from ..libraries.api import Library
@@ -57,6 +57,7 @@ class AcquisitionInvoice(IlsRecord):
     minter = acq_invoice_id_minter
     fetcher = acq_invoice_id_fetcher
     provider = AcquisitionInvoiceProvider
+    model_cls = AcquisitionInvoiceMetadata
 
     @classmethod
     def create(cls, data, id_=None, delete_pid=False,

diff --git a/rero_ils/modules/acq_invoices/models.py b/rero_ils/modules/acq_invoices/models.py
@@ -21,6 +21,7 @@
 
 from invenio_db import db
 from invenio_pidstore.models import RecordIdentifier
+from invenio_records.models import RecordMetadataBase
 
 
 class AcquisitionInvoiceIdentifier(RecordIdentifier):
@@ -33,3 +34,9 @@ class AcquisitionInvoiceIdentifier(RecordIdentifier):
         db.BigInteger().with_variant(db.Integer, 'sqlite'),
         primary_key=True, autoincrement=True,
     )
+
+
+class AcquisitionInvoiceMetadata(db.Model, RecordMetadataBase):
+    """AcquisitionInvoice record metadata."""
+
+    __tablename__ = 'acq_invoice_metadata'
diff --git a/rero_ils/modules/acq_order_lines/api.py b/rero_ils/modules/acq_order_lines/api.py
@@ -21,7 +21,7 @@
 
 from flask import current_app
 
-from .models import AcqOrderLineIdentifier
+from .models import AcqOrderLineIdentifier, AcqOrderLineMetadata
 from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch
 from ..fetchers import id_fetcher
 from ..minters import id_minter
@@ -55,6 +55,7 @@ class AcqOrderLine(IlsRecord):
     minter = acq_order_line_id_minter
     fetcher = acq_order_line_id_fetcher
     provider = AcqOrderLineProvider
+    model_cls = AcqOrderLineMetadata
 
     @classmethod
     def create(cls, data, id_=None, delete_pid=False,

diff --git a/rero_ils/modules/acq_order_lines/models.py b/rero_ils/modules/acq_order_lines/models.py
@@ -21,6 +21,7 @@
 
 from invenio_db import db
 from invenio_pidstore.models import RecordIdentifier
+from invenio_records.models import RecordMetadataBase
 
 
 class AcqOrderLineIdentifier(RecordIdentifier):
@@ -33,3 +34,9 @@ class AcqOrderLineIdentifier(RecordIdentifier):
         db.BigInteger().with_variant(db.Integer, 'sqlite'),
         primary_key=True, autoincrement=True,
     )
+
+
+class AcqOrderLineMetadata(db.Model, RecordMetadataBase):
+    """AcqOrderLine record metadata."""
+
+    __tablename__ = 'acq_order_line_metadata'
diff --git a/rero_ils/modules/acq_orders/api.py b/rero_ils/modules/acq_orders/api.py
@@ -21,7 +21,7 @@
 
 from flask import current_app
 
-from .models import AcqOrderIdentifier
+from .models import AcqOrderIdentifier, AcqOrderMetadata
 from ..acq_order_lines.api import AcqOrderLinesSearch
 from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch
 from ..fetchers import id_fetcher
@@ -57,6 +57,7 @@ class AcqOrder(IlsRecord):
     minter = acq_order_id_minter
     fetcher = acq_order_id_fetcher
     provider = AcqOrderProvider
+    model_cls = AcqOrderMetadata
 
     @classmethod
     def create(cls, data, id_=None, delete_pid=False,

diff --git a/rero_ils/modules/acq_orders/models.py b/rero_ils/modules/acq_orders/models.py
@@ -21,6 +21,7 @@
 
 from invenio_db import db
 from invenio_pidstore.models import RecordIdentifier
+from invenio_records.models import RecordMetadataBase
 
 
 class AcqOrderIdentifier(RecordIdentifier):
@@ -33,3 +34,9 @@ class AcqOrderIdentifier(RecordIdentifier):
         db.BigInteger().with_variant(db.Integer, 'sqlite'),
         primary_key=True, autoincrement=True,
     )
+
+
+class AcqOrderMetadata(db.Model, RecordMetadataBase):
+    """AcqOrder record metadata."""
+
+    __tablename__ = 'acq_order_metadata'
diff --git a/rero_ils/modules/budgets/api.py b/rero_ils/modules/budgets/api.py
@@ -19,7 +19,7 @@
 
 from functools import partial
 
-from .models import BudgetIdentifier
+from .models import BudgetIdentifier, BudgetMetadata
 from ..acq_accounts.api import AcqAccountsSearch
 from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch
 from ..fetchers import id_fetcher
@@ -55,6 +55,7 @@ class Budget(IlsRecord):
     minter = budget_id_minter
     fetcher = budget_id_fetcher
     provider = BudgetProvider
+    model_cls = BudgetMetadata
 
     def get_number_of_acq_accounts(self):
         """Get number of acq accounts."""

diff --git a/rero_ils/modules/budgets/models.py b/rero_ils/modules/budgets/models.py
@@ -21,6 +21,7 @@
 
 from invenio_db import db
 from invenio_pidstore.models import RecordIdentifier
+from invenio_records.models import RecordMetadataBase
 
 
 class BudgetIdentifier(RecordIdentifier):
@@ -33,3 +34,9 @@ class BudgetIdentifier(RecordIdentifier):
         db.BigInteger().with_variant(db.Integer, 'sqlite'),
         primary_key=True, autoincrement=True,
     )
+
+
+class BudgetMetadata(db.Model, RecordMetadataBase):
+    """Budget record metadata."""
+
+    __tablename__ = 'budget_metadata'
diff --git a/rero_ils/modules/circ_policies/api.py b/rero_ils/modules/circ_policies/api.py
@@ -23,7 +23,7 @@
 
 from elasticsearch_dsl import Q
 
-from .models import CircPolicyIdentifier
+from .models import CircPolicyIdentifier, CircPolicyMetadata
 from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch
 from ..fetchers import id_fetcher
 from ..libraries.api import Library
@@ -58,6 +58,7 @@ class CircPolicy(IlsRecord):
     minter = circ_policy_id_minter
     fetcher = circ_policy_id_fetcher
     provider = CircPolicyProvider
+    model_cls = CircPolicyMetadata
 
     def extended_validation(self, **kwargs):
         """Validate record against schema.

diff --git a/rero_ils/modules/circ_policies/models.py b/rero_ils/modules/circ_policies/models.py
@@ -21,6 +21,7 @@
 
 from invenio_db import db
 from invenio_pidstore.models import RecordIdentifier
+from invenio_records.models import RecordMetadataBase
 
 
 class CircPolicyIdentifier(RecordIdentifier):
@@ -33,3 +34,9 @@ class CircPolicyIdentifier(RecordIdentifier):
         db.BigInteger().with_variant(db.Integer, 'sqlite'),
         primary_key=True, autoincrement=True,
     )
+
+
+class CircPolicyMetadata(db.Model, RecordMetadataBase):
+    """CircPolicy record metadata."""
+
+    __tablename__ = 'circ_policy_metadata'
diff --git a/rero_ils/modules/documents/api.py b/rero_ils/modules/documents/api.py
@@ -24,7 +24,7 @@
 from invenio_circulation.search.api import search_by_pid
 from invenio_search.api import RecordsSearch
 
-from .models import DocumentIdentifier
+from .models import DocumentIdentifier, DocumentMetadata
 from .utils import edition_format_text, publication_statement_text, \
     series_format_text, title_format_text_head
 from ..acq_order_lines.api import AcqOrderLinesSearch
@@ -62,6 +62,7 @@ class Document(IlsRecord):
     minter = document_id_minter
     fetcher = document_id_fetcher
     provider = DocumentProvider
+    model_cls = DocumentMetadata
 
     def is_available(self, view_code):
         """Get availability for document."""

diff --git a/rero_ils/modules/documents/models.py b/rero_ils/modules/documents/models.py
@@ -21,6 +21,7 @@
 
 from invenio_db import db
 from invenio_pidstore.models import RecordIdentifier
+from invenio_records.models import RecordMetadataBase
 
 
 class DocumentIdentifier(RecordIdentifier):
@@ -33,3 +34,9 @@ class DocumentIdentifier(RecordIdentifier):
         db.BigInteger().with_variant(db.Integer, 'sqlite'),
         primary_key=True, autoincrement=True,
     )
+
+
+class DocumentMetadata(db.Model, RecordMetadataBase):
+    """Document record metadata."""
+
+    __tablename__ = 'document_metadata'
diff --git a/rero_ils/modules/holdings/api.py b/rero_ils/modules/holdings/api.py
@@ -30,7 +30,7 @@
 from invenio_search.api import RecordsSearch
 from jinja2 import Template
 
-from .models import HoldingIdentifier
+from .models import HoldingIdentifier, HoldingMetadata
 from ..api import IlsRecord, IlsRecordsIndexer
 from ..documents.api import Document
 from ..errors import MissingRequiredParameterError
@@ -74,7 +74,7 @@ class Holding(IlsRecord):
     minter = holding_id_minter
     fetcher = holding_id_fetcher
     provider = HoldingProvider
-    # model_cls = HoldingMetadata
+    model_cls = HoldingMetadata
 
     def delete_from_index(self):
         """Delete record from index."""

diff --git a/rero_ils/modules/holdings/models.py b/rero_ils/modules/holdings/models.py
@@ -21,8 +21,7 @@
 
 from invenio_db import db
 from invenio_pidstore.models import RecordIdentifier
-
-# from invenio_records.models import RecordMetadataBase
+from invenio_records.models import RecordMetadataBase
 
 
 class HoldingIdentifier(RecordIdentifier):
@@ -37,7 +36,7 @@ class HoldingIdentifier(RecordIdentifier):
     )
 
 
-# class HoldingMetadata(db.Model, RecordMetadataBase):
-#     """Holding record metadata."""
+class HoldingMetadata(db.Model, RecordMetadataBase):
+    """Holding record metadata."""
 
-#     __tablename__ = 'holdings_metadata'
+    __tablename__ = 'holding_metadata'
diff --git a/rero_ils/modules/item_types/api.py b/rero_ils/modules/item_types/api.py
@@ -24,7 +24,7 @@
 from elasticsearch_dsl import Q
 from flask_babelex import gettext as _
 
-from .models import ItemTypeIdentifier
+from .models import ItemTypeIdentifier, ItemTypeMetadata
 from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch
 from ..circ_policies.api import CircPoliciesSearch
 from ..fetchers import id_fetcher
@@ -59,6 +59,7 @@ class ItemType(IlsRecord):
     minter = item_type_id_minter
     fetcher = item_type_id_fetcher
     provider = ItemTypeProvider
+    model_cls = ItemTypeMetadata
 
     def extended_validation(self, **kwargs):
         """Validate record against schema.

diff --git a/rero_ils/modules/item_types/models.py b/rero_ils/modules/item_types/models.py
@@ -21,6 +21,7 @@
 
 from invenio_db import db
 from invenio_pidstore.models import RecordIdentifier
+from invenio_records.models import RecordMetadataBase
 
 
 class ItemTypeIdentifier(RecordIdentifier):
@@ -33,3 +34,9 @@ class ItemTypeIdentifier(RecordIdentifier):
         db.BigInteger().with_variant(db.Integer, 'sqlite'),
         primary_key=True, autoincrement=True,
     )
+
+
+class ItemTypeMetadata(db.Model, RecordMetadataBase):
+    """ItemType record metadata."""
+
+    __tablename__ = 'item_type_metadata'
diff --git a/rero_ils/modules/items/api.py b/rero_ils/modules/items/api.py
@@ -31,7 +31,7 @@
 from invenio_i18n.ext import current_i18n
 from invenio_search import current_search
 
-from .models import ItemIdentifier, ItemStatus
+from .models import ItemIdentifier, ItemMetadata, ItemStatus
 from ..api import IlsRecord, IlsRecordError, IlsRecordsIndexer, \
     IlsRecordsSearch
 from ..circ_policies.api import CircPolicy
@@ -156,6 +156,7 @@ class Item(IlsRecord):
     minter = item_id_minter
     fetcher = item_id_fetcher
     provider = ItemProvider
+    model_cls = ItemMetadata
 
     statuses = {
         'ITEM_ON_LOAN': 'on_loan',

diff --git a/rero_ils/modules/items/models.py b/rero_ils/modules/items/models.py
@@ -21,6 +21,7 @@
 
 from invenio_db import db
 from invenio_pidstore.models import RecordIdentifier
+from invenio_records.models import RecordMetadataBase
 
 
 class ItemIdentifier(RecordIdentifier):
@@ -36,6 +37,12 @@ class ItemIdentifier(RecordIdentifier):
     )
 
 
+class ItemMetadata(db.Model, RecordMetadataBase):
+    """Item record metadata."""
+
+    __tablename__ = 'item_metadata'
+
+
 class ItemStatus(object):
     """Class holding all availabe circulation item statuses."""
 

diff --git a/rero_ils/modules/libraries/api.py b/rero_ils/modules/libraries/api.py
@@ -24,7 +24,7 @@
 from dateutil import parser
 from dateutil.rrule import FREQNAMES, rrule
 
-from .models import LibraryIdentifier
+from .models import LibraryIdentifier, LibraryMetadata
 from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch
 from ..fetchers import id_fetcher
 from ..locations.api import LocationsSearch
@@ -65,6 +65,7 @@ class Library(IlsRecord):
     minter = library_id_minter
     fetcher = library_id_fetcher
     provider = LibraryProvider
+    model_cls = LibraryMetadata
 
     @property
     def online_location(self):

diff --git a/rero_ils/modules/libraries/models.py b/rero_ils/modules/libraries/models.py
@@ -21,6 +21,7 @@
 
 from invenio_db import db
 from invenio_pidstore.models import RecordIdentifier
+from invenio_records.models import RecordMetadataBase
 
 
 class LibraryIdentifier(RecordIdentifier):
@@ -33,3 +34,9 @@ class LibraryIdentifier(RecordIdentifier):
         db.BigInteger().with_variant(db.Integer, 'sqlite'),
         primary_key=True, autoincrement=True,
     )
+
+
+class LibraryMetadata(db.Model, RecordMetadataBase):
+    """Library record metadata."""
+
+    __tablename__ = 'library_metadata'