fixtures: generate new files

* Replaces 10k files by 1k file.
* Improves code to insert pids to identifier table.

Co-Authored-by: Aly Badr [email protected]

rero_ils/dojson/cli.py

@@ -17,6 +17,7 @@
 
 """Click command-line utilities."""
 
+import json
 
 import click
 from flask import current_app

rero_ils/modules/cli.py

@@ -40,28 +40,14 @@
 from sqlalchemy import MetaData, create_engine
 from werkzeug.local import LocalProxy
 
-from .holdings.api import Holding
 from .items.cli import create_items, reindex_items
 from .loans.cli import create_loans
 from .patrons.cli import import_users
+from .providers import append_pids_to_table
 
 _datastore = LocalProxy(lambda: current_app.extensions['security'].datastore)
 
 
-def append_pid_to_table(table, pids):
-    """Insert pids into an indentifier table."""
-    # TODO: avoid the connection to the database, recreate code
-    with current_app.app_context():
-        URI = current_app.config.get('SQLALCHEMY_DATABASE_URI')
-        engine = create_engine(URI)
-        metadata = MetaData(engine, reflect=True)
-        table_object = metadata.tables[table]
-        for pid in pids:
-            statement = "insert into {0} values('{1}')".format(
-                table_object, pid)
-            engine.execute(statement)
-
-
 @click.group()
 def fixtures():
     """Fixtures management commands."""
@@ -249,8 +235,8 @@ def create(infile, pid_type, schema, verbose, dbcommit, reindex, append):
             )
     if append:
         pids = record_class.get_all_pids()
-        table = record_class.provider.identifier.__tablename__
-        append_pid_to_table(table, pids)
+        table = record_class.provider.identifier
+        append_pids_to_table(table, pids)
 
 
 fixtures.add_command(create)

rero_ils/modules/documents/dojson/contrib/marc21tojson/model.py

@@ -207,7 +207,7 @@ def marc21_to_translatedFrom(self, key, value):
                     'value': lang_value.strip(),
                     'type': 'bf:Language'
                 }
-            language.append(lang)
+                language.append(lang)
     return language or None
 
 

rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1.json

@@ -89,18 +89,21 @@
             "description": "Language value.",
             "type": "string",
             "enum": [
-              "fre",
-              "ger",
-              "eng",
-              "ita",
-              "spa",
               "ara",
               "chi",
-              "lat",
+              "eng",
+              "fre",
+              "ger",
+              "gre",
               "heb",
+              "ita",
               "jpn",
+              "lat",
+              "pol",
               "por",
-              "rus"
+              "rus",
+              "spa",
+              "und"
             ]
           }
         }

rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1.json

@@ -88,18 +88,21 @@
             "description": "Language value.",
             "type": "string",
             "enum": [
-              "fre",
-              "ger",
-              "eng",
-              "ita",
-              "spa",
               "ara",
               "chi",
-              "lat",
+              "eng",
+              "fre",
+              "ger",
+              "gre",
               "heb",
+              "ita",
               "jpn",
+              "lat",
+              "pol",
               "por",
-              "rus"
+              "rus",
+              "spa",
+              "und"
             ]
           }
         }

rero_ils/modules/providers.py

@@ -19,11 +19,20 @@
 
 from __future__ import absolute_import, print_function
 
+from invenio_db import db
 from invenio_pidstore.errors import PIDDoesNotExistError
 from invenio_pidstore.models import PIDStatus
 from invenio_pidstore.providers.base import BaseProvider
 
 
+def append_pids_to_table(table, pids):
+    """Insert pids into an indentifier table."""
+    for pid in pids:
+        data = table(recid=pid)
+        db.session.add(data)
+    db.session.commit()
+
+
 class Provider(BaseProvider):
     """CircPolicy identifier provider.
 

scripts/setup

@@ -93,16 +93,19 @@ pipenv run invenio fixtures import_users ${DATA_PATH}/users.json -v
 # # to generate data file
 # pipenv run invenio fixtures create_items -c 10000 -i 5 -t ${DATA_PATH}/items.json -h ${DATA_PATH}/holdings.json
 # pipenv run flask utils check_json -r data/holdings.json
+
 # # Marc to xml transformation
-# yaz-marcdump -o marcxml data/10k.mrc > data/10k.xml
+# yaz-marcdump -o marcxml data/1k.mrc > data/1k.xml
+
 # # xml to json transformation for rero marcxml records
-# pipenv run dojson -i ${DATA_PATH}/10k.xml -l marcxml -d pjson do marc21tojson > ${DATA_PATH}/10k.json
+# pipenv run dojson -i ${DATA_PATH}/1k.xml -l marcxml -d pjson do marc21tojson > ${DATA_PATH}/1k.json
 # pipenv run dojson -i ${DATA_PATH}/300.xml -l marcxml -d pjson do marc21tojson > ${DATA_PATH}/300.json
+
 if [ "$1" = "--deployment" ]
 then
-    DOCUMENTS=${DATA_PATH}/10k.json
-    ITEMS=${DATA_PATH}/items10k.json
-    HOLDINGS=${DATA_PATH}/holdings10k.json
+    DOCUMENTS=${DATA_PATH}/1k.json
+    ITEMS=${DATA_PATH}/items1k.json 
+    HOLDINGS=${DATA_PATH}/holdings1k.json
 else
     DOCUMENTS=${DATA_PATH}/300.json
     ITEMS=${DATA_PATH}/items300.json
@@ -115,11 +118,11 @@ pipenv run invenio fixtures create --pid_type doc --schema 'http://ils.rero.ch/s
 
 # create items
 # NEEDS item type !!!
-# to generate 10k items file 10k documents must exist in DB
-# pipenv run invenio fixtures create_items -i 5 -t ${DATA_PATH}/items10k.json -h ${DATA_PATH}/holdings10k.json
-# pipenv run flask utils check_json -r data/holdings10k.json
+# to generate 1k items file 1k documents must exist in DB
+# pipenv run invenio fixtures create_items -i 2 -t ${DATA_PATH}/items1k.json -h ${DATA_PATH}/holdings1k.json
+# pipenv run flask utils check_json -r data/holdings1k.json
 # to generate 300 items file 300 documents must exist in DB
-# pipenv run invenio fixtures create_items -i 5 -t ${DATA_PATH}/items300.json -h ${DATA_PATH}/holdings300.json
+# pipenv run invenio fixtures create_items -i 2 -t ${DATA_PATH}/items300.json -h ${DATA_PATH}/holdings300.json
 # pipenv run flask utils check_json -r data/holdings300.json
 
 echo -e ${GREEN}Holdings:${NC} ${HOLDINGS}