OAIHARVESTING: bulk indexing of oai reords

* Improves indexing of harvested oai records.

Co-Authored-by: Peter Weber <[email protected]>

rero_ils/config.py

@@ -224,6 +224,7 @@ def _(x):
 
 }
 CELERY_BROKER_HEARTBEAT = 0
+INDEXER_BULK_REQUEST_TIMEOUT = 60
 
 # Database
 # ========

rero_ils/modules/ebooks/tasks.py

@@ -23,13 +23,15 @@
 from flask import current_app
 
 from ..documents.api import Document, DocumentsSearch
+from ..utils import bulk_index
 
 
 @shared_task(ignore_result=True)
 def create_records(records):
     """Records creation and indexing."""
     n_updated = 0
     n_created = 0
+    uuids = []
     for record in records:
         record['$schema'] = \
             'https://ils.rero.ch/schema/documents/document-minimal-v0.0.1.json'
@@ -55,15 +57,19 @@ def create_records(records):
             existing_record.update(
                 record,
                 dbcommit=True,
-                reindex=True)
+                reindex=False
+            )
             n_updated += 1
+            uuids.append(existing_record.id)
         else:
             # create a new record
-            Document.create(
+            new_record = Document.create(
                 record,
                 dbcommit=True,
-                reindex=True
+                reindex=False
             )
             n_created += 1
+            uuids.append(new_record.id)
+    bulk_index(uuids, process=True)
     current_app.logger.info('create_records: {} updated, {} new'
                             .format(n_updated, n_created))

rero_ils/modules/utils.py

@@ -18,8 +18,10 @@
 """Utilities for rero-ils editor."""
 
 from datetime import time
+from time import sleep
 
-from flask import url_for
+import click
+from flask import current_app
 from invenio_indexer.api import RecordIndexer
 
 
@@ -30,3 +32,29 @@ def strtotime(strtime):
         hour=int(splittime[0]),
         minute=int(splittime[1])
     )
+
+
+def bulk_index(uuids, process=False, verbose=False):
+    """Bulk index records."""
+    if verbose:
+        click.echo(' add to index: {count}'.format(count=len(uuids)))
+    indexer = RecordIndexer()
+    retry = True
+    minutes = 1
+    while retry:
+        try:
+            indexer.bulk_index(uuids)
+            retry = False
+        except Exception as exc:
+            msg = 'Bulk Index Error: retry in {minutes} min {exc}'.format(
+                exc=exc,
+                minutes=minutes
+            )
+            current_app.logger.error(msg)
+            if verbose:
+                click.secho(msg, fg='red')
+            sleep(minutes * 60)
+            retry = True
+            minutes *= 2
+    if process:
+        indexer.process_bulk_queue()