metadata: import, export, create csv data files

* Implements import and export of resources in csv format.
* Implements creation of csv files from json files.
* Changes from sqlite to postgres as test DB.

Co-Authored-by: Peter Weber <[email protected]>

data/pid_dependencies_small.json

@@ -0,0 +1,255 @@
+[
+  {
+    "name": "organisation",
+    "filename": "organisations.json"
+  },
+  {
+    "name": "library",
+    "filename": "libraries.json",
+    "dependencies": [
+      {
+        "name": "organisation"
+      }
+    ]
+  },
+  {
+    "name": "location",
+    "filename": "locations.json",
+    "dependencies": [
+      {
+        "name": "library"
+      }
+    ]
+  },
+  {
+    "name": "pickup_location",
+    "filename": "locations.json"
+  },
+  {
+    "name": "item_type",
+    "filename": "item_types.json",
+    "dependencies": [
+      {
+        "name": "organisation"
+      }
+    ]
+  },
+  {
+    "name": "patron_type",
+    "filename": "patron_types.json",
+    "dependencies": [
+      {
+        "name": "organisation"
+      },
+      {
+        "name": "library_exceptions",
+        "optional": "True",
+        "sublist": [
+          {
+            "name": "library"
+          }
+        ]
+      }
+    ]
+  },
+  {
+    "name": "circulation_policie",
+    "filename": "circulation_policies.json",
+    "dependencies": [
+      {
+        "name": "organisation"
+      },
+      {
+        "name": "settings",
+        "optional": "True",
+        "sublist": [
+          {
+            "name": "patron_type"
+          },
+          {
+            "name": "item_type"
+          }
+        ]
+      },
+      {
+        "name": "library",
+        "optional": "True"
+      }
+    ]
+  },
+  {
+    "name": "vendor",
+    "filename": "vendors.json",
+    "dependencies": [
+      {
+        "name": "organisation"
+      }
+    ]
+  },
+  {
+    "name": "patron",
+    "filename": "users.json",
+    "dependencies": [
+      {
+        "name": "library",
+        "optional": "True"
+      },
+      {
+        "name": "patron",
+        "optional": "True",
+        "sublist": [
+          {
+            "name": "patron_type",
+            "optional": "True"
+          }
+        ]
+      }
+    ]
+  },
+  {
+    "name": "document",
+    "filename": "documents_small.json"
+  },
+  {
+    "name": "holding",
+    "filename": "holdings_small.json",
+    "dependencies": [
+      {
+        "name": "location"
+      },
+      {
+        "name": "circulation_category",
+        "ref": "item_type"
+      },
+      {
+        "name": "document"
+      }
+    ]
+  },
+  {
+    "name": "item",
+    "filename": "items_small.json",
+    "dependencies": [
+      {
+        "name": "location"
+      },
+      {
+        "name": "document"
+      },
+      {
+        "name": "item_type"
+      },
+      {
+        "name": "holding",
+        "optional": "True"
+      },
+      {
+        "name": "organisation",
+        "optional": "True"
+      }
+    ]
+  },
+  {
+    "name": "patterns",
+    "filename": "patterns.json"
+  },
+  {
+    "name": "budget",
+    "filename": "budgets.json",
+    "dependencies": [
+      {
+        "name": "organisation",
+        "optional": "True"
+      }
+    ]
+  },
+  {
+    "name": "collection",
+    "filename": "collections.json",
+    "dependencies": [
+      {
+        "name": "librarie",
+        "ref": "libraries",
+        "optional": "True"
+      },
+      {
+        "name": "item",
+        "ref": "items",
+        "optional": "True"
+      },
+      {
+        "name": "organisation"
+      }
+    ]
+  },
+  {
+    "name": "acq_account",
+    "filename": "acq_accounts.json",
+    "dependencies": [
+      {
+        "name": "budget"
+      },
+      {
+        "name": "library"
+      },
+      {
+        "name": "organisation",
+        "optional": "True"
+      }
+    ]
+  },
+  {
+    "name": "template",
+    "filename": "templates.json",
+    "dependencies": [
+      {
+        "name": "organisation"
+      },
+      {
+        "name": "creator",
+        "ref": "patron"
+      }
+    ]
+  },
+  {
+    "name": "loan",
+    "filename": "loans.json",
+    "dependencies": [
+      {
+        "name": "item",
+        "optional": "True"
+      },
+      {
+        "name": "patron",
+        "optional": "True"
+      },
+      {
+        "name": "document",
+        "optional": "True"
+      },
+      {
+        "name": "organisation",
+        "optional": "True"
+      }
+    ]
+  },
+  {
+    "name": "local_field",
+    "filename": "local_fields.json",
+    "dependencies": [
+      {
+        "name": "organisation",
+        "optional": "True"
+      },
+      {
+        "name": "parent",
+        "optional": "True",
+        "refs": {
+          "document": "documents",
+          "item": "items",
+          "holding": "holdings"
+        }
+      }
+    ]
+  }
+]

rero_ils/modules/api.py

@@ -40,11 +40,11 @@
 from invenio_records_rest.utils import obj_or_import_string
 from invenio_search import current_search
 from invenio_search.api import RecordsSearch
+from jsonschema.exceptions import ValidationError
 from kombu.compat import Consumer
 from sqlalchemy import text
 from sqlalchemy.orm.exc import NoResultFound
 
-from .errors import RecordValidationError
 from .utils import extracted_data_from_ref
 
 
@@ -142,7 +142,7 @@ def _validate(self, **kwargs):
                 not_required=self.pids_exist_check.get('not_required', {})
             ) or True
         if validation_message is not True:
-            raise RecordValidationError(validation_message)
+            raise ValidationError(validation_message)
         return json
 
     def extended_validation(self, **kwargs):
@@ -441,6 +441,13 @@ def organisation_pid(self):
         """Get organisation pid for circulation policy."""
         return extracted_data_from_ref(self.get('organisation'))
 
+    @classmethod
+    def get_metadata_identifier_names(cls):
+        """Get metadata and identif table names."""
+        metadata = cls.model_cls.__tablename__
+        identifier = cls.provider.identifier
+        return metadata, identifier
+
 
 class IlsRecordsIndexer(RecordIndexer):
     """Indexing class for ils."""