documents: complete the data conversion

    * Implements transformation from Marc21 to JSON RERO ILS for:
        * frequency (L32).
        * bf:usageAndAccessPolicy (L74).
    * closes rero#1617.
    * closes rero#1951.
    * closes rero#1987.
    * closes rero#1996.
Co-Authored-by: Gianni Pante <[email protected]>

rero_ils/dojson/utils.py

@@ -962,8 +962,11 @@ def do(self, blob, ignore_missing=True, exception_handlers=None):
             self.field_008_data = ''
             self.date1_from_008 = None
             self.date2_from_008 = None
+            self.original_date_from_008 = None
             self.date_type_from_008 = ''
             self.date = {'start_date': None}
+            self.serial_type = ''
+            self.is_top_level_record = False
             fields_008 = self.get_fields(tag='008')
             if fields_008:
                 self.field_008_data = self.get_control_field_data(
@@ -995,9 +998,9 @@ def do(self, blob, ignore_missing=True, exception_handlers=None):
             # identifiy a top level record (has 019 $a Niveau supérieur)
             regexp = re.compile(r'Niveau sup[eé]rieur', re.IGNORECASE)
             fields_019 = self.get_fields(tag='019')
-            note = ''
             notes_from_019_and_351 = []
             for field_019 in fields_019:
+                note = ''
                 for subfield_a in self.get_subfields(field_019, 'a'):
                     note += ' | ' + subfield_a
                     if regexp.search(subfield_a):
@@ -1107,7 +1110,7 @@ def init_lang_from(fields_041, code):
                         langs_from_041.append(lang_from_041)
             return langs_from_041
 
-        self.lang_from_008 = ''
+        self.lang_from_008 = None
         self.langs_from_041_a = []
         self.langs_from_041_h = []
         try:

rero_ils/modules/documents/dojson/contrib/marc21tojson/model.py

@@ -217,6 +217,18 @@
 
 marc21 = ReroIlsMarc21Overdo()
 
+def build_place():
+    """Build place data for provisionActivity."""
+
+    place = {}
+    if marc21.cantons:
+        place['canton'] = marc21.cantons[0]
+    if marc21.country:
+        place['country'] = marc21.country
+    if place:
+        place['type'] = 'bf:Place'
+    return place
+
 
 def get_contribution_link(bibid, reroid, id, key):
     """Get MEF contribution link.
@@ -346,7 +358,14 @@ def marc21_to_language(self, key, value):
         if fields_264:
             error_print('WARNING INVALID 264', marc21.bib_id, marc21.rero_id,
                         fields_264)
-        self['provisionActivity'] = [{'type': 'bf:Publication'}]
+        publication = {
+            'type': 'bf:Publication'
+        }
+        place = build_place()
+        if place:
+            publication['place'] = [place]
+        self['provisionActivity'] = [publication]
+
         if (marc21.date_type_from_008 == 'q' or
                 marc21.date_type_from_008 == 'n'):
             self['provisionActivity'][0][
@@ -743,16 +762,6 @@ def build_agent_data(code, label, index, link):
                 index += 1
         return statement
 
-    def build_place():
-        place = {}
-        if marc21.cantons:
-            place['canton'] = marc21.cantons[0]
-        if marc21.country:
-            place['country'] = marc21.country
-        if place:
-            place['type'] = 'bf:Place'
-        return place
-
     # the function marc21_to_provisionActivity start here
     ind2 = key[4]
     type_per_ind2 = {
@@ -870,6 +879,47 @@ def marc21_to_summary(self, key, value):
         self['tableOfContents'] = table_of_contents_list
 
 
+@marc21.over('usageAndAccessPolicy', '^(506|540)..')
+@utils.ignore_value
+def marc21_to_usage_and_access_policy_from_field_506_540(self, key, value):
+    """Get usageAndAccessPolicy from fields: 506, 540."""
+    subfield_a = not_repetitive(marc21.bib_id, marc21.rero_id,
+                                key, value, 'a', default='').strip()
+    if subfield_a:
+        policy = {
+            'type': 'bf:UsageAndAccessPolicy',
+            'label': subfield_a
+        }
+        usage_and_access_policy = self.get('usageAndAccessPolicy', [])
+        usage_and_access_policy.append(policy)
+    return usage_and_access_policy or None
+
+
+@marc21.over('frequency', '^(310|321)..')
+@utils.ignore_value
+def marc21_to_frequency_field_310_321(self, key, value):
+    """Get frequency from fields: 310, 321."""
+    subfield_a = not_repetitive(
+        marc21.bib_id, marc21.rero_id,
+        key, value, 'a', default='missing_label').strip()
+    subfield_b = not_repetitive(
+        marc21.bib_id, marc21.rero_id,
+        key, value, 'b', default='').strip()
+
+    frequency = {
+        'label': remove_trailing_punctuation(
+                    data=subfield_a,
+                    punctuation=',',
+                    spaced_punctuation=','
+                )
+        }
+    if subfield_b:
+        frequency['date'] = subfield_b
+    frequency_list = self.get('frequency', [])
+    frequency_list.append(frequency)
+    return frequency_list or None
+
+
 @marc21.over('dissertation', '^502..')
 @utils.for_each_value
 @utils.ignore_value
@@ -1329,7 +1379,7 @@ def marc21_to_identifiedBy_from_field_930(self, key, value):
     return identifiedBy or None
 
 
-@marc21.over('note', '^(500|510|530|545|580)..')
+@marc21.over('note', '^(500|510|530|545|555|580)..')
 @utils.for_each_value
 @utils.ignore_value
 def marc21_to_notes_and_original_title(self, key, value):