Skip to content

Commit

Permalink
data model: implement bnf title and variants transformation
Browse files Browse the repository at this point in the history 
* Implements title and variants transformation for BnF UNIMARC.
* Fixes title_format_text_head for handling AttrDict accordingly.
* Adapts email templates to use new title format.
* Improves the way for getting the _text form bf:Title.
* Changes order of the display of title: original alphabet before transliteration.
* Changes order of the display of responsibilityStatement: original alphabet before transliteration.
* Changes order of the display of editionStatement: original alphabet before transliteration.
* Changes order of the display of provisionActivity: original alphabet before transliteration.
* Closes #876.
* Fixes bug in document JSONSchemas that prevent saving new document record in the editor.

Co-Authored-by: Gianni Pante <[email protected]>
Co-Authored-by: Peter Weber <[email protected]>
Co-Authored-by: Johnny Mariethoz <[email protected]>
  • Loading branch information
@reropag @rerowep @jma
3 people committed Apr 6, 2020
1 parent 2ca5138 commit b7626d3
Show file tree
Hide file tree
Showing 47 changed files with 17,101 additions and 16,034 deletions.
16 changes: 4 additions & 12 deletions rero_ils/dojson/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -491,10 +491,8 @@ def build_variant_title_data(self, string_set):
items = get_field_items(field_246['subfields'])
tag_link, link = get_field_link_data(field_246)
part_list = TitlePartList(
tag='246',
part_number_code='n',
part_name_code='p',
link=link
part_name_code='p'
)

subfield_selection = {'a', 'n', 'p'}
Expand All @@ -520,7 +518,7 @@ def build_variant_title_data(self, string_set):
'246', blob_key, blob_value,
index, link, ',.', ':;/-=')
part_list.update_part(
value_data, blob_key, blob_value, index)
value_data, blob_key, blob_value)
if blob_key != '__order__':
index += 1
the_part_list = part_list.get_part_list()
Expand All @@ -547,26 +545,20 @@ class TitlePartList(object):
method for each of these subfields. At the end of the parsing, the method
"get_part_list" provides the list of constructed parts.
:param tag: the marc field tag
:type tag: int
:param part_number_code: the specific subfield code
:type part_number_code: char
:param part_name_code: the specific subfield code
:type part_name_code: char
:param link: the link code to the alternate graphic field 880
:type link: str
"""

def __init__(self, tag, part_number_code, part_name_code, link):
def __init__(self, part_number_code, part_name_code):
"""Constructor method."""
self.tag = tag
self.link = link
self.part_number_waiting_name = {}
self.part_list = []
self.part_number_code = part_number_code
self.part_name_code = part_name_code

def update_part(self, value_data, subfield_code, subfield_data, index):
def update_part(self, value_data, subfield_code, subfield_data):
"""Update part data.
The part object is progressively build with the data collected by
Expand Down
3 changes: 1 addition & 2 deletions rero_ils/modules/documents/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@

from .models import DocumentIdentifier
from .utils import edition_format_text, publication_statement_text, \
series_format_text, title_format_text, title_format_text_head, \
title_variant_format_text
series_format_text, title_format_text_head
from ..acq_order_lines.api import AcqOrderLinesSearch
from ..api import IlsRecord, IlsRecordIndexer
from ..fetchers import id_fetcher
Expand Down
6 changes: 2 additions & 4 deletions rero_ils/modules/documents/dojson/contrib/marc21tojson/model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,10 +186,8 @@ def marc21_to_title(self, key, value):
title_list = []
title_data = {}
part_list = TitlePartList(
tag='245',
part_number_code='n',
part_name_code='p',
link=link
part_name_code='p'
)
parallel_titles = []
pararalel_title_data_list = []
Expand Down Expand Up @@ -219,7 +217,7 @@ def marc21_to_title(self, key, value):
elif blob_key == 'c':
responsibility = build_responsibility_data(value_data)
elif blob_key in ['n', 'p']:
part_list.update_part(value_data, blob_key, blob_value, index)
part_list.update_part(value_data, blob_key, blob_value)
if blob_key != '__order__':
index += 1
title_data['type'] = 'bf:Title'
Expand Down
103 changes: 75 additions & 28 deletions rero_ils/modules/documents/dojson/contrib/unimarctojson/model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
from dojson.utils import force_list
from pkg_resources import resource_string

from rero_ils.dojson.utils import ReroIlsOverdo, get_field_items, make_year, \
remove_trailing_punctuation
from rero_ils.dojson.utils import ReroIlsOverdo, TitlePartList, \
get_field_items, make_year, remove_trailing_punctuation

unimarctojson = ReroIlsOverdo()

Expand Down Expand Up @@ -83,20 +83,64 @@ def unimarc_bnf_id(self, key, value):
@unimarctojson.over('title', '^200..')
@utils.ignore_value
def unimarc_title(self, key, value):
"""Get title.
title: 200$a
If there's a $e, then 245$a : $e
"""Get title data.
field 200: non repetitive
$a : repetitive
$e : repetitive
$f : repetitive
$g : repetitive
$h : repetitive
$i : repetitive
field 510,512,514,515,516,517,518,519,532: repetitive
$a : non repetitive
$e : repetitive
$h : repetitive
$i : repetitive
"""
main_title = value.get('a')
if main_title:
main_title = utils.force_list(main_title)[0]
else:
main_title = ''
sub_title = utils.force_list(value.get('e'))
if sub_title:
main_title += ' : ' + ' : '.join(sub_title)
return main_title or None
title_list = []
responsibilites = []
for tag in ['200', '510',
'512', '514', '515', '516', '517', '518', '519', '532']:
for field in unimarctojson.get_fields(tag=tag):
title_data = {}
part_list = TitlePartList(
part_number_code='h',
part_name_code='i'
)
subfields_a = unimarctojson.get_subfields(field, 'a')
subfields_e = unimarctojson.get_subfields(field, 'e')
if subfields_a:
title_data['mainTitle'] = [dict(value=subfields_a[0])]
for subfield_e in subfields_e:
title_data['subtitle'] = [dict(value=subfield_e)]
title_type = 'bf:VariantTitle'
if tag == '200':
title_type = 'bf:Title'
elif tag == '510':
title_type = 'bf:ParallelTitle'
# build title parts
items = get_field_items(field['subfields'])
for blob_key, blob_value in items:
if blob_key in ['f', 'g'] and tag == '200':
responsibilites.append([dict(value=blob_value)])
if blob_key in ['h', 'i']:
part_list.update_part(
[dict(value=blob_value)], blob_key, blob_value)
title_data['type'] = title_type
the_part_list = part_list.get_part_list()
if the_part_list:
title_data['part'] = the_part_list
if title_data:
title_list.append(title_data)

# extract responsibilities
if responsibilites:
new_responsibility = self.get('responsibilityStatement', [])
for resp in responsibilites:
new_responsibility.append(resp)
self['responsibilityStatement'] = new_responsibility
return title_list or None


@unimarctojson.over('titlesProper', '^500..')
Expand All @@ -107,7 +151,7 @@ def unimarc_titles_proper(self, key, value):
titleProper: 500$a
"""
return value.get('a')
return value.get('a', '')


@unimarctojson.over('language', '^101')
Expand Down Expand Up @@ -154,15 +198,16 @@ def unimarc_to_author(self, key, value):
712 Nom de collectivité – Responsabilité secondaire
"""
author = {}
author['name'] = value.get('a')
author['name'] = ', '.join(utils.force_list(value.get('a', '')))
author['type'] = 'person'
if key[1] == '1':
author['type'] = 'organisation'

if value.get('b'):
author['name'] += ', ' + ', '.join(utils.force_list(value.get('b')))
if value.get('d'):
author['name'] += ' ' + ' '.join(utils.force_list(value.get('d')))
if author['name']:
if value.get('b'):
author['name'] += \
', ' + ', '.join(utils.force_list(value.get('b')))
if value.get('d'):
author['name'] += ' ' + ' '.join(utils.force_list(value.get('d')))

if value.get('c'):
author['qualifier'] = value.get('c')
Expand Down Expand Up @@ -366,7 +411,7 @@ def unimarc_abstracts(self, key, value):
abstract: [330$a repetitive]
"""
return ', '.join(utils.force_list(value.get('a')))
return ', '.join(utils.force_list(value.get('a', '')))


@unimarctojson.over('identifiedBy', '^073..')
Expand Down Expand Up @@ -399,7 +444,7 @@ def unimarc_notes(self, key, value):
note: [300$a repetitive]
"""
return value.get('a')
return value.get('a', '')


@unimarctojson.over('subjects', '^6((0[0-9])|(1[0-7]))..')
Expand Down Expand Up @@ -430,8 +475,10 @@ def unimarc_subjects(self, key, value):
@utils.ignore_value
def marc21_to_electronicLocator_from_field_856(self, key, value):
"""Get electronicLocator from field 856."""
electronic_locator = {
'url': value.get('u'),
'type': 'resource'
}
electronic_locator = None
if value.get('u'):
electronic_locator = {
'url': value.get('u'),
'type': 'resource'
}
return electronic_locator
16 changes: 13 additions & 3 deletions rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1_src.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,13 +156,17 @@
"minItems": 1,
"items": {
"$ref": "#/definitions/language_script"
},
"form": {
"hide": true
}
},
"part": {
"title": "Part",
"title": "Parts",
"type": "array",
"minItems": 1,
"items": {
"title": "Part",
"type": "object",
"propertiesOrder": [
"partNumber",
Expand Down Expand Up @@ -201,6 +205,7 @@
"minItems": 1,
"title": "Responsibilities",
"items": {
"title": "Responsibility",
"type": "array",
"minItems": 1,
"items": {
Expand Down Expand Up @@ -254,6 +259,7 @@
],
"additionalProperties": false,
"propertiesOrder": [
"type",
"value"
],
"properties": {
Expand All @@ -266,6 +272,7 @@
"bf:Language"
],
"form": {
"hideExpression": "true",
"options": [
{
"label": "bf:Language",
Expand Down Expand Up @@ -3671,7 +3678,10 @@
"und-jpan",
"und-kore",
"und-zyyy"
]
],
"form": {
"hide": true
}
},
"language_script": {
"title": "Values",
Expand Down Expand Up @@ -5779,4 +5789,4 @@
}
}
}
}
}
16 changes: 13 additions & 3 deletions rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1_src.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,13 +156,17 @@
"minItems": 1,
"items": {
"$ref": "#/definitions/language_script"
},
"form": {
"hide": true
}
},
"part": {
"title": "Part",
"title": "Parts",
"type": "array",
"minItems": 1,
"items": {
"title": "Part",
"type": "object",
"propertiesOrder": [
"partNumber",
Expand Down Expand Up @@ -201,6 +205,7 @@
"minItems": 1,
"title": "Responsibilities",
"items": {
"title": "Responsibility",
"type": "array",
"minItems": 1,
"items": {
Expand Down Expand Up @@ -254,6 +259,7 @@
],
"additionalProperties": false,
"propertiesOrder": [
"type",
"value"
],
"properties": {
Expand All @@ -266,6 +272,7 @@
"bf:Language"
],
"form": {
"hideExpression": "true",
"options": [
{
"label": "bf:Language",
Expand Down Expand Up @@ -3671,7 +3678,10 @@
"und-jpan",
"und-kore",
"und-zyyy"
]
],
"form": {
"hide": true
}
},
"language_script": {
"title": "Values",
Expand Down Expand Up @@ -5779,4 +5789,4 @@
}
}
}
}
}
3 changes: 3 additions & 0 deletions rero_ils/modules/documents/serializers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,18 @@ def preprocess_record(self, pid, record, links_factory=None, **kwargs):
"""Prepare a record and persistent identifier for serialization."""
rec = record
titles = rec.get('title', [])
# build responsibility data for display purpose
responsibility_statement = rec.get('responsibilityStatement', [])
responsibilities = \
create_title_responsibilites(responsibility_statement)
if responsibilities:
rec['ui_responsibilities'] = responsibilities
# build alternate graphic title data for display purpose
altgr_titles = create_title_alternate_graphic(titles)
if altgr_titles:
rec['ui_title_altgr'] = altgr_titles
variant_titles = create_title_variants(titles)
# build variant title data for display purpose
if variant_titles:
rec['ui_title_variants'] = variant_titles
if request and request.args.get('resolve') == '1':
Expand Down
2 changes: 1 addition & 1 deletion rero_ils/modules/documents/templates/rero_ils/detailed_view_documents.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
</icon-thumbnail>
</div>
{% endif %}
<h1 class="col-sm-10 col-md-11">{{ record.dumps().title[0]._text }}</h1>
<h1 class="col-sm-10 col-md-11">{{ record.title | create_title_text }}</h1>
</header>

{%- block record_body %}
Expand Down
Loading

0 comments on commit b7626d3

Please sign in to comment.