diff --git a/Pipfile b/Pipfile index 76444cde..740854bc 100644 --- a/Pipfile +++ b/Pipfile @@ -7,7 +7,7 @@ name = "pypi" [packages] -Django = "*" +django = "*" dj-database-url = "*" django-dirtyfields = "*" django-filter = "*" @@ -16,7 +16,7 @@ django-tequila = {git = "https://github.com/epfl-idevelop/django-tequila.git", e "django-bootstrap4" = "*" gevent = "*" gunicorn = "*" -Markdown = "*" +markdown = "*" "psycopg2-binary" = "*" pymarc = "*" pytz = "*" @@ -31,9 +31,9 @@ codecov = "*" django-debug-toolbar = "*" django-extensions = "*" django-nose = "*" -"Fabric3" = "*" -factory_boy = "*" -Faker = "*" +"fabric3" = "*" +factory-boy = "*" +faker = "*" "flake8" = "*" graypy = "*" ipdb = "*" @@ -44,6 +44,7 @@ nose-progressive = "*" selenium = "*" requests = "*" rope = "*" +pytest = "*" [requires] diff --git a/Pipfile.lock b/Pipfile.lock index 68e8e558..d02ca9a4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "58622a2b45de462ea974c66c259ae5a45d66d735434e93811b0c00710dd12920" + "sha256": "6374e4040c7f215cf0219ffdab17246d1d5739daabe703714706845a6026884c" }, "pipfile-spec": 6, "requires": { @@ -219,6 +219,13 @@ ], "version": "==0.24.0" }, + "attrs": { + "hashes": [ + "sha256:1c7960ccfd6a005cd9f7ba884e6316b5e430a3f1a6c37c5f87d8b43f83b54ec9", + "sha256:a17a9573a6f475c99b551c0e0a812707ddda1ec9653bed04c13841404ed6f450" + ], + "version": "==17.4.0" + }, "autopep8": { "hashes": [ "sha256:c7be71ab0cb2f50c9c22c82f0c9acaafc6f57492c3fbfee9790c415005c2b9a5" @@ -329,6 +336,14 @@ "index": "pypi", "version": "==2.0.15" }, + "colorama": { + "hashes": [ + "sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda", + "sha256:48eb22f4f8461b1df5734a074b57042430fb06e1d61bd1e11b078c0fe6d7a1f1" + ], + "markers": "sys_platform == 'win32'", + "version": "==0.3.9" + }, "coverage": { "hashes": [ "sha256:03481e81d558d30d230bc12999e3edffe392d244349a90f4ef9b88425fac74ba", @@ -432,11 +447,11 @@ }, "django-extensions": { "hashes": [ - "sha256:1f424a7f87974c2e2602b8b41cae52eb08105523f0c70320203abf58bcb84404", - "sha256:24c24bbc6ef6dd36fe6b2b7c48d171a9d22fe76895610fe19087af657fa27930" + "sha256:37a543af370ee3b0721ff50442d33c357dd083e6ea06c5b94a199283b6f9e361", + "sha256:bc9f2946c117bb2f49e5e0633eba783787790ae810ea112fe7fd82fa64de2ff1" ], "index": "pypi", - "version": "==2.0.5" + "version": "==2.0.6" }, "django-nose": { "hashes": [ @@ -484,6 +499,14 @@ "index": "pypi", "version": "==3.5.0" }, + "funcsigs": { + "hashes": [ + "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", + "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" + ], + "markers": "python_version < '3.0'", + "version": "==1.0.2" + }, "graypy": { "hashes": [ "sha256:1a38da6a17b02b8d016fde64618effc611b6fcb092200728da622760e612c175", @@ -631,6 +654,12 @@ ], "version": "==0.7.4" }, + "pluggy": { + "hashes": [ + "sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff" + ], + "version": "==0.6.0" + }, "prompt-toolkit": { "hashes": [ "sha256:1df952620eccb399c53ebb359cc7d9a8d3a9538cb34c5a1344bdbeb29fbcc381", @@ -646,6 +675,13 @@ ], "version": "==0.5.2" }, + "py": { + "hashes": [ + "sha256:8cca5c229d225f8c1e3085be4fcf306090b00850fefad892f9d96c7b6e2f310f", + "sha256:ca18943e28235417756316bfada6cd96b23ce60dd532642690dcfdaba988a76d" + ], + "version": "==1.5.2" + }, "pyasn1": { "hashes": [ "sha256:0d7f6e959fe53f3960a23d73f35e1fce61348b30915b6664309ca756de7c1f89", @@ -718,6 +754,14 @@ ], "version": "==1.2.1" }, + "pytest": { + "hashes": [ + "sha256:062027955bccbc04d2fcd5d79690947e018ba31abe4c90b2c6721abec734261b", + "sha256:117bad36c1a787e1a8a659df35de53ba05f9f3398fb9e4ac17e80ad5903eb8c5" + ], + "index": "pypi", + "version": "==3.4.2" + }, "python-dateutil": { "hashes": [ "sha256:07009062406cffd554a9b4135cd2ff167c9bf6b7aac61fe946c93e69fad1bbd8", diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 03408546..36c38059 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -52,4 +52,4 @@ selenium: image: selenium/standalone-chrome-debug:3.11 ports: - "4444:4444" # Selenium - - "5900:5900" # VNC server, pass is "secret" + - "${VNC_PORT}:5900" # VNC server, pass is "secret" diff --git a/env/django.env b/env/django.env index f66494a5..796541ba 100644 --- a/env/django.env +++ b/env/django.env @@ -19,6 +19,7 @@ SERVER_HOST=${ALLOWED_HOSTS} # -> 80 in production # -> 8000 or 8080 in dev DEV_PORT=8000 +VNC_PORT=5900 # `SITE_PATH` is the root path of the application (it not hosted on a bare domain) SITE_PATH=/publications-exports diff --git a/infoscience_exports/exports/marc21xml.py b/infoscience_exports/exports/marc21xml.py index 66a68e98..36754018 100644 --- a/infoscience_exports/exports/marc21xml.py +++ b/infoscience_exports/exports/marc21xml.py @@ -3,7 +3,7 @@ """ Parse a marc-21-xml file """ - +from logging import getLogger from django.utils.translation import gettext as _ from django.conf import settings from os.path import dirname, splitext @@ -12,27 +12,22 @@ from pymarc import marcxml import unicodedata +logger = getLogger(__name__) -def get_attributes(subfields): - res_value = {} - for element1 in subfields: - for key1, value1 in element1.items(): - res_value[key1] = value1 - return res_value +class Author: -# Authors is a list of a dictionary of author: full name, initial name, url in infoscience -def set_authors(authors): - result = [] - for author in authors: - author_record = {} - author_record['fullname'] = author - author_record['search_url'] = "{}/search?p={}".format( + def __init__(self, author): + self.fullname = author + self.search_url = "{}/search?p={}".format( settings.SITE_DOMAIN, author.replace(",", "+").replace(" ", "+")) + self.initname = self.compute_name() - names = author.split(',') + def compute_name(self): + names = self.fullname.split(',') family = names[0].strip() if len(names) > 0 else '' fnames = names[1].split(' ') if len(names) > 1 else '' + initname = "" for fname in fnames: if not fname: @@ -53,23 +48,22 @@ def set_authors(authors): if family: initname += family - author_record['initname'] = initname - result.append(author_record) + return initname - return result + +# Authors is a list of Author instance: full name, initial name, url in infoscience +def set_authors(authors): + return [Author(author) for author in authors] # get only the year in a date-string def set_year(date): - if len(date) == 4: - return date - dates = date.split("-") - year = date - for val in dates: - if len(val) == 4: - year = val - break - return year + dates = [val for val in date.split("-") if len(val) == 4] + if len(dates) == 1: + return dates[0] + else: + logger.warning("Year not found in %s, decomposed as %s", date, dates) + return '' # get fulltext: link to pdf or link to repository if several links @@ -110,6 +104,14 @@ def set_fulltext(fulltexts): return result +def get_attributes(subfields): + res_value = {} + for element1 in subfields: + for key1, value1 in element1.items(): + res_value[key1] = value1 + return res_value + + # get dictionary (icon, fulltexts) of ELA def get_ELA_fields(field): ela_fulltexts = [] diff --git a/infoscience_exports/exports/options_notices.py b/infoscience_exports/exports/options_notices.py index df937a94..9f5e6fc3 100644 --- a/infoscience_exports/exports/options_notices.py +++ b/infoscience_exports/exports/options_notices.py @@ -13,7 +13,7 @@ DOC_TYPE_ORDERED = { - #('ARTICLE', _("Journal Articles")), + # ('ARTICLE', _("Journal Articles")), 'ARTICLE': _("Articles & Reviews"), 'CONF': _("Conference Papers"), 'REVIEW': _("Reviews"), @@ -75,17 +75,17 @@ def get_sorted_by_doc_types(notices): # add doc_types not listed in DOC_TYPE_ORDERED for index, head in enumerate(groups_head): if head not in doc_type_keys: - groups_list_ordered.extend(groups_list[index]) + groups_list_ordered.extend(groups_list[index]) return groups_list_ordered def get_sorted_by_year(notices, url): queries = parse_qs(urlsplit(url).query) - is_ascending = queries.get('so', ['d'])[0] == "a" - if is_ascending: - notices = sorted(notices, key=lambda k: k['Publication_Year']) - else: - notices = sorted(notices, key=lambda k: k['Publication_Year'], reverse=True) + is_ascending = queries.get('so', ['d'])[0] == "a" + if is_ascending: + notices = sorted(notices, key=lambda k: k['Publication_Year']) + else: + notices = sorted(notices, key=lambda k: k['Publication_Year'], reverse=True) return notices diff --git a/infoscience_exports/exports/test/test_authors.py b/infoscience_exports/exports/test/test_authors.py new file mode 100644 index 00000000..7de16cc1 --- /dev/null +++ b/infoscience_exports/exports/test/test_authors.py @@ -0,0 +1,26 @@ +from django.conf import settings + +from exports.marc21xml import Author + + +def expected_url(name): + return "{}/search?p={}".format(settings.SITE_DOMAIN, name) + + +def test_standard_case(): + author = Author("Emmanuel Bréton") + assert author.fullname == "Emmanuel Bréton" + assert author.search_url == expected_url("Emmanuel+Bréton") + assert author.initname == "Emmanuel Bréton" + + +def test_search_url(): + assert Author("Jean-Marc Marco").search_url == expected_url("Jean-Marc+Marco") + assert Author("Jean, Marc Marco").search_url == expected_url("Jean++Marc+Marco") + + +def test_initial_names(): + assert Author("Marco Jean Marc").initname == "Marco Jean Marc" + assert Author("Marco, Jean").initname == "J. Marco" + assert Author("Marco, Jean Marc").initname == "J. M. Marco" + assert Author("Marco, Jean-Marc").initname == "J.-M. Marco" diff --git a/infoscience_exports/exports/test/test_dates.py b/infoscience_exports/exports/test/test_dates.py new file mode 100644 index 00000000..95590de7 --- /dev/null +++ b/infoscience_exports/exports/test/test_dates.py @@ -0,0 +1,14 @@ +from exports.marc21xml import set_year + + +def test_year_only(): + assert set_year('2018') == '2018' + + +def test_full_date(): + assert set_year('2018-01-01') == '2018' + assert set_year('01-01-2018') == '2018' + + +def test_error(): + assert set_year('2018.01.01') == '' diff --git a/infoscience_exports/exports/versions.py b/infoscience_exports/exports/versions.py index 4ed39666..bd9e0da9 100644 --- a/infoscience_exports/exports/versions.py +++ b/infoscience_exports/exports/versions.py @@ -3,7 +3,7 @@ # the release comes from git and should not be modified # => read-only -_release = '0.3.7-3-g3a4e66e' +_release = '0.3.7-16-g5e0cae8' # you can set the next version number manually # if you do not, the system will make sure that version > release @@ -13,5 +13,4 @@ # the build number will generate conflicts on each PR merge # just keep yours every time # => read-only - -_build = '086e59f5d9fdf7c643968137b2f1fe3936657d3d' +_build = '5e0cae8b46c21d3448ab566f21295f59d76bbf8f' \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index b3d81ebc..4cde4666 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,7 @@ -r requirements.txt appnope==0.1.0; sys_platform == 'darwin' asn1crypto==0.24.0 +attrs==17.4.0 autopep8==1.3.4 bcrypt==3.1.4 blessings==1.6.1 @@ -9,12 +10,13 @@ cffi==1.11.5 chardet==3.0.4 click==6.7 codecov==2.0.15 +colorama==0.3.9; sys_platform == 'win32' coverage==4.5.1 coveralls==1.3.0 cryptography==2.1.4 decorator==4.2.1 django-debug-toolbar==1.9.1 -django-extensions==2.0.5 +django-extensions==2.0.6 django-nose==1.4.5 django==2.0.3 docopt==0.6.2 @@ -22,6 +24,7 @@ fabric3==1.14.post1 factory-boy==2.10.0 faker==0.8.12 flake8==3.5.0 +funcsigs==1.0.2; python_version < '3.0' graypy==0.2.14 idna==2.6 ipdb==0.11 @@ -42,14 +45,17 @@ parso==0.1.1 pbr==3.1.1 pexpect==4.4.0; sys_platform != 'win32' pickleshare==0.7.4 +pluggy==0.6.0 prompt-toolkit==1.0.15 ptyprocess==0.5.2 +py==1.5.2 pyasn1==0.4.2 pycodestyle==2.3.1 pycparser==2.18 pyflakes==1.6.0 pygments==2.2.0 pynacl==1.2.1 +pytest==3.4.2 python-dateutil==2.7.0 pytz==2018.3 pyyaml==3.12