fixed build

Signed-off-by: Spiros Delviniotis <[email protected]>

hepcrawl/crawler2hep.py

@@ -35,12 +35,6 @@ def _get_updated_fft_fields(current_fft_fields, record_files):
         for record_file in record_files
     }
     new_fft_fields = []
-    import logging
-    logger = logging.getLogger(__name__)
-    logger.log(logging.INFO,
-               "-------------------- _get_updated_fft_fields -------------------")
-    logger.log(logging.INFO,
-               'current_fft_fields: {}'.format(current_fft_fields))
     for fft_field in current_fft_fields:
         file_name = os.path.basename(fft_field['path'])
         if file_name in record_files_index:
@@ -151,7 +145,7 @@ def item_to_hep(
         )
     elif item.record_format == 'hepcrawl':
         item = _normalize_hepcrawl_record(
-            item=item,
+            item=item.record,
             source=source,
         )
         return hepcrawl_to_hep(dict(item))
@@ -160,10 +154,11 @@ def item_to_hep(
 
 
 def hep_to_hep(hep_record, record_files):
-    hep_record['_fft'] = _get_updated_fft_fields(
-        current_fft_fields=hep_record['_fft'],
-        record_files=record_files,
-    )
+    if record_files:
+        hep_record['_fft'] = _get_updated_fft_fields(
+            current_fft_fields=hep_record['_fft'],
+            record_files=record_files,
+        )
 
     return hep_record
 

hepcrawl/spiders/alpha_spider.py

@@ -149,8 +149,8 @@ def parse(self, response):
             record.add_value('collections', ['HEP', 'THESIS'])
 
             parsed_item = ParsedItem(
-                item=record.load_item(),
-                item_format='hepcrawl',
+                record=record.load_item(),
+                record_format='hepcrawl',
             )
 
             yield parsed_item

hepcrawl/spiders/aps_spider.py

@@ -117,8 +117,8 @@ def parse(self, response):
             record.add_value('collections', ['HEP', 'Citeable', 'Published'])
 
             parsed_item = ParsedItem(
-                item=record.load_item(),
-                item_format='hepcrawl',
+                record=record.load_item(),
+                record_format='hepcrawl',
             )
 
             yield parsed_item

hepcrawl/spiders/arxiv_spider.py

@@ -116,8 +116,8 @@ def parse_node(self, response, node):
         record.add_value('license', license)
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/spiders/base_spider.py

@@ -199,8 +199,8 @@ def build_item(self, response):
         record.add_value('collections', ['HEP', 'THESIS'])
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/spiders/brown_spider.py

@@ -225,8 +225,8 @@ def build_item(self, response):
         record.add_value('collections', ['HEP', 'THESIS'])
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/spiders/dnb_spider.py

@@ -226,8 +226,8 @@ def build_item(self, response):
         record.add_value('collections', ['HEP', 'THESIS'])
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/spiders/edp_spider.py

@@ -320,8 +320,8 @@ def build_item_rich(self, response):
         record.add_value("urls", response.meta.get("urls"))
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item
@@ -395,8 +395,8 @@ def build_item_jats(self, response):
         record.add_value("references", references)
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/spiders/elsevier_spider.py

@@ -1036,8 +1036,8 @@ def build_item(self, response):
         record.add_value('references', self.get_references(node))
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/spiders/hindawi_spider.py

@@ -226,8 +226,8 @@ def parse_node(self, response, node):
                          "./datafield[@tag='260']/subfield[@code='b']/text()")
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/spiders/infn_spider.py

@@ -243,8 +243,8 @@ def build_item(self, response):
         record.add_value('collections', ['HEP', 'THESIS'])
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/spiders/iop_spider.py

@@ -224,8 +224,8 @@ def parse_node(self, response, node):
                                  self.add_fft_file(pdf_file_path, file_access, file_type))
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/spiders/magic_spider.py

@@ -180,8 +180,8 @@ def build_item(self, response):
         record.add_value('collections', ['HEP', 'THESIS'])
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         yield parsed_item

hepcrawl/spiders/mit_spider.py

@@ -228,8 +228,8 @@ def build_item(self, response):
         record.add_value('collections', ['HEP', 'THESIS'])
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/spiders/phenix_spider.py

@@ -130,8 +130,8 @@ def parse_node(self, response, node):
         record.add_value('collections', ['HEP', 'THESIS'])
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/spiders/phil_spider.py

@@ -165,8 +165,8 @@ def build_item(self, response):
                 record.add_value('journal_year', int(jsonrecord['year']))
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/spiders/pos_spider.py

@@ -136,8 +136,8 @@ def build_item(self, response):
         record.add_value('collections', ['HEP', 'ConferencePaper'])
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/spiders/t2k_spider.py

@@ -168,8 +168,8 @@ def build_item(self, response):
         record.add_value('collections', ['HEP', 'THESIS'])
 
         parsed_item = ParsedItem(
-            item=record.load_item(),
-            item_format='hepcrawl',
+            record=record.load_item(),
+            record_format='hepcrawl',
         )
 
         yield parsed_item

hepcrawl/spiders/wsp_spider.py

@@ -72,7 +72,15 @@ class WorldScientificSpider(Jats, XMLFeedSpider):
         'rapid-communications'
     ]
 
-    def __init__(self, package_path=None, ftp_folder="/WSP", ftp_host=None, ftp_netrc=None, *args, **kwargs):
+    def __init__(
+        self,
+        package_path=None,
+        ftp_folder="/WSP",
+        ftp_host=None,
+        ftp_netrc=None,
+        *args,
+        **kwargs
+    ):
         """Construct WSP spider."""
         super(WorldScientificSpider, self).__init__(*args, **kwargs)
         self.ftp_folder = ftp_folder
@@ -206,8 +214,8 @@ def parse_node(self, response, node):
         record.add_value('collections', self._get_collections(node, article_type, journal_title))
 
         parsed_item = ParsedItem(
-            item=dict(record.load_item()),
-            item_format='hepcrawl',
+            record=dict(record.load_item()),
+            record_format='hepcrawl',
         )
 
         return parsed_item

hepcrawl/utils.py

@@ -348,15 +348,10 @@ def get_license_by_text(license_text):
     return license
 
 
-def get_absolute_file_path(file_path):
-    """Returns the absolute path of a relative path."""
-    return os.path.abspath(file_path)
-
-
 class RecordFile(object):
     """Metadata of a file needed for a record.
 
-    Params:
+    Args:
         path(str): local path to the file.
         name(str): Optional, name of the file, if not passed, will use the name in the path.
 
@@ -377,14 +372,14 @@ def __init__(self, path, name=None):
 class ParsedItem(dict):
     """Each of the individual items returned by the spider to the pipeline.
 
-    Params:
+    Args:
         record(dict): Information about the crawled record, might be in different formats.
         record_format(str): Format of the above record, for example ``"hep"`` or ``"hepcrawl"``.
         file_urls(list(str)): URLs to the files to be downloaded by ``FftFilesPipeline``.
-        ftp_params(dict): Parameter for the ``FftFilesPipeline`` to be able to connect to the ftp
-            server, if any.
+        ftp_params(dict): Parameter for the ``FftFilesPipeline`` to be able to connect to the
+         ftp server, if any.
         record_files(list(RecordFile)): files attached to the record, usually populated by
-            ``FftFilesPipeline`` from the ``file_urls`` parameter.
+        ``FftFilesPipeline`` from the ``file_urls`` parameter.
     """
     def __init__(
             self,

tests/functional/desy/fixtures/desy_local_records.json

@@ -13,7 +13,7 @@
       "version": 1,
       "creation_datetime": "2017-06-27T09:43:17", "description": "00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \\textsc{core}NFW models. Colors and symbols are as in Figure \\ref{fig:dc14_fits}.",
       "format": ".txt",
-      "path": "/tmp/file_urls/full/49e42fc70c5d7b0cd9dc7aa5defa12ded530e135.txt",
+      "path": "/tmp/file_urls/full/796483eeaa779dfc00871228dd70dc9809ebc3c0.txt",
       "type": "Main",
       "filename": "cNFW_rogue_curves"
     },
@@ -22,7 +22,7 @@
       "creation_datetime": "2017-06-27T09:43:16",
       "description": "00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \\cite{dutton14} (left) and the stellar mass-halo mass relation from \\cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \\cite{maccio08} and the stellar mass-halo mass relation from \\cite{behroozi13} are also shown as the black dashed lines.",
       "format": ".txt",
-      "path": "/tmp/file_urls/full/c1cdb1640202896b1ffc446f20d0d660977fc2db.txt",
+      "path": "/tmp/file_urls/full/ff1ccb47d9a3abb75acb91279e0ec2a4b530ba3e.txt",
       "type": "Main",
       "filename": "scalingRelations_DutBeh_DC14_all_Oh"
     }
@@ -90,7 +90,7 @@
       "version": 1,
       "creation_datetime": "2017-06-27T09:43:17", "description": "00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \\textsc{core}NFW models. Colors and symbols are as in Figure \\ref{fig:dc14_fits}.",
       "format": ".txt",
-      "path": "/tmp/file_urls/full/49e42fc70c5d7b0cd9dc7aa5defa12ded530e135.txt",
+      "path": "/tmp/file_urls/full/796483eeaa779dfc00871228dd70dc9809ebc3c0.txt",
       "type": "Main",
       "filename": "cNFW_rogue_curves"
     },
@@ -99,7 +99,7 @@
       "creation_datetime": "2017-06-27T09:43:16",
       "description": "00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \\cite{dutton14} (left) and the stellar mass-halo mass relation from \\cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \\cite{maccio08} and the stellar mass-halo mass relation from \\cite{behroozi13} are also shown as the black dashed lines.",
       "format": ".txt",
-      "path": "/tmp/file_urls/full/c1cdb1640202896b1ffc446f20d0d660977fc2db.txt",
+      "path": "/tmp/file_urls/full/ff1ccb47d9a3abb75acb91279e0ec2a4b530ba3e.txt",
       "type": "Main",
       "filename": "scalingRelations_DutBeh_DC14_all_Oh"
     }
@@ -167,7 +167,7 @@
       "version": 1,
       "creation_datetime": "2017-06-27T09:43:17", "description": "00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \\textsc{core}NFW models. Colors and symbols are as in Figure \\ref{fig:dc14_fits}.",
       "format": ".txt",
-      "path": "/tmp/file_urls/full/49e42fc70c5d7b0cd9dc7aa5defa12ded530e135.txt",
+      "path": "/tmp/file_urls/full/796483eeaa779dfc00871228dd70dc9809ebc3c0.txt",
       "type": "Main",
       "filename": "cNFW_rogue_curves"
     },
@@ -176,7 +176,7 @@
       "creation_datetime": "2017-06-27T09:43:16",
       "description": "00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \\cite{dutton14} (left) and the stellar mass-halo mass relation from \\cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \\cite{maccio08} and the stellar mass-halo mass relation from \\cite{behroozi13} are also shown as the black dashed lines.",
       "format": ".txt",
-      "path": "/tmp/file_urls/full/c1cdb1640202896b1ffc446f20d0d660977fc2db.txt",
+      "path": "/tmp/file_urls/full/ff1ccb47d9a3abb75acb91279e0ec2a4b530ba3e.txt",
       "type": "Main",
       "filename": "scalingRelations_DutBeh_DC14_all_Oh"
     }
@@ -244,7 +244,7 @@
       "version": 1,
       "creation_datetime": "2017-06-27T09:43:17", "description": "00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \\textsc{core}NFW models. Colors and symbols are as in Figure \\ref{fig:dc14_fits}.",
       "format": ".txt",
-      "path": "/tmp/file_urls/full/49e42fc70c5d7b0cd9dc7aa5defa12ded530e135.txt",
+      "path": "/tmp/file_urls/full/796483eeaa779dfc00871228dd70dc9809ebc3c0.txt",
       "type": "Main",
       "filename": "cNFW_rogue_curves"
     },
@@ -253,7 +253,7 @@
       "creation_datetime": "2017-06-27T09:43:16",
       "description": "00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \\cite{dutton14} (left) and the stellar mass-halo mass relation from \\cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\\sigma$ and 2$\\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \\cite{maccio08} and the stellar mass-halo mass relation from \\cite{behroozi13} are also shown as the black dashed lines.",
       "format": ".txt",
-      "path": "/tmp/file_urls/full/c1cdb1640202896b1ffc446f20d0d660977fc2db.txt",
+      "path": "/tmp/file_urls/full/ff1ccb47d9a3abb75acb91279e0ec2a4b530ba3e.txt",
       "type": "Main",
       "filename": "scalingRelations_DutBeh_DC14_all_Oh"
     }

tests/functional/desy/fixtures/ftp_server/DESY/desy_collection_records.xml

@@ -51,7 +51,7 @@
             <subfield code="p">INSPIRE:HEP</subfield>
         </datafield>
         <datafield tag="FFT" ind1=" " ind2=" ">
-            <subfield code="a">DESY/FFT/test_fft_1.txt</subfield>
+            <subfield code="a">FFT/test_fft_1.txt</subfield>
             <subfield code="d">00013 Decomposition of the problematic rotation curves in our sample according to the best-fit \textsc{core}NFW models. Colors and symbols are as in Figure \ref{fig:dc14_fits}.</subfield>
             <subfield code="f">.txt</subfield>
             <subfield code="n">cNFW_rogue_curves</subfield>
@@ -62,7 +62,7 @@
             <subfield code="z"/>
         </datafield>
         <datafield tag="FFT" ind1=" " ind2=" ">
-            <subfield code="a">DESY/FFT/test_fft_2.txt</subfield>
+            <subfield code="a">FFT/test_fft_2.txt</subfield>
             <subfield code="d">00005 Comparison of the parameters of the best-fit DC14 models to the cosmological halo mass-concentration relation from \cite{dutton14} (left) and the stellar mass-halo mass relation from \cite{behroozi13} (right). The error bars correspond to the extremal values of the multidimensional 68\% confidence region for each fit. The theoretical relations are shown as red lines and their 1$\sigma$ and 2$\sigma$ scatter are represented by the dark and light grey bands, respectively. The mass-concentration relation from \cite{maccio08} and the stellar mass-halo mass relation from \cite{behroozi13} are also shown as the black dashed lines.</subfield>
             <subfield code="f">.txt</subfield>
             <subfield code="n">scalingRelations_DutBeh_DC14_all_Oh</subfield>