diff --git a/python/nistoar/pdr/preserv/service/siphandler.py b/python/nistoar/pdr/preserv/service/siphandler.py index 7c418df9e..5b2c6638a 100644 --- a/python/nistoar/pdr/preserv/service/siphandler.py +++ b/python/nistoar/pdr/preserv/service/siphandler.py @@ -20,6 +20,7 @@ from .. import PreservationException, sys as _sys from . import status from ...ingest.rmm import IngestClient +from ...utils import write_json log = logging.getLogger(_sys.system_abbrev).getChild(_sys.subsystem_abbrev) @@ -460,10 +461,11 @@ def bagit(self, serialtype=None, destdir=None, params=None): self.bagger.bagbldr._unset_logfile() # disengage the internal log # Stage the full NERDm record for ingest into the RMM + bag = NISTBag(self.bagger.bagdir) + nerdm = bag.nerdm_record() if self._ingester: try: - bag = NISTBag(self.bagger.bagdir) - self._ingester.stage(bag.nerdm_record(), self.bagger.name) + self._ingester.stage(nerdm, self.bagger.name) except Exception as ex: msg = "Failure staging NERDm record for " + self.bagger.name + \ " for ingest: " + str(ex) @@ -567,6 +569,18 @@ def bagit(self, serialtype=None, destdir=None, params=None): log.warn("Failed to clean up the metadata bag lock file: "+ mdbag + ".lock: "+str(ex)) + # cache the latest nerdm record under the staging directory + try: + mdcache = os.path.join(self.stagedir, '_nerd') + staged = os.path.join(mdcache, self.bagger.name+".json") + if os.path.isdir(mdcache): + write_json(nerdm, staged) + except Exception as ex: + log.error("Failed to cache the new NERDm record: "+str(ex)) + if os.path.exists(staged): + # remove the old record as it is now out of date + os.remove(staged) + self.set_state(status.SUCCESSFUL) # submit NERDm record to ingest service diff --git a/python/tests/nistoar/pdr/preserv/service/test_siphandler.py b/python/tests/nistoar/pdr/preserv/service/test_siphandler.py index 1d871990c..d34201229 100644 --- a/python/tests/nistoar/pdr/preserv/service/test_siphandler.py +++ b/python/tests/nistoar/pdr/preserv/service/test_siphandler.py @@ -1,4 +1,4 @@ -import os, pdb, sys, logging, yaml +import os, pdb, sys, logging, yaml, stat import unittest as test from nistoar.testing import * @@ -147,6 +147,11 @@ def test_bagit(self): self.assertEqual(len(staged), 1) self.assertTrue(os.path.basename(staged[0]).endswith("-0.zip")) + # we don't have a nerdm staging area, so we shouldn't a cached nerdm file + # under staging area + staged = os.path.join(self.stagedir,'_nerd',self.midasid+".json") + self.assertFalse(os.path.exists(staged)) + # has the metadata bag been cleaned up? mdbagdir = os.path.join(self.sip.mdbagdir, self.midasid) self.assertFalse( os.path.exists(mdbagdir), @@ -157,7 +162,62 @@ def test_bagit(self): bb = self.midasid+".1_0_0.mbag0_4-" bfs = [f for f in os.listdir(pd) if f.startswith(bb)] self.assertEqual(len(bfs), 0) - + + def test_bagit_withnerdstaging(self): + mdcache = os.path.join(self.stagedir, "_nerd") + if not os.path.exists(mdcache): + os.mkdir(mdcache) + + self.assertEqual(self.sip.state, status.FORGOTTEN) + self.assertEqual(len(os.listdir(self.sip.stagedir)), 1) + self.sip.bagit() + self.assertTrue(os.path.exists(os.path.join(self.store, + self.midasid+".1_0_0.mbag0_4-0.zip"))) + + # do we have a cached nerdm file under staging area? + staged = os.path.join(self.stagedir,'_nerd',self.midasid+".json") + self.assertTrue(os.path.exists(staged)) + + def test_bagit_nerdstagingfail(self): + mdcache = os.path.join(self.stagedir, "_nerd") + if not os.path.exists(mdcache): + os.mkdir(mdcache) + staged = os.path.join(self.stagedir,'_nerd',self.midasid+".json") + with open(staged, 'w') as fd: + pass + + self.assertEqual(self.sip.state, status.FORGOTTEN) + self.assertEqual(len(os.listdir(self.sip.stagedir)), 1) + try: + os.chmod(staged, stat.S_IREAD) + os.chmod(mdcache, stat.S_IREAD|stat.S_IXUSR) + with self.assertRaises(OSError): + self.sip.bagit() + finally: + os.chmod(mdcache, stat.S_IREAD|stat.S_IWRITE|stat.S_IXUSR) + os.chmod(staged, stat.S_IREAD|stat.S_IWRITE|stat.S_IROTH|stat.S_IWOTH) + + + def test_bagit_nerdstagingclean(self): + mdcache = os.path.join(self.stagedir, "_nerd") + if not os.path.exists(mdcache): + os.mkdir(mdcache) + staged = os.path.join(mdcache, self.midasid+".json") + with open(staged, 'w') as fd: + pass + self.assertTrue(os.path.exists(staged)) + + self.assertEqual(self.sip.state, status.FORGOTTEN) + self.assertEqual(len(os.listdir(self.sip.stagedir)), 1) + try: + os.chmod(staged, stat.S_IREAD) + self.sip.bagit() + + # do we have a cached nerdm file under staging area? + self.assertFalse(os.path.exists(staged)) + finally: + if os.path.exists(staged): + os.chmod(staged,stat.S_IREAD|stat.S_IWRITE|stat.S_IROTH|stat.S_IWOTH) def test_bagit_nooverwrite(self): self.assertEqual(self.sip.state, status.FORGOTTEN)