From 7d88abbf1a942fe79aceda0c77feb25caa149fd0 Mon Sep 17 00:00:00 2001 From: Beck Davis Date: Tue, 2 Apr 2024 15:24:50 -0400 Subject: [PATCH 1/5] Add test for stale files in the update directory path Co-authored-by: Jane Sandberg --- spec/jobs/scsb_import_full_job_spec.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/spec/jobs/scsb_import_full_job_spec.rb b/spec/jobs/scsb_import_full_job_spec.rb index fd9566636..0529c8fc7 100644 --- a/spec/jobs/scsb_import_full_job_spec.rb +++ b/spec/jobs/scsb_import_full_job_spec.rb @@ -14,4 +14,19 @@ expect(event.dump.dump_type.constant).to eq 'PARTNER_RECAP_FULL' expect(Scsb::PartnerUpdates).to have_received(:full) end + + describe 'when there stale files in the update directory path' do + let(:update_directory_path) { Rails.root.join("tmp", "specs", "update_directory") } + + before do + allow(ENV).to receive(:[]).with('SCSB_PARTNER_UPDATE_DIRECTORY').and_return(update_directory_path) + FileUtils.cp('spec/fixtures/scsb_updates/CUL_20210429_192300.zip', update_directory_path) + FileUtils.cp('spec/fixtures/scsb_updates/NYPL_20210430_015000.zip', update_directory_path) + FileUtils.cp('spec/fixtures/scsb_updates/HL_20210716_063500.zip', update_directory_path) + end + + it 'removes stale files' do + expect {one = one} + end + end end From 7f8d3d797b52aaa83b32bdbd5b972a54431ecdec Mon Sep 17 00:00:00 2001 From: Beck Davis Date: Wed, 3 Apr 2024 15:07:15 -0400 Subject: [PATCH 2/5] Gets test failing for right reason Co-authored-by: Jane Sandberg --- spec/jobs/scsb_import_full_job_spec.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/spec/jobs/scsb_import_full_job_spec.rb b/spec/jobs/scsb_import_full_job_spec.rb index 0529c8fc7..d5465f62c 100644 --- a/spec/jobs/scsb_import_full_job_spec.rb +++ b/spec/jobs/scsb_import_full_job_spec.rb @@ -26,7 +26,12 @@ end it 'removes stale files' do - expect {one = one} + + expect { described_class.perform_now }.to change { Event.count }.by(1) + + expect(File.file?(Rails.root.join("tmp", "specs", "update_directory", 'CUL_20210429_192300.zip'))).to be false + expect(File.file?(Rails.root.join("tmp", "specs", "update_directory", 'NYPL_20210430_015000.zip'))).to be false + expect(File.file?(Rails.root.join("tmp", "specs", "update_directory", 'HL_20210716_063500.zip'))).to be false end end end From 32689e48568f686c34433672cf550a8b9ca6640e Mon Sep 17 00:00:00 2001 From: Jane Sandberg Date: Wed, 3 Apr 2024 12:20:50 -0700 Subject: [PATCH 3/5] Delete stale files in the SCSB_PARTNER_UPDATE_DIRECTORY Co-authored-by: Beck Davis --- app/jobs/scsb_import_full_job.rb | 1 + spec/jobs/scsb_import_full_job_spec.rb | 2 ++ 2 files changed, 3 insertions(+) diff --git a/app/jobs/scsb_import_full_job.rb b/app/jobs/scsb_import_full_job.rb index b050655de..00107d88f 100644 --- a/app/jobs/scsb_import_full_job.rb +++ b/app/jobs/scsb_import_full_job.rb @@ -1,5 +1,6 @@ class ScsbImportFullJob < ApplicationJob def perform + FileUtils.rm Dir.glob("#{ENV['SCSB_PARTNER_UPDATE_DIRECTORY']}/*.zip") Event.record do |event| event.dump = created_dump event.save! diff --git a/spec/jobs/scsb_import_full_job_spec.rb b/spec/jobs/scsb_import_full_job_spec.rb index d5465f62c..3aa7ea9f9 100644 --- a/spec/jobs/scsb_import_full_job_spec.rb +++ b/spec/jobs/scsb_import_full_job_spec.rb @@ -19,6 +19,7 @@ let(:update_directory_path) { Rails.root.join("tmp", "specs", "update_directory") } before do + allow(ENV).to receive(:[]).and_call_original allow(ENV).to receive(:[]).with('SCSB_PARTNER_UPDATE_DIRECTORY').and_return(update_directory_path) FileUtils.cp('spec/fixtures/scsb_updates/CUL_20210429_192300.zip', update_directory_path) FileUtils.cp('spec/fixtures/scsb_updates/NYPL_20210430_015000.zip', update_directory_path) @@ -32,6 +33,7 @@ expect(File.file?(Rails.root.join("tmp", "specs", "update_directory", 'CUL_20210429_192300.zip'))).to be false expect(File.file?(Rails.root.join("tmp", "specs", "update_directory", 'NYPL_20210430_015000.zip'))).to be false expect(File.file?(Rails.root.join("tmp", "specs", "update_directory", 'HL_20210716_063500.zip'))).to be false + expect(Dir.exist?(Rails.root.join("tmp", "specs", "update_directory"))).to be true end end end From 435d7ef293f432fe2ac2d5d52679fd24acb6a585 Mon Sep 17 00:00:00 2001 From: Beck Davis Date: Fri, 5 Apr 2024 13:23:59 -0400 Subject: [PATCH 4/5] Add test for removing stale xml files Co-authored-by: Jane Sandberg --- app/jobs/scsb_import_full_job.rb | 8 +++++++- .../scsb_updates/scsb_update_20240108_183400_1.xml | 6 ++++++ spec/jobs/scsb_import_full_job_spec.rb | 7 ++++--- 3 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 spec/fixtures/scsb_updates/scsb_update_20240108_183400_1.xml diff --git a/app/jobs/scsb_import_full_job.rb b/app/jobs/scsb_import_full_job.rb index 00107d88f..8630e5e5c 100644 --- a/app/jobs/scsb_import_full_job.rb +++ b/app/jobs/scsb_import_full_job.rb @@ -1,6 +1,7 @@ class ScsbImportFullJob < ApplicationJob def perform - FileUtils.rm Dir.glob("#{ENV['SCSB_PARTNER_UPDATE_DIRECTORY']}/*.zip") + delete_stale_files + Event.record do |event| event.dump = created_dump event.save! @@ -18,4 +19,9 @@ def dump_type def created_dump Dump.create!(dump_type:) end + + def delete_stale_files + FileUtils.rm Dir.glob("#{ENV['SCSB_PARTNER_UPDATE_DIRECTORY']}/*.zip") + FileUtils.rm Dir.glob("#{ENV['SCSB_PARTNER_UPDATE_DIRECTORY']}/*.xml") + end end diff --git a/spec/fixtures/scsb_updates/scsb_update_20240108_183400_1.xml b/spec/fixtures/scsb_updates/scsb_update_20240108_183400_1.xml new file mode 100644 index 000000000..461004d27 --- /dev/null +++ b/spec/fixtures/scsb_updates/scsb_update_20240108_183400_1.xml @@ -0,0 +1,6 @@ + + +02152cam a2200493 i 4500SCSB-2170814NNC20240105113043.0810220s1976 txu 000 0beng 39998 76620048 (OCoLC)ocm03111350DLCengDLCOCLOCLCQBAKERBTCTAYDXCPOCLCGNLEHEBISOCLCOOCLCQOCLCFOCLCOFC@OCLCANLCTYCOCLCQOCLUKMGBOCLCQOCLCOOCLNIUGZNOCLCOOCLCL781538947can087959022X9780879590222(OCoLC)3111350PR5366.L38 1976822/.9/1218Bcci1icclaccZCUALaurence, Dan H.Shaw, books, and libraries /Dan H. Laurence.[Austin] :Humanities Research Center, University of Texas at Austin,©1976.28 pages ;23 cmtexttxtrdacontentunmediatednrdamediavolumencrdacarrierBibliographical monograph series ;no. 9"Five hundred copies have been printed on Curtis Rag. The type is Intertype Baskerville. Design by William R. Holman"--Colophon.Shaw, Bernard,1856-1950Books and reading.Dramatists, Irish20th centuryBiography.Dramatists, English20th centuryBiography.Shaw, Bernard,1856-1950fasthttps://id.oclc.org/worldcat/entity/E39PBJfmqrHfjT8qvwDyYWVKVC(OCoLC)fst00030129Dramatists, Englishfast(OCoLC)fst00897581Dramatists, Irishfast(OCoLC)fst008976081900-1999fastAmerican cinema filmsShaw, Bernard1856-1950Biographiesfast(OCoLC)fst01919896Holman, William Roger,designer.Online version:Laurence, Dan H.Shaw, books, and libraries.[Austin] : Humanities Research Center, University of Texas at Austin, ©1976(OCoLC)560755727Bibliographical monograph ;no. 9.PR5366 .L38 19762137624scsbcul21376243304030AvailableCU553988631SharedCURECAP + \ No newline at end of file diff --git a/spec/jobs/scsb_import_full_job_spec.rb b/spec/jobs/scsb_import_full_job_spec.rb index 3aa7ea9f9..bcaf3c4a4 100644 --- a/spec/jobs/scsb_import_full_job_spec.rb +++ b/spec/jobs/scsb_import_full_job_spec.rb @@ -24,16 +24,17 @@ FileUtils.cp('spec/fixtures/scsb_updates/CUL_20210429_192300.zip', update_directory_path) FileUtils.cp('spec/fixtures/scsb_updates/NYPL_20210430_015000.zip', update_directory_path) FileUtils.cp('spec/fixtures/scsb_updates/HL_20210716_063500.zip', update_directory_path) + FileUtils.cp('spec/fixtures/scsb_updates/scsb_update_20240108_183400_1.xml', update_directory_path) end it 'removes stale files' do - expect { described_class.perform_now }.to change { Event.count }.by(1) - + expect(File.file?(Rails.root.join("tmp", "specs", "update_directory", 'CUL_20210429_192300.zip'))).to be false expect(File.file?(Rails.root.join("tmp", "specs", "update_directory", 'NYPL_20210430_015000.zip'))).to be false expect(File.file?(Rails.root.join("tmp", "specs", "update_directory", 'HL_20210716_063500.zip'))).to be false + expect(File.file?(Rails.root.join("tmp", "specs", "update_directory", 'scsb_update_20240108_183400_1.xml'))).to be false expect(Dir.exist?(Rails.root.join("tmp", "specs", "update_directory"))).to be true end - end + end end From 10f36c6cc65c576c4c1540ef4fcceb7e9fcbb7ce Mon Sep 17 00:00:00 2001 From: Beck Davis Date: Fri, 5 Apr 2024 13:46:47 -0400 Subject: [PATCH 5/5] Add line to make directory for the test Co-authored-by: Jane Sandberg --- spec/jobs/scsb_import_full_job_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/jobs/scsb_import_full_job_spec.rb b/spec/jobs/scsb_import_full_job_spec.rb index bcaf3c4a4..7294250f3 100644 --- a/spec/jobs/scsb_import_full_job_spec.rb +++ b/spec/jobs/scsb_import_full_job_spec.rb @@ -19,6 +19,7 @@ let(:update_directory_path) { Rails.root.join("tmp", "specs", "update_directory") } before do + FileUtils.mkdir_p(update_directory_path) allow(ENV).to receive(:[]).and_call_original allow(ENV).to receive(:[]).with('SCSB_PARTNER_UPDATE_DIRECTORY').and_return(update_directory_path) FileUtils.cp('spec/fixtures/scsb_updates/CUL_20210429_192300.zip', update_directory_path)