From 3316f89e2e3878b3c4343102b0cfb2f1f40d8f32 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 3 Jan 2024 12:33:48 -0800
Subject: [PATCH 01/31] queue derivative jobs in new :resource_intensive queue

This sets us up for future improvements to derivative job processing
---
 app/jobs/create_derivatives_job_decorator.rb | 11 +++++++++++
 config/sidekiq.yml                           |  1 +
 2 files changed, 12 insertions(+)
 create mode 100644 app/jobs/create_derivatives_job_decorator.rb

diff --git a/app/jobs/create_derivatives_job_decorator.rb b/app/jobs/create_derivatives_job_decorator.rb
new file mode 100644
index 000000000..fd7dbf0d9
--- /dev/null
+++ b/app/jobs/create_derivatives_job_decorator.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module CreateDerivativesJobDecorator
+  def self.prepended(base)
+    base.class_eval do
+      queue_as :resource_intensive
+    end
+  end
+end
+
+CreateDerivativesJob.prepend(CreateDerivativesJobDecorator)
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index a72d01223..fb2531c82 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -3,3 +3,4 @@
   - default
   - import
   - export
+  - resource_intensive

From ab95c9ac60a7b9adb5cfd6060ef5bf4c68ff43e3 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 3 Jan 2024 16:00:34 -0800
Subject: [PATCH 02/31] attempt to only put video and audio jobs in separate
 queue

This doesn't work. This commit is only here to document that this
solution was attempted.
---
 app/jobs/create_derivatives_job_decorator.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/jobs/create_derivatives_job_decorator.rb b/app/jobs/create_derivatives_job_decorator.rb
index fd7dbf0d9..8b9de47a4 100644
--- a/app/jobs/create_derivatives_job_decorator.rb
+++ b/app/jobs/create_derivatives_job_decorator.rb
@@ -3,9 +3,13 @@
 module CreateDerivativesJobDecorator
   def self.prepended(base)
     base.class_eval do
-      queue_as :resource_intensive
+      before_perform :reassign_queue, if: ->() { arguments.first.video? || arguments.first.audio? }
     end
   end
+
+  def reassign_queue
+    self.queue_name = 'resource_intensive'
+  end
 end
 
 CreateDerivativesJob.prepend(CreateDerivativesJobDecorator)

From cceb54444a5e387d4b9de7ad653d372d4c1daf51 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 3 Jan 2024 19:07:58 -0800
Subject: [PATCH 03/31] create child job class to use new Sidekiq queue

There is no way to reassign a job to another queue once it has been
created. To use the new :resource_intensive Sidekiq queue, we need a
separate job class that is configured to use that queue.

Enter CreateLargeDerivativesJob.

It functions exactly the same as CreateDerivativesJob, except that it
queues into the :resource_intensive queue and has a higher priority.
This way, we can have two separate job classes with different Sidekiq
configurations while using the same application logic for both.
---
 app/jobs/create_derivatives_job_decorator.rb  | 12 ++-
 app/jobs/create_large_derivatives_job.rb      |  6 ++
 spec/jobs/create_derivatives_job_spec.rb      | 79 +++++++++++++++++++
 .../jobs/create_large_derivatives_job_spec.rb | 41 ++++++++++
 4 files changed, 131 insertions(+), 7 deletions(-)
 create mode 100644 app/jobs/create_large_derivatives_job.rb
 create mode 100644 spec/jobs/create_derivatives_job_spec.rb
 create mode 100644 spec/jobs/create_large_derivatives_job_spec.rb

diff --git a/app/jobs/create_derivatives_job_decorator.rb b/app/jobs/create_derivatives_job_decorator.rb
index 8b9de47a4..2528a70c9 100644
--- a/app/jobs/create_derivatives_job_decorator.rb
+++ b/app/jobs/create_derivatives_job_decorator.rb
@@ -1,14 +1,12 @@
 # frozen_string_literal: true
 
 module CreateDerivativesJobDecorator
-  def self.prepended(base)
-    base.class_eval do
-      before_perform :reassign_queue, if: ->() { arguments.first.video? || arguments.first.audio? }
-    end
-  end
+  def perform(file_set, file_id, filepath = nil)
+    return super if self.is_a?(CreateLargeDerivativesJob)
+    return super unless file_set.video? || file_set.audio?
 
-  def reassign_queue
-    self.queue_name = 'resource_intensive'
+    CreateLargeDerivativesJob.perform_later(*self.arguments)
+    true
   end
 end
 
diff --git a/app/jobs/create_large_derivatives_job.rb b/app/jobs/create_large_derivatives_job.rb
new file mode 100644
index 000000000..b75c8d369
--- /dev/null
+++ b/app/jobs/create_large_derivatives_job.rb
@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+class CreateLargeDerivativesJob < CreateDerivativesJob
+  queue_as :resource_intensive
+  queue_with_priority 20 # TODO: necessary?
+end
diff --git a/spec/jobs/create_derivatives_job_spec.rb b/spec/jobs/create_derivatives_job_spec.rb
new file mode 100644
index 000000000..d2ff943b4
--- /dev/null
+++ b/spec/jobs/create_derivatives_job_spec.rb
@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+RSpec.describe CreateDerivativesJob do
+  around do |example|
+    ffmpeg_enabled = Hyrax.config.enable_ffmpeg
+    Hyrax.config.enable_ffmpeg = true
+    example.run
+    Hyrax.config.enable_ffmpeg = ffmpeg_enabled
+  end
+
+  describe 'recreating self as a CreateLargeDerivativesJob' do
+    let(:id)       { '123' }
+    let(:file_set) { FileSet.new }
+    let(:file) do
+      Hydra::PCDM::File.new.tap do |f|
+        f.content = 'foo'
+        f.original_name = filename
+        f.save!
+      end
+    end
+
+    before do
+      allow(FileSet).to receive(:find).with(id).and_return(file_set)
+      allow(file_set).to receive(:mime_type).and_return(mime_type)
+      allow(file_set).to receive(:id).and_return(id)
+      # Short-circuit irrelevant logic
+      allow(file_set).to receive(:reload)
+      allow(file_set).to receive(:update_index)
+    end
+
+    context 'with an image file' do
+      let(:mime_type) { 'image/jpeg' }
+      let(:filename) { 'picture.jpg' }
+
+      before do
+        # Short-circuit irrelevant logic
+        allow(Hydra::Derivatives::ImageDerivatives).to receive(:create)
+      end
+
+      it 'does not recreate as a CreateLargeDerivativesJob' do
+        expect(CreateLargeDerivativesJob).not_to receive(:perform_later)
+
+        described_class.perform_now(file_set, file.id)
+      end
+    end
+
+    context 'with an video file' do
+      let(:mime_type) { 'video/mp4' }
+      let(:filename) { 'video.mp4' }
+
+      before do
+        # Short-circuit irrelevant logic
+        allow(Hydra::Derivatives::VideoDerivatives).to receive(:create)
+      end
+
+      it 'recreates as a CreateLargeDerivativesJob' do
+        expect(CreateLargeDerivativesJob).to receive(:perform_later)
+
+        described_class.perform_now(file_set, file.id)
+      end
+    end
+
+    context 'with an audio file' do
+      let(:mime_type) { 'audio/x-wav' }
+      let(:filename) { 'audio.wav' }
+
+      before do
+        # Short-circuit irrelevant logic
+        allow(Hydra::Derivatives::AudioDerivatives).to receive(:create)
+      end
+
+      it 'recreates as a CreateLargeDerivativesJob' do
+        expect(CreateLargeDerivativesJob).to receive(:perform_later)
+
+        described_class.perform_now(file_set, file.id)
+      end
+    end
+  end
+end
diff --git a/spec/jobs/create_large_derivatives_job_spec.rb b/spec/jobs/create_large_derivatives_job_spec.rb
new file mode 100644
index 000000000..b39f6d82a
--- /dev/null
+++ b/spec/jobs/create_large_derivatives_job_spec.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+RSpec.describe CreateLargeDerivativesJob, type: :job do
+  let(:id)       { '123' }
+  let(:file_set) { FileSet.new }
+  let(:file) do
+    Hydra::PCDM::File.new.tap do |f|
+      f.content = 'foo'
+      f.original_name = 'video.mp4'
+      f.save!
+    end
+  end
+
+  before do
+    allow(FileSet).to receive(:find).with(id).and_return(file_set)
+    allow(file_set).to receive(:id).and_return(id)
+    # Short-circuit irrelevant logic
+    allow(file_set).to receive(:reload)
+    allow(file_set).to receive(:update_index)
+  end
+
+  it 'runs in the :resource_intensive queue' do
+    expect { described_class.perform_later(file_set, file.id) }
+      .to have_enqueued_job(described_class)
+      .on_queue('resource_intensive')
+  end
+
+  # @see CreateDerivativesJobDecorator#perform
+  it "doesn't schedule itself infinitly" do
+    expect(described_class).not_to receive(:perform_later)
+
+    described_class.perform_now(file_set, file.id)
+  end
+
+  it 'successfully calls the logic in CreateDerivativesJob' do
+    allow(file_set).to receive(:mime_type).and_return('video/mp4')
+    expect(Hydra::Derivatives::VideoDerivatives).to receive(:create)
+
+    described_class.perform_now(file_set, file.id)
+  end
+end

From acb09ad653adcb94da04afd770d952fcafc2b607 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 4 Jan 2024 10:03:21 -0800
Subject: [PATCH 04/31] rubocop fixes in CreateDerivativesJobDecorator

---
 app/jobs/create_derivatives_job_decorator.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/jobs/create_derivatives_job_decorator.rb b/app/jobs/create_derivatives_job_decorator.rb
index 2528a70c9..75aba75a3 100644
--- a/app/jobs/create_derivatives_job_decorator.rb
+++ b/app/jobs/create_derivatives_job_decorator.rb
@@ -2,10 +2,10 @@
 
 module CreateDerivativesJobDecorator
   def perform(file_set, file_id, filepath = nil)
-    return super if self.is_a?(CreateLargeDerivativesJob)
+    return super if is_a?(CreateLargeDerivativesJob)
     return super unless file_set.video? || file_set.audio?
 
-    CreateLargeDerivativesJob.perform_later(*self.arguments)
+    CreateLargeDerivativesJob.perform_later(*arguments)
     true
   end
 end

From 70bdf486546b28db5151b7220166103030d84c32 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 4 Jan 2024 10:06:08 -0800
Subject: [PATCH 05/31] remove CreateLargeDerivativesJob priority override

At present, this is the only job going into this queue, so specifying a
higher priority doesn't make sense since all jobs in the queue will have
the same priority regardless
---
 app/jobs/create_large_derivatives_job.rb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/jobs/create_large_derivatives_job.rb b/app/jobs/create_large_derivatives_job.rb
index b75c8d369..47b092c47 100644
--- a/app/jobs/create_large_derivatives_job.rb
+++ b/app/jobs/create_large_derivatives_job.rb
@@ -2,5 +2,4 @@
 
 class CreateLargeDerivativesJob < CreateDerivativesJob
   queue_as :resource_intensive
-  queue_with_priority 20 # TODO: necessary?
 end

From debb2f1cd452ab188b2ce98b9a19c047d8c47e3f Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 4 Jan 2024 12:25:56 -0800
Subject: [PATCH 06/31] new worker to run resource intensive jobs

---
 config/sidekiq.yml                    |  1 -
 config/sidekiq_resource_intensive.yml |  7 +++
 docker-compose.production.yml         | 43 ++++++++++--------
 docker-compose.yml                    | 63 +++++++++++++++------------
 4 files changed, 69 insertions(+), 45 deletions(-)
 create mode 100644 config/sidekiq_resource_intensive.yml

diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index fb2531c82..a72d01223 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -3,4 +3,3 @@
   - default
   - import
   - export
-  - resource_intensive
diff --git a/config/sidekiq_resource_intensive.yml b/config/sidekiq_resource_intensive.yml
new file mode 100644
index 000000000..9865178ef
--- /dev/null
+++ b/config/sidekiq_resource_intensive.yml
@@ -0,0 +1,7 @@
+---
+:concurrency: 1
+:queues:
+  - default
+  - import
+  - export
+  - resource_intensive
diff --git a/docker-compose.production.yml b/docker-compose.production.yml
index 8412c2ad5..e0b369661 100644
--- a/docker-compose.production.yml
+++ b/docker-compose.production.yml
@@ -14,6 +14,25 @@ x-app: &app
   networks:
     internal:
 
+x-app-worker: &app-worker
+  <<: *app
+  image: ghcr.io/samvera/hyku/worker:${TAG:-latest}
+  depends_on:
+    check_volumes:
+      condition: service_completed_successfully
+    initialize_app:
+      condition: service_completed_successfully
+    db:
+      condition: service_started
+    solr:
+      condition: service_started
+    fcrepo:
+      condition: service_started
+    redis:
+      condition: service_started
+    zoo:
+      condition: service_started
+
 volumes:
   assets:
   cache:
@@ -133,29 +152,19 @@ services:
         condition: service_started
       worker:
         condition: service_started
+      worker_resource_intensive:
+        condition: service_started
       initialize_app:
         condition: service_completed_successfully
     expose:
       - 3000
 
   worker:
-    <<: *app
-    image: ghcr.io/samvera/hyku/worker:${TAG:-latest}
-    depends_on:
-      check_volumes:
-        condition: service_completed_successfully
-      initialize_app:
-        condition: service_completed_successfully
-      db:
-        condition: service_started
-      solr:
-        condition: service_started
-      fcrepo:
-        condition: service_started
-      redis:
-        condition: service_started
-      zoo:
-        condition: service_started
+    <<: *app-worker
+
+  worker_resource_intensive:
+    <<: *app-worker
+    command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq_resource_intensive.yml'
 
   # Do not recurse through all of tmp. derivitives will make booting
   # very slow and eventually just time out as data grows
diff --git a/docker-compose.yml b/docker-compose.yml
index 900b85592..f79d8daa0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -20,6 +20,35 @@ x-app: &app
   networks:
     internal:
 
+x-app-worker: &app-worker
+  <<: *app
+  build:
+    context: .
+    target: hyku-worker
+    args:
+      - EXTRA_APK_PACKAGES=less vim bash openjdk11-jre ffmpeg rsync exiftool
+      - HYKU_BULKRAX_ENABLED=true
+    cache_from:
+      - ghcr.io/scientist-softserv/palni-palci:${TAG:-latest}
+      - ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
+  image: ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
+  command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq.yml'
+  depends_on:
+    check_volumes:
+      condition: service_completed_successfully
+    initialize_app:
+      condition: service_completed_successfully
+    db:
+      condition: service_started
+    solr:
+      condition: service_started
+    fcrepo:
+      condition: service_started
+    redis:
+      condition: service_started
+    zoo:
+      condition: service_started
+
 volumes:
   assets:
   cache:
@@ -145,6 +174,8 @@ services:
         condition: service_started
       worker:
         condition: service_started
+      worker_resource_intensive:
+        condition: service_started
       initialize_app:
         condition: service_completed_successfully
     # ports:
@@ -153,33 +184,11 @@ services:
       - 3000
 
   worker:
-    <<: *app
-    build:
-      context: .
-      target: hyku-worker
-      args:
-        - EXTRA_APK_PACKAGES=less vim bash openjdk11-jre ffmpeg rsync exiftool
-        - HYKU_BULKRAX_ENABLED=true
-      cache_from:
-        - ghcr.io/scientist-softserv/palni-palci:${TAG:-latest}
-        - ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
-    image: ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
-    command: sh -l -c 'bundle && bundle exec sidekiq'
-    depends_on:
-      check_volumes:
-        condition: service_completed_successfully
-      initialize_app:
-        condition: service_completed_successfully
-      db:
-        condition: service_started
-      solr:
-        condition: service_started
-      fcrepo:
-        condition: service_started
-      redis:
-        condition: service_started
-      zoo:
-        condition: service_started
+    <<: *app-worker
+
+  worker_resource_intensive:
+    <<: *app-worker
+    command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq_resource_intensive.yml'
 
   # Do not recurse through all of tmp. derivitives will make booting
   # very slow and eventually just time out as data grows

From 1abe7bd69cd2d1f27f08a65399c24bffd0c8966c Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 4 Jan 2024 12:27:35 -0800
Subject: [PATCH 07/31] deploy resource intensive worker with 4x the normal
 resources

---
 ops/demo-deploy.tmpl.yaml       | 18 ++++++++++++++++++
 ops/production-deploy.tmpl.yaml | 10 ++++++++++
 2 files changed, 28 insertions(+)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 78c69c659..8b41784cb 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -189,6 +189,24 @@ worker:
     runAsGroup: 101
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
+
+worker_resource_intensive:
+  replicaCount: 1
+  resources:
+    limits:
+      memory: "16Gi"
+      cpu: "1200m"
+    requests:
+      memory: "8Gi"
+      cpu: "600m"
+  extraVolumeMounts: *volMounts
+  extraEnvVars: *envVars
+  podSecurityContext:
+    runAsUser: 1001
+    runAsGroup: 101
+    fsGroup: 101
+    fsGroupChangePolicy: "OnRootMismatch"
+
 podSecurityContext:
   runAsUser: 1001
   runAsGroup: 101
diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
index 1f3361d99..20e7caddc 100644
--- a/ops/production-deploy.tmpl.yaml
+++ b/ops/production-deploy.tmpl.yaml
@@ -199,6 +199,16 @@ worker:
       memory: "2Gi"
       cpu: "150m"
 
+worker_resource_intensive:
+  replicaCount: 1
+  resources:
+    limits:
+      memory: "16Gi"
+      cpu: "1200m"
+    requests:
+      memory: "8Gi"
+      cpu: "600m"
+
   extraVolumeMounts: *volMounts
   extraEnvVars: *envVars
   podSecurityContext:

From 23202f14ec93a66e7eb84180d21721c653b15a2a Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 4 Jan 2024 12:35:55 -0800
Subject: [PATCH 08/31] no need to specify default file location

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index f79d8daa0..2bd13f130 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -32,7 +32,7 @@ x-app-worker: &app-worker
       - ghcr.io/scientist-softserv/palni-palci:${TAG:-latest}
       - ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
   image: ghcr.io/scientist-softserv/palni-palci/worker:${TAG:-latest}
-  command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq.yml'
+  command: sh -l -c 'bundle && bundle exec sidekiq'
   depends_on:
     check_volumes:
       condition: service_completed_successfully

From 38b4f8ad0687c882916df8a6880fde57b3b89649 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 4 Jan 2024 15:05:25 -0800
Subject: [PATCH 09/31] add code docs

---
 app/jobs/create_derivatives_job_decorator.rb |  4 ++++
 app/jobs/create_large_derivatives_job.rb     | 11 +++++++++++
 2 files changed, 15 insertions(+)

diff --git a/app/jobs/create_derivatives_job_decorator.rb b/app/jobs/create_derivatives_job_decorator.rb
index 75aba75a3..88d66a3ec 100644
--- a/app/jobs/create_derivatives_job_decorator.rb
+++ b/app/jobs/create_derivatives_job_decorator.rb
@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
+# OVERRIDE Hyrax v3.6.0
+# @see CreateLargeDerivativesJob
 module CreateDerivativesJobDecorator
+  # OVERRIDE: Divert audio and video derivative
+  # creation to CreateLargeDerivativesJob.
   def perform(file_set, file_id, filepath = nil)
     return super if is_a?(CreateLargeDerivativesJob)
     return super unless file_set.video? || file_set.audio?
diff --git a/app/jobs/create_large_derivatives_job.rb b/app/jobs/create_large_derivatives_job.rb
index 47b092c47..6f678c40f 100644
--- a/app/jobs/create_large_derivatives_job.rb
+++ b/app/jobs/create_large_derivatives_job.rb
@@ -1,5 +1,16 @@
 # frozen_string_literal: true
 
+# CreateLargeDerivativesJob is intended to be used for resource-intensive derivative
+# generation (e.g. video processing). It is functionally similar to CreateDerivativesJob,
+# except that it queues jobs in the :resource_intensive queue.
+#
+# The worker responsible for processing jobs in the :resource_intensive queue should be
+# configured to have more resources dedicated to it, especially CPU. Otherwise, the
+# `ffmpeg` commands that this job class eventually triggers could be throttled.
+#
+# @see CreateDerivativesJobDecorator
+# @see Hydra::Derivatives::Processors::Ffmpeg
+# @see https://github.com/scientist-softserv/palni-palci/issues/852
 class CreateLargeDerivativesJob < CreateDerivativesJob
   queue_as :resource_intensive
 end

From 817657cd22d402a2b7d999464b7d28ecd93f16a6 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 4 Jan 2024 15:39:15 -0800
Subject: [PATCH 10/31] =?UTF-8?q?Revert=20"=F0=9F=A9=B9=20Temporary=20fix?=
 =?UTF-8?q?=20for=20video=20processing"?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit e8af16542ada7a7dc411341ca95c9fd4ed2ffdfb.
---
 .../hyrax/iiif_av/displays_content_decorator.rb | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/app/presenters/concerns/hyrax/iiif_av/displays_content_decorator.rb b/app/presenters/concerns/hyrax/iiif_av/displays_content_decorator.rb
index a2fb7347e..b249008a3 100644
--- a/app/presenters/concerns/hyrax/iiif_av/displays_content_decorator.rb
+++ b/app/presenters/concerns/hyrax/iiif_av/displays_content_decorator.rb
@@ -42,18 +42,11 @@ def video_display_content(_url, label = '')
           height = solr_document.height&.try(:to_i) || 240
           duration = conformed_duration_in_seconds
           IIIFManifest::V3::DisplayContent.new(
-            # Hyrax::IiifAv::Engine.routes.url_helpers.iiif_av_content_url(
-            #   solr_document.id,
-            #   label: label,
-            #   host: request.base_url
-            # ),
-            # TODO: This is a hack to pull the download url from hyrax as the video resource.
-            #       Ultimately we want to fix the processing times of the video derivatives so it doesn't take
-            #       hours to days to complete.  The draw back of doing it this way is that we're using the original
-            #       video file which is fine if it's already processed, but if it's a raw, then it is not ideal for
-            #       streaming purposes.  The good thing is that PALs seem to be processing the video derivatives out
-            #       of band first before ingesting so we shouldn't run into this issue.
-            Hyrax::Engine.routes.url_helpers.download_url(solr_document.id, host: request.base_url, protocol: 'https'),
+            Hyrax::IiifAv::Engine.routes.url_helpers.iiif_av_content_url(
+              solr_document.id,
+              label: label,
+              host: request.base_url
+            ),
             label: label,
             width: width,
             height: height,

From 307236a1f820b419447b739634af74433d5d0b78 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 4 Jan 2024 15:42:29 -0800
Subject: [PATCH 11/31] =?UTF-8?q?Revert=20"=F0=9F=90=9B=20HACK:=20Disable?=
 =?UTF-8?q?=20mp4=20and=20webm=20derivative=20generation"?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit 9194865b80a05d548f9c75e3a75dc83bb5086111.
---
 .../file_set_derivatives_overrides.rb             | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/config/initializers/file_set_derivatives_overrides.rb b/config/initializers/file_set_derivatives_overrides.rb
index d67ca84e6..af88c5318 100644
--- a/config/initializers/file_set_derivatives_overrides.rb
+++ b/config/initializers/file_set_derivatives_overrides.rb
@@ -48,17 +48,12 @@ def create_video_derivatives(filename)
     original_size = "#{width}x#{height}"
     size = width.nil? || height.nil? ? DEFAULT_VIDEO_SIZE : original_size
     Hydra::Derivatives::Processors::Video::Processor.config.size_attributes = size
-    # HACK: Commented out the non-thumbnail derivative generation as they are clogging the ecosystem.
-    # See https://github.com/scientist-softserv/palni-palci/issues/924
-    # rubocop:disable Style/TrailingCommaInHashLiteral
     Hydra::Derivatives::VideoDerivatives.create(filename,
                                                 outputs: [{ label: :thumbnail, format: 'jpg',
                                                             url: derivative_url('thumbnail') },
-                                                          # { label: 'webm', format: 'webm',
-                                                          #   url: derivative_url('webm') },
-                                                          # { label: 'mp4', format: 'mp4',
-                                                          #   url: derivative_url('mp4') }
-                                                         ])
-    # rubocop:enable Style/TrailingCommaInHashLiteral
+                                                          { label: 'webm', format: 'webm',
+                                                            url: derivative_url('webm') },
+                                                          { label: 'mp4', format: 'mp4',
+                                                            url: derivative_url('mp4') }])
   end
-end
+end
\ No newline at end of file

From 69b41657e4a30f58ae4682ab3871f6131996c65e Mon Sep 17 00:00:00 2001
From: Rob Kaufman <rob@notch8.com>
Date: Fri, 5 Jan 2024 13:14:19 -0800
Subject: [PATCH 12/31] helm deployment of intensive worker

---
 bin/helm_deploy           |   2 +-
 bin/worker                |   6 +-
 ops/demo-deploy.tmpl.yaml | 166 +++++++++++++++++++++++++++++++++++---
 3 files changed, 161 insertions(+), 13 deletions(-)

diff --git a/bin/helm_deploy b/bin/helm_deploy
index 631b57b9f..9972877c6 100755
--- a/bin/helm_deploy
+++ b/bin/helm_deploy
@@ -23,7 +23,7 @@ WORKER_IMAGE="${WORKER_IMAGE:-ghcr.io/samvera/hyku/worker}"
 DEPLOY_TAG="${DEPLOY_TAG:-latest}"
 WORKER_TAG="${WORKER_TAG:-$DEPLOY_TAG}"
 
-helm pull oci://ghcr.io/samvera/charts/hyrax --version 2.0.0 --untar --untardir charts
+helm pull oci://ghcr.io/samvera/charts/hyrax --version 3.5.1 --untar --untardir charts
 
 helm repo update
 
diff --git a/bin/worker b/bin/worker
index b7605486b..a369ffc11 100755
--- a/bin/worker
+++ b/bin/worker
@@ -9,4 +9,8 @@ else
   puts 'DATABASE_URL not set, no pool change needed'
 end
 
-exec "echo $DATABASE_URL && bundle exec sidekiq"
+if ENV['SIDEKIQ_CONFIG']
+  exec "echo $DATABASE_URL && bundle exec sidekiq -C #{ENV['SIDEKIQ_CONFIG']}"
+else
+  exec "echo $DATABASE_URL && bundle exec sidekiq"
+end
diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 8b41784cb..9082b90a0 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -47,9 +47,11 @@ ingress:
     - host: commons-archive.org
       paths:
         - path: /
+          pathType: ImplementationSpecific
     - host: "*.commons-archive.org"
       paths:
         - path: /
+          pathType: ImplementationSpecific
   annotations: {
     kubernetes.io/ingress.class: "nginx",
     nginx.ingress.kubernetes.io/proxy-body-size: "0",
@@ -190,7 +192,7 @@ worker:
     fsGroup: 101
     fsGroupChangePolicy: "OnRootMismatch"
 
-worker_resource_intensive:
+workerIntensive:
   replicaCount: 1
   resources:
     limits:
@@ -199,13 +201,152 @@ worker_resource_intensive:
     requests:
       memory: "8Gi"
       cpu: "600m"
-  extraVolumeMounts: *volMounts
-  extraEnvVars: *envVars
-  podSecurityContext:
-    runAsUser: 1001
-    runAsGroup: 101
-    fsGroup: 101
-    fsGroupChangePolicy: "OnRootMismatch"
+  extraEnvVars:
+    - name: SIDEKIQ_CONFIG
+      value: "config/sidekiq_resource_intensive.yml"
+
+extraDeploy:
+  - |-
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata:
+        name: {{ include "hyrax.fullname" . }}-intensive-worker
+        labels:
+          {{- include "hyrax.labels" . | nindent 4 }}
+      spec:
+        replicas: {{ .Values.workerIntensive.replicaCount }}
+        selector:
+          matchLabels:
+            {{- include "hyrax.workerSelectorLabels" . | nindent 6 }}
+        template:
+          metadata:
+          {{- with .Values.podAnnotations }}
+            annotations:
+              {{- toYaml . | nindent 8 }}
+          {{- end }}
+            labels:
+              {{- include "hyrax.workerSelectorLabels" . | nindent 8 }}
+          spec:
+            initContainers:
+              - name: db-wait
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                env:
+                  {{- toYaml .Values.workerIntensive.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                command:
+                  - sh
+                  - -c
+                  - db-wait.sh "$REDIS_HOST:6379"
+              {{- if .Values.worker.extraInitContainers }}
+              {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
+              {{- end }}
+            {{- with .Values.imagePullSecrets }}
+            imagePullSecrets:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            serviceAccountName: {{ include "hyrax.serviceAccountName" . }}
+            securityContext:
+              {{- toYaml .Values.worker.podSecurityContext | nindent 8 }}
+            containers:
+              - name: {{ .Chart.Name }}-worker
+                securityContext:
+                  {{- toYaml .Values.worker.securityContext | nindent 12 }}
+                image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag | default .Chart.AppVersion }}"
+                imagePullPolicy: {{ .Values.worker.image.pullPolicy }}
+                envFrom:
+                  - configMapRef:
+                      name: {{ include "hyrax.fullname" . }}-env
+                  - secretRef:
+                      name: {{ template "hyrax.fullname" . }}
+                  {{- if .Values.solrExistingSecret }}
+                  - secretRef:
+                      name: {{ .Values.solrExistingSecret }}
+                  {{- end }}
+                  {{- with .Values.worker.extraEnvFrom }}
+                    {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                env:
+                  {{- toYaml .Values.workerIntensive.extraEnvVars | nindent 12 }}
+                  {{- toYaml .Values.worker.extraEnvVars | nindent 12 }}
+                {{- if .Values.worker.readinessProbe.enabled }}
+                readinessProbe:
+                  exec:
+                    command:
+                      {{- toYaml .Values.worker.readinessProbe.command | nindent 16 }}
+                  failureThreshold: {{ .Values.worker.readinessProbe.failureThreshold }}
+                  initialDelaySeconds: {{ .Values.worker.readinessProbe.initialDelaySeconds }}
+                  periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
+                  successThreshold: {{ .Values.worker.readinessProbe.successThreshold }}
+                  timeoutSeconds: {{ .Values.worker.readinessProbe.timeoutSeconds }}
+                {{- end }}
+                volumeMounts:
+                  - name: derivatives
+                    mountPath: /app/samvera/derivatives
+                  - name: uploads
+                    subPath: file_cache
+                    mountPath: /app/samvera/file_cache
+                  - name: uploads
+                    subPath: uploads
+                    mountPath: /app/samvera/uploads
+                  {{- if .Values.applicationExistingClaim }}
+                  - name: application
+                    mountPath: /app/samvera/hyrax-webapp
+                  {{- end }}
+                  {{- with .Values.worker.extraVolumeMounts }}
+                  {{- toYaml . | nindent 12 }}
+                  {{- end }}
+                resources:
+                  {{- toYaml .Values.workerIntensive.resources | nindent 12 }}
+                {{- with .Values.extraContainerConfiguration }}
+                {{- toYaml . | nindent 10 }}
+                {{- end }}
+            volumes:
+              - name: "derivatives"
+                {{- if and .Values.derivativesVolume.enabled .Values.derivativesVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.derivativesVolume.existingClaim }}
+                {{- else if .Values.derivativesVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-derivatives
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              - name: "uploads"
+                {{- if and .Values.uploadsVolume.enabled .Values.uploadsVolume.existingClaim }}
+                persistentVolumeClaim:
+                  claimName: {{ .Values.uploadsVolume.existingClaim }}
+                {{- else if .Values.uploadsVolume.enabled }}
+                persistentVolumeClaim:
+                  claimName: {{ template "hyrax.fullname" . }}-uploads
+                {{ else }}
+                emptyDir: {}
+                {{ end }}
+              {{- if .Values.applicationExistingClaim }}
+              - name: "application"
+                persistentVolumeClaim:
+                  claimName: {{ .Values.applicationExistingClaim }}
+              {{- end }}
+              {{- with .Values.worker.extraVolumes }}
+              {{- toYaml . | nindent 8 }}
+              {{- end }}
+            {{- with .Values.worker.nodeSelector }}
+            nodeSelector:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.affinity }}
+            affinity:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
+            {{- with .Values.worker.tolerations }}
+            tolerations:
+              {{- toYaml . | nindent 8 }}
+            {{- end }}
 
 podSecurityContext:
   runAsUser: 1001
@@ -229,6 +370,10 @@ redis:
   password: $REDIS_PASSWORD
 solr:
   enabled: false
+fits:
+  enabled: true
+  servicePort: 8080
+  subPath: /fits
 
 externalPostgresql:
   host: postgres-cluster-alpha-ha.postgres.svc.cluster.local
@@ -242,7 +387,7 @@ externalSolrCollection: demo-palni-palci
 externalSolrPassword: $SOLR_ADMIN_PASSWORD
 
 global:
-  hyraxName: palni-palci-demo-pals
+  hyraxHostName: palni-palci-demo-pals
 
 nginx:
   enabled: true
@@ -254,8 +399,7 @@ nginx:
     tag: 1.21.5-debian-10-r4
   serverBlock: |-
     upstream rails_app {
-      server {{ .Values.global.hyraxName }};
-    }
+      server {{ .Values.global.hyraxHostName }};}
 
     map ${DOLLAR}status ${DOLLAR}loggable {
         ~^444  0;

From 47546b6f69849a85f280e6e4bf5bffd01edda30c Mon Sep 17 00:00:00 2001
From: Rob Kaufman <rob@notch8.com>
Date: Fri, 5 Jan 2024 13:17:01 -0800
Subject: [PATCH 13/31] resource adjustments

---
 ops/demo-deploy.tmpl.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 9082b90a0..9ce32c101 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -196,11 +196,11 @@ workerIntensive:
   replicaCount: 1
   resources:
     limits:
-      memory: "16Gi"
-      cpu: "1200m"
+      memory: "12Gi"
+      cpu: "4"
     requests:
-      memory: "8Gi"
-      cpu: "600m"
+      memory: "4Gi"
+      cpu: "2"
   extraEnvVars:
     - name: SIDEKIQ_CONFIG
       value: "config/sidekiq_resource_intensive.yml"

From e176af1bd07e099cdb946761c92cc8c8bca265f0 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 5 Jan 2024 15:35:13 -0800
Subject: [PATCH 14/31] undo changes to unused file

---
 docker-compose.production.yml | 43 ++++++++++++++---------------------
 1 file changed, 17 insertions(+), 26 deletions(-)

diff --git a/docker-compose.production.yml b/docker-compose.production.yml
index e0b369661..8412c2ad5 100644
--- a/docker-compose.production.yml
+++ b/docker-compose.production.yml
@@ -14,25 +14,6 @@ x-app: &app
   networks:
     internal:
 
-x-app-worker: &app-worker
-  <<: *app
-  image: ghcr.io/samvera/hyku/worker:${TAG:-latest}
-  depends_on:
-    check_volumes:
-      condition: service_completed_successfully
-    initialize_app:
-      condition: service_completed_successfully
-    db:
-      condition: service_started
-    solr:
-      condition: service_started
-    fcrepo:
-      condition: service_started
-    redis:
-      condition: service_started
-    zoo:
-      condition: service_started
-
 volumes:
   assets:
   cache:
@@ -152,19 +133,29 @@ services:
         condition: service_started
       worker:
         condition: service_started
-      worker_resource_intensive:
-        condition: service_started
       initialize_app:
         condition: service_completed_successfully
     expose:
       - 3000
 
   worker:
-    <<: *app-worker
-
-  worker_resource_intensive:
-    <<: *app-worker
-    command: sh -l -c 'bundle && bundle exec sidekiq -C config/sidekiq_resource_intensive.yml'
+    <<: *app
+    image: ghcr.io/samvera/hyku/worker:${TAG:-latest}
+    depends_on:
+      check_volumes:
+        condition: service_completed_successfully
+      initialize_app:
+        condition: service_completed_successfully
+      db:
+        condition: service_started
+      solr:
+        condition: service_started
+      fcrepo:
+        condition: service_started
+      redis:
+        condition: service_started
+      zoo:
+        condition: service_started
 
   # Do not recurse through all of tmp. derivitives will make booting
   # very slow and eventually just time out as data grows

From f82c85870eacd6e0fe1592c53afd03dd67e678f7 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 5 Jan 2024 16:04:10 -0800
Subject: [PATCH 15/31] use ENV var for intensive worker thread count

---
 .env                                  | 1 +
 config/sidekiq_resource_intensive.yml | 2 +-
 ops/demo-deploy.tmpl.yaml             | 3 ++-
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.env b/.env
index d61ae725a..07a6e9335 100644
--- a/.env
+++ b/.env
@@ -23,6 +23,7 @@ PASSENGER_APP_ENV=development
 RAILS_LOG_TO_STDOUT=true
 REDIS_HOST=redis
 SECRET_KEY_BASE=asdf
+SIDEKIQ_INTENSIVE_THREAD_COUNT=1
 SOLR_ADMIN_PASSWORD=SolrRocks
 SOLR_ADMIN_USER=solr
 SOLR_COLLECTION_NAME=hydra-development
diff --git a/config/sidekiq_resource_intensive.yml b/config/sidekiq_resource_intensive.yml
index 9865178ef..fc520467e 100644
--- a/config/sidekiq_resource_intensive.yml
+++ b/config/sidekiq_resource_intensive.yml
@@ -1,5 +1,5 @@
 ---
-:concurrency: 1
+:concurrency: <%= ENV['SIDEKIQ_INTENSIVE_THREAD_COUNT'] %>
 :queues:
   - default
   - import
diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 9ce32c101..2f0dc1574 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -399,7 +399,8 @@ nginx:
     tag: 1.21.5-debian-10-r4
   serverBlock: |-
     upstream rails_app {
-      server {{ .Values.global.hyraxHostName }};}
+      server {{ .Values.global.hyraxHostName }};
+    }
 
     map ${DOLLAR}status ${DOLLAR}loggable {
         ~^444  0;

From 1c89d521f05064d04a4b8714f54465bb1dc00d88 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 5 Jan 2024 16:41:52 -0800
Subject: [PATCH 16/31] adjust intensive worker prod resources

Co-authored-by: Rob Kaufman <rob@notch8.com>
---
 ops/demo-deploy.tmpl.yaml       |  2 ++
 ops/production-deploy.tmpl.yaml | 15 ++++++++++-----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 2f0dc1574..1e38a6bd1 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -204,6 +204,8 @@ workerIntensive:
   extraEnvVars:
     - name: SIDEKIQ_CONFIG
       value: "config/sidekiq_resource_intensive.yml"
+    - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
+      value: 1
 
 extraDeploy:
   - |-
diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
index 20e7caddc..ed6a16c1b 100644
--- a/ops/production-deploy.tmpl.yaml
+++ b/ops/production-deploy.tmpl.yaml
@@ -199,15 +199,20 @@ worker:
       memory: "2Gi"
       cpu: "150m"
 
-worker_resource_intensive:
+workerIntensive:
   replicaCount: 1
   resources:
     limits:
-      memory: "16Gi"
-      cpu: "1200m"
+      memory: "12Gi"
+      cpu: "4"
     requests:
-      memory: "8Gi"
-      cpu: "600m"
+      memory: "4Gi"
+      cpu: "2"
+  extraEnvVars:
+    - name: SIDEKIQ_CONFIG
+      value: "config/sidekiq_resource_intensive.yml"
+    - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
+      value: 1
 
   extraVolumeMounts: *volMounts
   extraEnvVars: *envVars

From 10e834b78fafac7331e8091565e24abb86cfb300 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 5 Jan 2024 17:04:01 -0800
Subject: [PATCH 17/31] fix disarranged YAML

---
 ops/production-deploy.tmpl.yaml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
index ed6a16c1b..e9b3c0e39 100644
--- a/ops/production-deploy.tmpl.yaml
+++ b/ops/production-deploy.tmpl.yaml
@@ -198,6 +198,13 @@ worker:
     requests:
       memory: "2Gi"
       cpu: "150m"
+  extraVolumeMounts: *volMounts
+  extraEnvVars: *envVars
+  podSecurityContext:
+    runAsUser: 1001
+    runAsGroup: 101
+    fsGroup: 101
+    fsGroupChangePolicy: "OnRootMismatch"
 
 workerIntensive:
   replicaCount: 1
@@ -214,13 +221,6 @@ workerIntensive:
     - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
       value: 1
 
-  extraVolumeMounts: *volMounts
-  extraEnvVars: *envVars
-  podSecurityContext:
-    runAsUser: 1001
-    runAsGroup: 101
-    fsGroup: 101
-    fsGroupChangePolicy: "OnRootMismatch"
 podSecurityContext:
   runAsUser: 1001
   runAsGroup: 101

From 8171b377f0a4c2a7b58c162f3e3dd034d11ceb64 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Fri, 5 Jan 2024 17:42:16 -0800
Subject: [PATCH 18/31] update prod to Hyrax helm chart v3.5.1

---
 ops/production-deploy.tmpl.yaml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/ops/production-deploy.tmpl.yaml b/ops/production-deploy.tmpl.yaml
index e9b3c0e39..ab9468c51 100644
--- a/ops/production-deploy.tmpl.yaml
+++ b/ops/production-deploy.tmpl.yaml
@@ -47,9 +47,11 @@ ingress:
     - host: hykucommons.org
       paths:
         - path: /
+          pathType: ImplementationSpecific
     - host: "*.hykucommons.org"
       paths:
         - path: /
+          pathType: ImplementationSpecific
   annotations: {
     kubernetes.io/ingress.class: "nginx",
     nginx.ingress.kubernetes.io/proxy-body-size: "0",
@@ -243,6 +245,10 @@ redis:
   password: $REDIS_PASSWORD
 solr:
   enabled: false
+fits:
+  enabled: true
+  servicePort: 8080
+  subPath: /fits
 
 externalPostgresql:
   host: postgres-cluster-alpha-ha.postgres.svc.cluster.local
@@ -256,7 +262,7 @@ externalSolrCollection: production-palni-palci
 externalSolrPassword: $SOLR_ADMIN_PASSWORD
 
 global:
-  hyraxName: palni-palci-production-pals
+  hyraxHostName: palni-palci-production-pals
 
 nginx:
   enabled: true
@@ -268,7 +274,7 @@ nginx:
     tag: 1.21.5-debian-10-r4
   serverBlock: |-
     upstream rails_app {
-      server {{ .Values.global.hyraxName }};
+      server {{ .Values.global.hyraxHostName }};
     }
 
     map ${DOLLAR}status ${DOLLAR}loggable {

From a4d5347b75898afee9aebec62c16c63dbb7fc96d Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Tue, 9 Jan 2024 10:52:43 -0800
Subject: [PATCH 19/31] fix demo redis deploy issue

Change pulled from:
- https://github.com/samvera/hyku/pull/2145
---
 ops/demo-deploy.tmpl.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 1e38a6bd1..006cedffe 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -367,9 +367,10 @@ fcrepo:
 postgresql:
   enabled: false
 redis:
-  cluster:
-    enabled: false
-  password: $REDIS_PASSWORD
+  enabled: true
+  architecture: standalone
+  auth:
+    password: $REDIS_PASSWORD
 solr:
   enabled: false
 fits:

From dc7d470140c55abc917c80040b16744b064b177f Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 17 Jan 2024 10:09:20 -0800
Subject: [PATCH 20/31] env vars need to be strings

---
 ops/demo-deploy.tmpl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index d48c772ff..97cd13bf9 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -207,7 +207,7 @@ workerIntensive:
     - name: SIDEKIQ_CONFIG
       value: "config/sidekiq_resource_intensive.yml"
     - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
-      value: 1
+      value: "1"
 
 extraDeploy:
   - |-

From 0aea9de59d7a2613dd7afbd00b0c5fc769a1e189 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 17 Jan 2024 10:09:20 -0800
Subject: [PATCH 21/31] env vars need to be strings

---
 ops/demo-deploy.tmpl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 006cedffe..83190d98d 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -205,7 +205,7 @@ workerIntensive:
     - name: SIDEKIQ_CONFIG
       value: "config/sidekiq_resource_intensive.yml"
     - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
-      value: 1
+      value: "1"
 
 extraDeploy:
   - |-

From f763d4db546ae4447cf7a6ee7d29fa2723c393fe Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Tue, 23 Jan 2024 17:06:10 -0800
Subject: [PATCH 22/31] correctly parse Redis host value

Because of the special way `extraDeploy` gets interpreted, fetching ENV
values using `$` will not work.
---
 ops/demo-deploy.tmpl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 83190d98d..e4f747e06 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -244,7 +244,7 @@ extraDeploy:
                 command:
                   - sh
                   - -c
-                  - db-wait.sh "$REDIS_HOST:6379"
+                  - db-wait.sh {{ include "hyrax.redis.host" . }}
               {{- if .Values.worker.extraInitContainers }}
               {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
               {{- end }}

From 3576eb86967020ba6c5f7be5a43d011db5210622 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 24 Jan 2024 13:12:42 -0800
Subject: [PATCH 23/31] prevent variable substitution for db-wait step

The correct value should be `$REDIS_HOST:6379`. In Helm templates, using
`$$` will skip variable substitution and use the correct value.
---
 ops/demo-deploy.tmpl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 281cdbae0..985fc94dc 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -246,7 +246,7 @@ extraDeploy:
                 command:
                   - sh
                   - -c
-                  - db-wait.sh {{ include "hyrax.redis.host" . }}
+                  - db-wait.sh "$$REDIS_HOST:6379"
               {{- if .Values.worker.extraInitContainers }}
               {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
               {{- end }}

From 6e632d8985b78f419c5e151998ede6ab21a19a97 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 24 Jan 2024 13:12:42 -0800
Subject: [PATCH 24/31] prevent variable substitution for db-wait step

The correct value should be `$REDIS_HOST:6379`. In Helm templates, using
`$$` will skip variable substitution and use the correct value.
---
 ops/demo-deploy.tmpl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index e4f747e06..a1e1eb9f3 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -244,7 +244,7 @@ extraDeploy:
                 command:
                   - sh
                   - -c
-                  - db-wait.sh {{ include "hyrax.redis.host" . }}
+                  - db-wait.sh "$$REDIS_HOST:6379"
               {{- if .Values.worker.extraInitContainers }}
               {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
               {{- end }}

From 31e2564e2e5b166dfdccdfb1c9351b7d0855ed35 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 24 Jan 2024 13:38:46 -0800
Subject: [PATCH 25/31] attempt to escape variable substitution again

The previous commit/attempt produced:

`sh -c 'db-wait.sh "$:6379"'`

So some variable substitution is still happening. Wrapping the command
in single quotes may prevent that.
---
 ops/demo-deploy.tmpl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 985fc94dc..eff962b3a 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -246,7 +246,7 @@ extraDeploy:
                 command:
                   - sh
                   - -c
-                  - db-wait.sh "$$REDIS_HOST:6379"
+                  - 'db-wait.sh "$$REDIS_HOST:6379"'
               {{- if .Values.worker.extraInitContainers }}
               {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
               {{- end }}

From 7a4dedb2bcb816814fb4f6f8018b4fc6b6b6b497 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 24 Jan 2024 13:38:46 -0800
Subject: [PATCH 26/31] attempt to escape variable substitution again

The previous commit/attempt produced:

`sh -c 'db-wait.sh "$:6379"'`

So some variable substitution is still happening. Wrapping the command
in single quotes may prevent that.
---
 ops/demo-deploy.tmpl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index a1e1eb9f3..a40a8c6fe 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -244,7 +244,7 @@ extraDeploy:
                 command:
                   - sh
                   - -c
-                  - db-wait.sh "$$REDIS_HOST:6379"
+                  - 'db-wait.sh "$$REDIS_HOST:6379"'
               {{- if .Values.worker.extraInitContainers }}
               {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
               {{- end }}

From 9998530789d642bf5c467d4ce63b8c6a1a5d3d0f Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 24 Jan 2024 17:06:47 -0800
Subject: [PATCH 27/31] use helm to hopefully avoid env variable substitution

---
 ops/demo-deploy.tmpl.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index eff962b3a..fc007b718 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -208,6 +208,7 @@ workerIntensive:
       value: "config/sidekiq_resource_intensive.yml"
     - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
       value: "1"
+  redisUrl: "$REDIS_HOST:6379"
 
 extraDeploy:
   - |-
@@ -246,7 +247,7 @@ extraDeploy:
                 command:
                   - sh
                   - -c
-                  - 'db-wait.sh "$$REDIS_HOST:6379"'
+                  - db-wait.sh "{{ .Values.workerIntensive.redisUrl }}"
               {{- if .Values.worker.extraInitContainers }}
               {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
               {{- end }}

From a7b07b957a83d7eb70691785c87228d038abec5c Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 24 Jan 2024 17:06:47 -0800
Subject: [PATCH 28/31] use helm to hopefully avoid env variable substitution

---
 ops/demo-deploy.tmpl.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index a40a8c6fe..014e50853 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -206,6 +206,7 @@ workerIntensive:
       value: "config/sidekiq_resource_intensive.yml"
     - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
       value: "1"
+  redisUrl: "$REDIS_HOST:6379"
 
 extraDeploy:
   - |-
@@ -244,7 +245,7 @@ extraDeploy:
                 command:
                   - sh
                   - -c
-                  - 'db-wait.sh "$$REDIS_HOST:6379"'
+                  - db-wait.sh "{{ .Values.workerIntensive.redisUrl }}"
               {{- if .Values.worker.extraInitContainers }}
               {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
               {{- end }}

From eb3b2fac07245842d0511478bab8dc3972274470 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Wed, 24 Jan 2024 17:32:29 -0800
Subject: [PATCH 29/31] let's try this again...

---
 ops/demo-deploy.tmpl.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 014e50853..e4f747e06 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -206,7 +206,6 @@ workerIntensive:
       value: "config/sidekiq_resource_intensive.yml"
     - name: SIDEKIQ_INTENSIVE_THREAD_COUNT
       value: "1"
-  redisUrl: "$REDIS_HOST:6379"
 
 extraDeploy:
   - |-
@@ -245,7 +244,7 @@ extraDeploy:
                 command:
                   - sh
                   - -c
-                  - db-wait.sh "{{ .Values.workerIntensive.redisUrl }}"
+                  - db-wait.sh {{ include "hyrax.redis.host" . }}
               {{- if .Values.worker.extraInitContainers }}
               {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
               {{- end }}

From f07f23b7b8ff4654e15a00049e2bfaa35aadbd55 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 25 Jan 2024 14:46:05 -0800
Subject: [PATCH 30/31] add redis port

---
 ops/demo-deploy.tmpl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index 281cdbae0..2c4456d0c 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -246,7 +246,7 @@ extraDeploy:
                 command:
                   - sh
                   - -c
-                  - db-wait.sh {{ include "hyrax.redis.host" . }}
+                  - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379"
               {{- if .Values.worker.extraInitContainers }}
               {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
               {{- end }}

From 4733738c6d973a809527316a4e8123e5315db146 Mon Sep 17 00:00:00 2001
From: Benjamin Kiah Stroud <32469930+bkiahstroud@users.noreply.github.com>
Date: Thu, 25 Jan 2024 14:46:05 -0800
Subject: [PATCH 31/31] add redis port

---
 ops/demo-deploy.tmpl.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ops/demo-deploy.tmpl.yaml b/ops/demo-deploy.tmpl.yaml
index e4f747e06..cfc6a81ed 100644
--- a/ops/demo-deploy.tmpl.yaml
+++ b/ops/demo-deploy.tmpl.yaml
@@ -244,7 +244,7 @@ extraDeploy:
                 command:
                   - sh
                   - -c
-                  - db-wait.sh {{ include "hyrax.redis.host" . }}
+                  - "db-wait.sh {{ include "hyrax.redis.host" . }}:6379"
               {{- if .Values.worker.extraInitContainers }}
               {{- toYaml .Values.worker.extraInitContainers | nindent 8 }}
               {{- end }}