Add ability to run deposit services

.env

@@ -1,7 +1,15 @@
 PASS_VERSION=0.8.0-SNAPSHOT
 
 ###################################################
-# PASS_CORE config ####################################
+# Localstack AWS props config #####################
+###################################################
+AWS_ACCESS_KEY_ID=test
+AWS_SECRET_ACCESS_KEY=test
+AWS_REGION=us-east-1
+AWS_SQS_ENDPOINT_OVERRIDE=http://localstack:4566
+
+###################################################
+# PASS_CORE config ################################
 ###################################################
 PASS_CORE_POSTGRES_PORT=5432
 PASS_CORE_API_PORT=8080
@@ -26,13 +34,29 @@ PASS_CORE_JAVA_OPTS="-Djavax.persistence.schema-generation.database.action=creat
 PASS_CORE_BACKEND_USER=backend
 PASS_CORE_BACKEND_PASSWORD=backend
 
-PASS_CORE_USE_SQS=false
-PASS_CORE_EMBED_JMS_BROKER=true
+PASS_CORE_USE_SQS=true
+PASS_CORE_EMBED_JMS_BROKER=false
+
+PASS_CORE_SUBMISSION_QUEUE=submission
+PASS_CORE_DEPOSIT_QUEUE=deposit
+PASS_CORE_SUBMISSION_EVENT_QUEUE=submission-event
 
 PASS_CORE_POLICY_INSTITUTION=johnshopkins.edu
 PASS_CORE_POLICY_INSTITUTIONAL_POLICY_TITLE=Johns Hopkins University (JHU) Open Access Policy
 PASS_CORE_POLICY_INSTITUTIONAL_REPOSITORY_NAME=JScholarship
 
+###################################################
+# PASS_DEPOSIT_SERVICES config ####################
+###################################################
+PASS_DEPOSIT_REPOSITORY_CONFIGURATION:classpath:/jscholarship-repositories.json
+
+PASS_CLIENT_URL=http://pass-core:8080/
+PASS_CLIENT_USER=backend
+PASS_CLIENT_PASSWORD=backend
+
+DSPACE_HOST=dspace
+DSPACE_PORT=8080
+
 ###################################################
 # PASS_UI config ##################################
 # ## Changes here require new image build #########

DSPACE_LICENSE

@@ -0,0 +1,31 @@
+BSD 3-Clause License
+
+Copyright (c) 2002-2021, LYRASIS.  All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+- Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+- Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGE.

README.md

@@ -2,11 +2,34 @@ The "demo" compose file describes an early system meant to demonstrate some new
 
 ## Running:
 
-Docker compose works as normal, but for the demo you need to specify both correct `yml` file and env file. In order to run a local instance, you can run:
-``` sh
+Docker compose works as normal, but for the demo you need to specify both correct `yml` file and env file. 
+
+In order to run a local instance **_without_** deposit-service and dspace, you can run:
+```
 docker compose up -d --no-build --quiet-pull --pull always
 ```
 
+In order to stop a local instance, you can run:
+```
+docker compose -p pass-docker down -v
+```
+Note the `-v` to remove the volumes, **this is critical** so on subsequent starts, user data is not duplicated.
+
+In order to run a local instance **_with_** deposit-service and dspace, you can run:
+```
+docker compose -p pass-docker -f docker-compose.yml -f docker-compose-deposit.yml -f docker-compose-dspace.yml up -d --no-build --quiet-pull
+```
+
+Run the following to create a test admin user in dspace:
+```
+docker-compose -p pass-docker -f dspace-cli.yml run --rm dspace-cli create-administrator -e [email protected] -f admin -l user -p admin -c en
+```
+
+Run the following to load sample data into dspace:
+```
+docker-compose -p pass-docker -f dspace-cli.yml -f dspace-cli.ingest.yml run --rm dspace-cli
+```
+
 ## Services:
 
 ### [`pass-auth`](https://github.com/jaredgalanis/pass-auth)

docker-compose-deposit.yml

@@ -0,0 +1,28 @@
+#
+# Copyright 2023 Johns Hopkins University
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# The docker-compose file to start PASS deposit services.
+version: '3.8'
+
+services:
+  deposit-services:
+    image: "ghcr.io/eclipse-pass/deposit-services-core:${PASS_VERSION}"
+    container_name: pass-deposit-services
+    env_file:
+      - .env
+    networks:
+      - back
+    depends_on:
+      - pass-core

docker-compose-dspace.yml

@@ -0,0 +1,141 @@
+#
+# DSpace License: DSPACE_LICENSE file in root of this project.
+# Details available online at: http://www.dspace.org/license/
+#
+# The docker-compose file to start Dspace7 with SwordV2 enabled.
+# The dspace service definitions were taken from the dspace-angular project:
+#   https://github.com/DSpace/dspace-angular.git
+# The service definitions have been slightly modified to run within the PASS docker-compose process.
+#
+version: '3.8'
+
+services:
+  # DSpace (backend) webapp container
+  dspace:
+      container_name: dspace
+      environment:
+        # Below syntax may look odd, but it is how to override dspace.cfg settings via env variables.
+        # See https://github.com/DSpace/DSpace/blob/main/dspace/config/config-definition.xml
+        # __P__ => "." (e.g. dspace__P__dir => dspace.dir)
+        # __D__ => "-" (e.g. google__D__metadata => google-metadata)
+        # dspace.dir, dspace.server.url, dspace.ui.url and dspace.name
+        dspace__P__dir: /dspace
+        dspace__P__server__P__url: http://dspace:8080/server
+        dspace__P__ui__P__url: http://localhost:4000
+        dspace__P__name: 'DSpace Started with Docker Compose'
+        # db.url: Ensure we are using the 'dspacedb' image for our database
+        db__P__url: 'jdbc:postgresql://dspacedb:5432/dspace'
+        # solr.server: Ensure we are using the 'dspacesolr' image for Solr
+        solr__P__server: http://dspacesolr:8983/solr
+        # proxies.trusted.ipranges: This setting is required for a REST API running in Docker to trust requests
+        # from the host machine. This IP range MUST correspond to the 'dspacenet' subnet defined above.
+        proxies__P__trusted__P__ipranges: '172.23.0'
+        swordv2__D__server__P__enabled: true
+        swordv2__D__server__P__url: http://dspace:8080/server/swordv2
+      image: "${DOCKER_OWNER:-dspace}/dspace:${DSPACE_VER:-dspace-7_x-test}"
+      depends_on:
+        - dspacedb
+      networks:
+        back:
+      ports:
+        - published: 8080
+          target: 8080
+      stdin_open: true
+      tty: true
+      volumes:
+        - assetstore:/dspace/assetstore
+        # Mount DSpace's solr configs to a volume, so that we can share to 'dspacesolr' container (see below)
+        - solr_configs:/dspace/solr
+      # Ensure that the database is ready BEFORE starting tomcat
+      # 1. While a TCP connection to dspacedb port 5432 is not available, continue to sleep
+      # 2. Then, run database migration to init database tables
+      # 3. Finally, start Tomcat
+      entrypoint:
+        - /bin/bash
+        - '-c'
+        - |
+          while (!</dev/tcp/dspacedb/5432) > /dev/null 2>&1; do sleep 1; done;
+          /dspace/bin/dspace database migrate
+          catalina.sh run
+  # DSpace database container
+  dspacedb:
+      container_name: dspacedb
+      environment:
+        PGDATA: /pgdata
+        POSTGRES_PASSWORD: dspace
+      image: dspace/dspace-postgres-pgcrypto
+      networks:
+        back:
+      ports:
+        - published: 5432
+          target: 5432
+      stdin_open: true
+      tty: true
+      volumes:
+        - pgdata:/pgdata
+  # DSpace Solr container
+  dspacesolr:
+      container_name: dspacesolr
+      image: "${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-dspace-7_x}"
+      # Needs main 'dspace' container to start first to guarantee access to solr_configs
+      depends_on:
+        - dspace
+      networks:
+        back:
+      ports:
+        - published: 8983
+          target: 8983
+      stdin_open: true
+      tty: true
+      working_dir: /var/solr/data
+      volumes:
+        # Keep Solr data directory between reboots
+        - solr_data:/var/solr/data
+      # Initialize all DSpace Solr cores using the mounted local configsets (see above), then start Solr
+      # * First, run precreate-core to create the core (if it doesn't yet exist). If exists already, this is a no-op
+      # * Second, copy configsets to this core:
+      #   Updates to Solr configs require the container to be rebuilt/restarted:
+      #   `docker-compose -p d7 -f docker/docker-compose.yml -f docker/docker-compose-rest.yml up -d --build dspacesolr`
+      entrypoint:
+        - /bin/bash
+        - '-c'
+        - |
+          init-var-solr
+          precreate-core authority /opt/solr/server/solr/configsets/authority
+          cp -r /opt/solr/server/solr/configsets/authority/* authority
+          precreate-core oai /opt/solr/server/solr/configsets/oai
+          cp -r /opt/solr/server/solr/configsets/oai/* oai
+          precreate-core search /opt/solr/server/solr/configsets/search
+          cp -r /opt/solr/server/solr/configsets/search/* search
+          precreate-core statistics /opt/solr/server/solr/configsets/statistics
+          cp -r /opt/solr/server/solr/configsets/statistics/* statistics
+          exec solr -f
+
+  dspace-angular:
+      container_name: dspace-angular
+      environment:
+        DSPACE_UI_SSL: 'false'
+        DSPACE_UI_HOST: dspace-angular
+        DSPACE_UI_PORT: '4000'
+        DSPACE_UI_NAMESPACE: /
+        DSPACE_REST_SSL: 'false'
+        DSPACE_REST_HOST: localhost
+        DSPACE_REST_PORT: 8080
+        DSPACE_REST_NAMESPACE: /server
+      image: dspace/dspace-angular:dspace-7_x
+      networks:
+        back:
+      ports:
+        - published: 4000
+          target: 4000
+        - published: 9876
+          target: 9876
+      stdin_open: true
+      tty: true
+
+volumes:
+  assetstore:
+  pgdata:
+  solr_data:
+  # Special volume used to share Solr configs from 'dspace' to 'dspacesolr' container (see above)
+  solr_configs:

docker-compose.yml

@@ -1,3 +1,18 @@
+#
+# Copyright 2023 Johns Hopkins University
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 version: '3.8'
 
 # The base docker-compose file for eclipse-pass.org deployments
@@ -28,6 +43,7 @@ services:
       - back
     depends_on:
       - postgres
+      - localstack
     healthcheck:
       test: "curl -u $PASS_CORE_BACKEND_USER:$PASS_CORE_BACKEND_PASSWORD -f http://pass-core:8080/data/user || exit 1"
       start_period: 30s
@@ -131,6 +147,21 @@ services:
     volumes:
       - ./demo_data.json:/data.json
 
+  localstack:
+    container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}"
+    image: localstack/localstack:2.1.0
+    ports:
+      - "127.0.0.1:4566:4566"            # LocalStack Gateway
+      - "127.0.0.1:4510-4559:4510-4559"  # external services port range
+    environment:
+      - DOCKER_HOST=unix:///var/run/docker.sock
+    volumes:
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "./localstack/sqs_bootstrap.sh:/etc/localstack/init/ready.d/init-aws.sh"
+    networks:
+      - back
+
 volumes:
   db:
     driver: local
@@ -140,6 +171,9 @@ networks:
     driver: bridge
   back:
     driver: bridge
+    ipam:
+      config:
+        - subnet: 172.23.0.0/16
 
 secrets:
   idp_backchannel:

dspace-cli.ingest.yml

@@ -0,0 +1,33 @@
+#
+# DSpace License: DSPACE_LICENSE file in root of this project.
+# Details available online at: http://www.dspace.org/license/
+#
+# This is a copy of the cli.ingest.yml that is available in the DSpace/DSpace
+# (Backend) at:
+# https://github.com/DSpace/DSpace/blob/main/dspace/src/main/docker-compose/cli.ingest.yml
+#
+# Therefore, it should be kept in sync with that file
+version: "3.7"
+
+services:
+  dspace-cli:
+    environment:
+    - AIPZIP=https://github.com/DSpace-Labs/AIP-Files/raw/main/dogAndReport.zip
+    - [email protected]
+    - AIPDIR=/tmp/aip-dir
+    entrypoint:
+      - /bin/bash
+      - '-c'
+      - |
+          rm -rf $${AIPDIR}
+          mkdir $${AIPDIR} /dspace/upload
+          cd $${AIPDIR}
+          pwd
+          curl $${AIPZIP} -L -s --output aip.zip
+          unzip aip.zip
+          cd $${AIPDIR}
+
+          /dspace/bin/dspace packager -r -a -t AIP -e $${ADMIN_EMAIL} -f -u SITE*.zip
+          /dspace/bin/dspace database update-sequences
+
+          /dspace/bin/dspace index-discovery