Search integration configured as a service (PP-93) (#1554)

* Replace opensearch integration with a service configuration.
* Use docker compose for our build tests
* Add new environment variable info to readme.
* Bump up admin-ui version.

.github/workflows/test-build.yml

@@ -269,18 +269,6 @@ jobs:
       matrix:
         platform: ["linux/amd64", "linux/arm64"]
         image: ["scripts", "webapp"]
-    env:
-      POSTGRES_USER: palace_user
-      POSTGRES_PASSWORD: test
-      POSTGRES_DB: palace_circulation
-
-    services:
-      postgres:
-        image: postgres:12
-        env:
-          POSTGRES_USER: ${{ env.POSTGRES_USER }}
-          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
-          POSTGRES_DB: ${{ env.POSTGRES_DB }}
 
     steps:
       - uses: actions/checkout@v4
@@ -297,36 +285,23 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./docker/Dockerfile
-          tags: test_image
-          load: true
-          target: ${{ matrix.image }}
-          cache-from: type=gha,scope=buildkit-${{ github.run_id }}
-          platforms: ${{ matrix.platform }}
-          build-args: |
-            BASE_IMAGE=${{ needs.docker-image-build.outputs.baseimage }}
-
-      - name: Start container
-        run: >
-          docker run --rm --name test_container -d --platform ${{ matrix.platform }}
-          --network ${{job.services.postgres.network}}
-          -e SIMPLIFIED_PRODUCTION_DATABASE="postgresql://${{ env.POSTGRES_USER }}:${{ env.POSTGRES_PASSWORD }}@postgres:5432/${{ env.POSTGRES_DB }}"
-          test_image
+      - name: Build & Start container
+        run: docker compose up -d --build ${{ matrix.image }}
+        env:
+          BUILD_PLATFORM: ${{ matrix.platform }}
+          BUILD_CACHE_FROM: type=gha,scope=buildkit-${{ github.run_id }}
+          BUILD_BASE_IMAGE: ${{ needs.docker-image-build.outputs.baseimage }}
 
       - name: Run tests
-        run: ./docker/ci/test_${{ matrix.image }}.sh test_container
+        run: ./docker/ci/test_${{ matrix.image }}.sh ${{ matrix.image }}
 
       - name: Output logs
         if: failure()
-        run: docker logs test_container
+        run: docker logs circulation-${{ matrix.image }}-1
 
       - name: Stop container
         if: always()
-        run: docker stop test_container
+        run: docker compose down
 
   docker-image-push:
     name: Push circ-${{ matrix.image }}

README.md

@@ -120,7 +120,7 @@ Elasticsearch is no longer supported.
 We recommend that you run Opensearch with docker using the following docker commands:
 
 ```sh
-docker run --name opensearch -d --rm -p 9006:9200 -e "discovery.type=single-node" -e "plugins.security.disabled=true" "opensearchproject/opensearch:1"
+docker run --name opensearch -d --rm -p 9200:9200 -e "discovery.type=single-node" -e "plugins.security.disabled=true" "opensearchproject/opensearch:1"
 docker exec opensearch opensearch-plugin -s install analysis-icu
 docker restart opensearch
 ```
@@ -161,6 +161,22 @@ To let the application know which database to use, set the `SIMPLIFIED_PRODUCTIO
 export SIMPLIFIED_PRODUCTION_DATABASE="postgresql://palace:test@localhost:5432/circ"
 ```
 
+#### Opensearch
+
+To let the application know which Opensearch instance to use, you can set the following environment variables:
+
+- `PALACE_SEARCH_URL`: The url of the Opensearch instance (**required**).
+- `PALACE_SEARCH_INDEX_PREFIX`: The prefix to use for the Opensearch indices. The default is `circulation-works`.
+    This is useful if you want to use the same Opensearch instance for multiple CM (optional).
+- `PALACE_SEARCH_TIMEOUT`: The timeout in seconds to use when connecting to the Opensearch instance. The default is `20`
+  (optional).
+- `PALACE_SEARCH_MAXSIZE`: The maximum size of the connection pool to use when connecting to the Opensearch instance.
+  (optional).
+
+```sh
+export PALACE_SEARCH_URL="http://localhost:9200"
+```
+
 #### Storage Service
 
 The application optionally uses a s3 compatible storage service to store files. To configure the application to use
@@ -642,7 +658,7 @@ If you already have elastic search or postgres running locally, you can run them
 following environment variables:
 
 - `SIMPLIFIED_TEST_DATABASE`
-- `SIMPLIFIED_TEST_OPENSEARCH`
+- `PALACE_TEST_SEARCH_URL`
 
 Make sure the ports and usernames are updated to reflect the local configuration.
 

api/admin/config.py

@@ -16,7 +16,7 @@ class OperationalMode(str, Enum):
 class Configuration(LoggerMixin):
     APP_NAME = "Palace Collection Manager"
     PACKAGE_NAME = "@thepalaceproject/circulation-admin"
-    PACKAGE_VERSION = "1.12.0"
+    PACKAGE_VERSION = "1.13.0"
 
     STATIC_ASSETS = {
         "admin_js": "circulation-admin.js",

api/admin/controller/__init__.py

@@ -37,16 +37,9 @@ def setup_admin_controllers(manager: CirculationManager):
     )
     from api.admin.controller.patron_auth_services import PatronAuthServicesController
     from api.admin.controller.reset_password import ResetPasswordController
-    from api.admin.controller.search_service_self_tests import (
-        SearchServiceSelfTestsController,
-    )
     from api.admin.controller.self_tests import SelfTestsController
     from api.admin.controller.settings import SettingsController
     from api.admin.controller.sign_in import SignInController
-    from api.admin.controller.sitewide_services import (
-        SearchServicesController,
-        SitewideServicesController,
-    )
     from api.admin.controller.sitewide_settings import (
         SitewideConfigurationSettingsController,
     )
@@ -92,11 +85,6 @@ def setup_admin_controllers(manager: CirculationManager):
     manager.admin_individual_admin_settings_controller = (
         IndividualAdminSettingsController(manager)
     )
-    manager.admin_sitewide_services_controller = SitewideServicesController(manager)
-    manager.admin_search_service_self_tests_controller = (
-        SearchServiceSelfTestsController(manager)
-    )
-    manager.admin_search_services_controller = SearchServicesController(manager)
     manager.admin_catalog_services_controller = CatalogServicesController(manager)
     manager.admin_announcement_service = AnnouncementSettings(manager)
     manager.admin_search_controller = AdminSearchController(manager)

api/admin/controller/settings.py

@@ -24,7 +24,6 @@
 )
 from api.admin.validator import Validator
 from api.controller.circulation_manager import CirculationManagerController
-from core.external_search import ExternalSearchIndex
 from core.integration.base import (
     HasChildIntegrationConfiguration,
     HasIntegrationConfiguration,
@@ -406,11 +405,7 @@ def _get_prior_test_results(self, item, protocol_class=None, *extra_args):
         self_test_results = None
 
         try:
-            if self.type == "search service":
-                self_test_results = ExternalSearchIndex.prior_test_results(
-                    self._db, None, self._db, item
-                )
-            elif self.type == "metadata service" and protocol_class:
+            if self.type == "metadata service" and protocol_class:
                 self_test_results = protocol_class.prior_test_results(
                     self._db, *extra_args
                 )

api/admin/controller/work_editor.py

@@ -699,6 +699,6 @@ def custom_lists(self, identifier_type, identifier):
             # NOTE: This may not make a difference until the
             # works are actually re-indexed.
             for lane in affected_lanes:
-                lane.update_size(self._db, self.search_engine)
+                lane.update_size(self._db, search_engine=self.search_engine)
 
             return Response(str(_("Success")), 200)

api/admin/routes.py

@@ -484,32 +484,6 @@ def metadata_service_self_tests(identifier):
     )
 
 
-@app.route("/admin/search_services", methods=["GET", "POST"])
-@returns_json_or_response_or_problem_detail
-@requires_admin
-@requires_csrf_token
-def search_services():
-    return app.manager.admin_search_services_controller.process_services()
-
-
-@app.route("/admin/search_service/<service_id>", methods=["DELETE"])
-@returns_json_or_response_or_problem_detail
-@requires_admin
-@requires_csrf_token
-def search_service(service_id):
-    return app.manager.admin_search_services_controller.process_delete(service_id)
-
-
-@app.route("/admin/search_service_self_tests/<identifier>", methods=["GET", "POST"])
-@returns_json_or_response_or_problem_detail
-@requires_admin
-@requires_csrf_token
-def search_service_self_tests(identifier):
-    return app.manager.admin_search_service_self_tests_controller.process_search_service_self_tests(
-        identifier
-    )
-
-
 @app.route("/admin/catalog_services", methods=["GET", "POST"])
 @returns_json_or_response_or_problem_detail
 @requires_admin