Merge pull request #68 from mkardas/incremental-update

Improve parallel processing
kermitt2 · Apr 12, 2022 · 2c2b637 · 2c2b637
2 parents e8d4ca5 + 6a8a2d1
commit 2c2b637
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 6 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -18,7 +18,7 @@ VOLUME /app/glutton-source/.gradle
 
 # source
 COPY lookup/ ./lookup/
-COPY matching/ ./matching/
+COPY indexing/ ./indexing/
 
 RUN cd /app/glutton-source/lookup && ./gradlew clean assemble --no-daemon
 
@@ -31,10 +31,9 @@ RUN apt-get update -qq && apt-get -qy install curl build-essential unzip
 RUN mkdir -p /app
 WORKDIR /app
 
-RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
-RUN apt-get update -qq && apt-get -y install nodejs
-COPY --from=builder /app/glutton-source/matching /app/matching
-RUN cd matching; npm install
+RUN apt-get update -qq && apt-get -y install nodejs npm
+COPY --from=builder /app/glutton-source/indexing /app/indexing
+RUN cd indexing; npm install
 
 COPY --from=builder /app/glutton-source/lookup/build/distributions/lookup-service-shadow-*.zip ./lookup-service.zip
 

diff --git a/lookup/src/main/java/com/scienceminer/lookup/configuration/LookupConfiguration.java b/lookup/src/main/java/com/scienceminer/lookup/configuration/LookupConfiguration.java
@@ -175,6 +175,7 @@ public class Elastic {
 
         private String host;
         private String index;
+        private int maxConnections = 10;
 
         public String getHost() {
             return host;
@@ -191,6 +192,14 @@ public String getIndex() {
         public void setIndex(String index) {
             this.index = index;
         }
+
+        public int getMaxConnections() {
+            return maxConnections;
+        }
+
+        public void setMaxConnections(int maxConnections) {
+            this.maxConnections = maxConnections;
+        }
     }
 
     public class Crossref {

diff --git a/lookup/src/main/java/com/scienceminer/lookup/storage/lookup/MetadataMatching.java b/lookup/src/main/java/com/scienceminer/lookup/storage/lookup/MetadataMatching.java
@@ -95,7 +95,11 @@ private MetadataMatching(LookupConfiguration configuration, MetadataLookup metad
                         .setRequestConfigCallback(
                                 requestConfigBuilder -> requestConfigBuilder
                                         .setConnectTimeout(30000)
-                                        .setSocketTimeout(60000)));
+                                        .setSocketTimeout(60000))
+                        .setHttpClientConfigCallback(
+                                httpAsyncClientBuilder -> httpAsyncClientBuilder
+                                        .setMaxConnPerRoute(configuration.getElastic().getMaxConnections())
+                                        .setMaxConnTotal(configuration.getElastic().getMaxConnections())));
 
         // note: maxRetryTimeoutMillis is deprecated in ES 7 due to implementation issue 
         // https://github.com/elastic/elasticsearch/pull/38085

diff --git a/lookup/src/main/java/com/scienceminer/lookup/web/resource/LookupController.java b/lookup/src/main/java/com/scienceminer/lookup/web/resource/LookupController.java
@@ -458,6 +458,9 @@ protected void processByQuery(
                             }
                         });
                         return;
+                    } else {
+			asyncResponse.resume(matchingDocument.getException());
+                        return;
                     }
                 } else {
                     asyncResponse.resume(matchingDocument.getFinalJsonObject());