Combine synchronized storing and indexing, re-organize components, HAL support (no more DOI centric approach), dependency update, import format update

.dockerignore

@@ -1,2 +1,3 @@
-/.git
-/data
+.git
+data
+logs/*

.gitignore

@@ -19,11 +19,12 @@ data/db
 scripts/dump.json
 scripts/dump.json.gz
 scripts/node_modules
-lookup/build
+build
 .gradle
 *.mdb
 **/node_modules/**
 **/out
 **/package-lock.json
-lookup/data/db/*
-pubmed-glutton/build/*
+data/db/*
+pubmed-glutton/*
+indexing/*

build.gradle

@@ -0,0 +1,252 @@
+buildscript {
+    repositories {
+        mavenLocal()
+        mavenCentral()
+        maven {
+            url 'https://plugins.gradle.org/m2/'
+        }
+    }
+
+    dependencies {
+        classpath 'gradle.plugin.org.kt3k.gradle.plugin:coveralls-gradle-plugin:2.12.0'
+        classpath "gradle.plugin.com.github.jengelman.gradle.plugins:shadow:7.0.0"
+    }
+}
+
+apply plugin: 'application'
+apply plugin: 'java'
+apply plugin: 'java-library'
+//apply plugin: 'maven'
+apply plugin: 'maven-publish'
+apply plugin: 'com.github.johnrengelman.shadow'
+
+group 'com.scienceminer.glutton'
+version '0.3-SNAPSHOT'
+
+//sourceCompatibility = 1.8
+sourceCompatibility = 1.11
+targetCompatibility = 1.11
+
+// The main class of the application
+mainClassName = 'com.scienceminer.glutton.web.LookupServiceApplication'
+tasks.run.workingDir = rootProject.rootDir
+
+repositories {
+    mavenLocal()
+    mavenCentral()
+}
+
+sourceSets.main.resources {
+    srcDirs = ["src/main/resources"];
+}
+
+configurations {
+    all*.exclude group: 'org.slf4j', module: "slf4j-log4j12"
+    all*.exclude group: 'log4j', module: "log4j"
+    all*.exclude group: 'log4j2', module: "log4j2"
+    implementation.setCanBeResolved(true)
+}
+
+processResources {
+    filesMatching('config/glutton.yml') {
+        filter {
+            it.replace('project.version', project.property('version_placeholder'))
+        }
+    }
+}
+
+task install(dependsOn: installShadowDist)
+
+dependencies {
+    testImplementation group: 'junit', name: 'junit', version: '4.12'
+    testImplementation "org.hamcrest:hamcrest-all:1.3"
+    testImplementation "org.easymock:easymock:3.5"
+
+    implementation "com.google.code.gson:gson:2.8.1"
+
+    //Dropwizard
+    implementation 'ru.vyarus:dropwizard-guicey:7.0.0'
+
+    implementation 'io.dropwizard:dropwizard-bom:4.0.0'
+    implementation 'io.dropwizard:dropwizard-core:4.0.0'
+    implementation 'io.dropwizard:dropwizard-assets:4.0.0'
+    implementation 'io.dropwizard:dropwizard-testing:4.0.0'
+    implementation 'io.dropwizard.modules:dropwizard-testing-junit4:4.0.0'
+    implementation 'io.dropwizard:dropwizard-forms:4.0.0'
+    implementation 'io.dropwizard:dropwizard-client:4.0.0'
+    implementation 'io.dropwizard:dropwizard-auth:4.0.0'
+    implementation 'io.dropwizard.metrics:metrics-core:4.2.22'
+    implementation 'io.dropwizard.metrics:metrics-servlets:4.2.22'
+
+    implementation 'jakarta.validation:jakarta.validation-api:2.0.2'
+
+    implementation 'org.slf4j:slf4j-api:1.7.30'
+    implementation 'ch.qos.logback:logback-classic:1.2.3'
+    implementation 'org.apache.logging.log4j:log4j-to-slf4j:2.8.2'
+
+    implementation "com.rockymadden.stringmetric:stringmetric-core_2.10:0.27.3"
+
+    //Parsing XML/Json
+    implementation group: 'org.codehaus.woodstox', name: 'stax2-api', version: '4.0.0'
+    implementation group: 'org.codehaus.woodstox', name: 'woodstox-core-asl', version: '4.4.1'
+
+    implementation 'org.lmdbjava:lmdbjava:0.9.0'
+    implementation 'de.ruedigermoeller:fst:2.56'
+    implementation 'org.xerial.snappy:snappy-java:1.1.7.2'
+
+    implementation 'org.elasticsearch.client:elasticsearch-rest-high-level-client:7.17.2'
+    implementation 'co.elastic.clients:elasticsearch-java:8.12.1'
+    implementation ('org.apache.solr:solr-solrj:9.5.0') {
+        exclude group: 'org.eclipse.jetty', module: 'jetty-http2'
+        exclude group: 'org.eclipse.jetty.http2', module: 'http2-client'
+    }
+
+    implementation 'org.apache.commons:commons-collections4:4.1'
+    implementation 'commons-beanutils:commons-beanutils:1.9.4'
+    implementation 'commons-io:commons-io:2.6'
+    implementation group: 'org.apache.commons', name: 'commons-compress', version: '1.20'
+
+    implementation group: 'org.tukaani', name: 'xz', version: '1.8'
+
+    implementation "com.fasterxml.jackson.core:jackson-core:2.14.3"
+    implementation "com.fasterxml.jackson.core:jackson-databind:2.14.3"
+    implementation "com.fasterxml.jackson.module:jackson-module-afterburner:2.14.3"
+    implementation "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.14.3"
+
+    implementation group: 'com.opencsv', name: 'opencsv', version: '5.0'
+}
+
+distributions {
+    shadow {
+        /*contents {
+            from(new File(rootProject.rootDir, "data/config")) {
+                into "data/config"
+            }
+        }*/
+    }
+}
+
+shadowJar {
+    classifier = 'onejar'
+    mergeServiceFiles()
+    zip64 true
+    manifest {
+        attributes 'Main-Class': mainClassName
+    }
+    exclude("logback.xml")
+    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
+}
+
+distTar.enabled = false
+distZip.enabled = false
+shadowDistTar.enabled = false
+
+artifacts {
+    archives shadowJar
+}
+
+// arguments for exec tasks
+ext.getArg = { propName, defaultVal ->
+    return project.hasProperty(propName) ? project.getProperty(propName) : defaultVal;
+}
+
+task server(dependsOn: 'classes', type: JavaExec, group: 'service') {
+    main = 'com.scienceminer.glutton.web.LookupServiceApplication'
+    classpath = sourceSets.main.runtimeClasspath
+    args 'server', 'config/glutton.yml'
+    if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
+        jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.nio=ALL-UNNAMED", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens", "java.base/java.text=ALL-UNNAMED", "--add-opens", "java.base/java.net=ALL-UNNAMED", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "--add-opens", "java.base/java.math=ALL-UNNAMED", "--add-opens", "java.base/java.util=ALL-UNNAMED", "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED" 
+    } else {
+        jvmArgs '-Xmx3072m'
+    }
+}
+
+task crossref(dependsOn: 'classes', type: JavaExec, group: 'data') {
+    main = 'com.scienceminer.glutton.web.LookupServiceApplication'
+    classpath = sourceSets.main.runtimeClasspath
+    args 'crossref', '--input', getArg('input', ''), getArg('config', 'config/glutton.yml') 
+    if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
+        jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.nio=ALL-UNNAMED", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens", "java.base/java.text=ALL-UNNAMED", "--add-opens", "java.base/java.net=ALL-UNNAMED", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "--add-opens", "java.base/java.math=ALL-UNNAMED", "--add-opens", "java.base/java.util=ALL-UNNAMED", "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED"
+    } else {
+        jvmArgs '-Xmx3072m'
+    }
+}
+
+task gap_crossref(dependsOn: 'classes', type: JavaExec, group: 'data') {
+    main = 'com.scienceminer.glutton.web.LookupServiceApplication'
+    classpath = sourceSets.main.runtimeClasspath
+    args 'gap_crossref', getArg('config', 'config/glutton.yml') 
+    if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
+        jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.nio=ALL-UNNAMED", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens", "java.base/java.text=ALL-UNNAMED", "--add-opens", "java.base/java.net=ALL-UNNAMED", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "--add-opens", "java.base/java.math=ALL-UNNAMED", "--add-opens", "java.base/java.util=ALL-UNNAMED", "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED"
+    } else {
+        jvmArgs '-Xmx3072m'
+    }
+}
+
+task hal(dependsOn: 'classes', type: JavaExec, group: 'data') {
+    main = 'com.scienceminer.glutton.web.LookupServiceApplication'
+    classpath = sourceSets.main.runtimeClasspath
+    args 'hal', getArg('config', 'config/glutton.yml') 
+    if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
+        jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.nio=ALL-UNNAMED", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens", "java.base/java.text=ALL-UNNAMED", "--add-opens", "java.base/java.net=ALL-UNNAMED", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "--add-opens", "java.base/java.math=ALL-UNNAMED", "--add-opens", "java.base/java.util=ALL-UNNAMED", "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED"
+    } else {
+        jvmArgs '-Xmx3072m'
+    }
+}
+
+task pmid(dependsOn: 'classes', type: JavaExec, group: 'service') {
+    main = 'com.scienceminer.glutton.web.LookupServiceApplication'
+    classpath = sourceSets.main.runtimeClasspath
+    args 'pmid', getArg('config', 'config/glutton.yml') 
+    if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
+        jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.nio=ALL-UNNAMED", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens", "java.base/java.text=ALL-UNNAMED", "--add-opens", "java.base/java.net=ALL-UNNAMED", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "--add-opens", "java.base/java.math=ALL-UNNAMED", "--add-opens", "java.base/java.util=ALL-UNNAMED", "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED" 
+    } else {
+        jvmArgs '-Xmx3072m'
+    }
+}
+
+task unpaywall(dependsOn: 'classes', type: JavaExec, group: 'data') {
+    main = 'com.scienceminer.glutton.web.LookupServiceApplication'
+    classpath = sourceSets.main.runtimeClasspath
+    args 'unpaywall', '--input', getArg('input', ''), getArg('config', 'config/glutton.yml') 
+    if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
+        jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.nio=ALL-UNNAMED", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens", "java.base/java.text=ALL-UNNAMED", "--add-opens", "java.base/java.net=ALL-UNNAMED", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "--add-opens", "java.base/java.math=ALL-UNNAMED", "--add-opens", "java.base/java.util=ALL-UNNAMED", "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED"
+    } else {
+        jvmArgs '-Xmx3072m'
+    }
+}
+
+task istex(dependsOn: 'classes', type: JavaExec, group: 'data') {
+    main = 'com.scienceminer.glutton.web.LookupServiceApplication'
+    classpath = sourceSets.main.runtimeClasspath
+    args 'istex', '--input', getArg('input', ''), getArg('config', 'config/glutton.yml') 
+    if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
+        jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.nio=ALL-UNNAMED", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens", "java.base/java.text=ALL-UNNAMED", "--add-opens", "java.base/java.net=ALL-UNNAMED", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "--add-opens", "java.base/java.math=ALL-UNNAMED", "--add-opens", "java.base/java.util=ALL-UNNAMED", "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED"
+    } else {
+        jvmArgs '-Xmx3072m'
+    }
+}
+
+// only used it to re-create the search index from scratch, keeping the LMDB unchanged
+task index(dependsOn: 'classes', type: JavaExec, group: 'search') {
+    main = 'com.scienceminer.glutton.web.LookupServiceApplication'
+    classpath = sourceSets.main.runtimeClasspath
+    args 'index', getArg('config', 'config/glutton.yml') 
+    if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
+        jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.nio=ALL-UNNAMED", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens", "java.base/java.text=ALL-UNNAMED", "--add-opens", "java.base/java.net=ALL-UNNAMED", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "--add-opens", "java.base/java.math=ALL-UNNAMED", "--add-opens", "java.base/java.util=ALL-UNNAMED", "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED"
+    } else {
+        jvmArgs '-Xmx3072m'
+    }
+}
+
+// experimental
+task hal_audit(dependsOn: 'classes', type: JavaExec, group: 'search') {
+    main = 'com.scienceminer.glutton.web.LookupServiceApplication'
+    classpath = sourceSets.main.runtimeClasspath
+    args 'hal_audit', getArg('config', 'config/glutton.yml') 
+    if(JavaVersion.current().compareTo(JavaVersion.VERSION_1_8) > 0) {
+        jvmArgs '-Xmx3072m', "--add-opens", "java.base/java.nio=ALL-UNNAMED", "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED", "--add-opens", "java.base/java.text=ALL-UNNAMED", "--add-opens", "java.base/java.net=ALL-UNNAMED", "--add-opens", "java.base/java.lang=ALL-UNNAMED", "--add-opens", "java.base/java.math=ALL-UNNAMED", "--add-opens", "java.base/java.util=ALL-UNNAMED", "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED"
+    } else {
+        jvmArgs '-Xmx3072m'
+    }
+}