Baseline refactoring to support OpenSearch Serverless and OpenSearch Java JDK

pom.xml

@@ -116,6 +116,43 @@ POSSIBILITY OF SUCH DAMAGE.
             <artifactId>tika-core</artifactId>
             <version>1.28.3</version>
         </dependency>
+        <!-- three artifacts for complete configuration parsing -->
+        <dependency>
+          <groupId>jakarta.xml.bind</groupId>
+          <artifactId>jakarta.xml.bind-api</artifactId>
+          <version>4.0.1</version>
+        </dependency>
+        <dependency>
+          <groupId>com.sun.xml.bind</groupId>
+          <artifactId>jaxb-impl</artifactId>
+          <version>4.0.4</version>
+        </dependency>
+        <dependency>
+          <groupId>jakarta.activation</groupId>
+          <artifactId>jakarta.activation-api</artifactId>
+          <version>2.1.2</version>
+        </dependency>
+        <!-- four artifacts for opensearch serverless -->
+        <dependency>
+          <groupId>org.apache.httpcomponents.client5</groupId>
+          <artifactId>httpclient5</artifactId>
+          <version>5.3.1</version>
+        </dependency>
+        <dependency>
+          <groupId>org.opensearch.client</groupId>
+          <artifactId>opensearch-java</artifactId>
+          <version>2.10.0</version>
+        </dependency>
+        <dependency>
+          <groupId>software.amazon.awssdk</groupId>
+          <artifactId>opensearch</artifactId>
+          <version>2.25.31</version>
+        </dependency>
+        <dependency>
+          <groupId>software.amazon.awssdk</groupId>
+          <artifactId>apache-client</artifactId>
+          <version>2.25.31</version>
+        </dependency>
     </dependencies>
 
     <build>

src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java

@@ -0,0 +1,17 @@
+package gov.nasa.pds.registry.common;
+
+import org.apache.http.HttpHost;
+import org.apache.http.client.CredentialsProvider;
+
+public interface ConnectionFactory {
+  public ConnectionFactory clone();
+  public RestClient createRestClient() throws Exception;
+  public CredentialsProvider getCredentials();
+  public org.apache.hc.client5.http.auth.CredentialsProvider getCredentials5();
+  public HttpHost getHost();
+  public org.apache.hc.core5.http.HttpHost getHost5();
+  public String getHostName();
+  public String getIndexName();
+  public boolean isTrustingSelfSigned();
+  public ConnectionFactory setIndexName (String idxName);
+}

src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java

@@ -0,0 +1,29 @@
+package gov.nasa.pds.registry.common;
+
+import java.net.URL;
+import gov.nasa.pds.registry.common.connection.AuthContent;
+import gov.nasa.pds.registry.common.connection.KnownRegistryConnections;
+import gov.nasa.pds.registry.common.connection.UseOpensearchSDK1;
+import gov.nasa.pds.registry.common.connection.UseOpensearchSDK2;
+import gov.nasa.pds.registry.common.connection.RegistryConnectionContent;
+
+public class EstablishConnectionFactory {
+  public static ConnectionFactory from (String urlToRegistryConnection) throws Exception {
+    return EstablishConnectionFactory.from (urlToRegistryConnection, AuthContent.DEFAULT);
+  }
+  public static ConnectionFactory from (String urlToRegistryConnection, String authFile) throws Exception {
+    return EstablishConnectionFactory.from (urlToRegistryConnection, AuthContent.from(authFile));
+  }
+  private static synchronized ConnectionFactory from (String urlToRegistryConnection, AuthContent auth) throws Exception {
+    KnownRegistryConnections.initialzeAppHandler();
+    RegistryConnectionContent conn = RegistryConnectionContent.from (new URL(urlToRegistryConnection));
+
+    if (conn.isDirectConnection()) {
+      if (conn.getServerUrl().getSdk().intValue() == 1) return UseOpensearchSDK1.build(conn.getServerUrl(), auth).setIndexName(conn.getIndex());
+      if (conn.getServerUrl().getSdk().intValue() == 2) return UseOpensearchSDK2.build(conn.getServerUrl(), auth).setIndexName(conn.getIndex());
+      throw new RuntimeException("The SDK version '" + String.valueOf(conn.getServerUrl().getSdk().intValue()) + "' is not supported");
+    }
+    if (conn.isCognitoConnection()) return UseOpensearchSDK2.build(conn.getCognitoClientId(), auth).setIndexName(conn.getIndex());
+    throw new RuntimeException("New XML/Java choices in src/main/resources/registry_connection.xsd that are not handled.");
+  }
+}

src/main/java/gov/nasa/pds/registry/common/Request.java

@@ -0,0 +1,57 @@
+package gov.nasa.pds.registry.common;
+
+import java.util.Collection;
+import java.util.List;
+import gov.nasa.pds.registry.common.util.Tuple;
+
+public interface Request {
+  public interface Bulk { // _bulk
+    enum Refresh { False, True, WaitFor };
+    public void add (String statement, String document); // create, index, update
+    public Bulk buildUpdateStatus(Collection<String> lidvids, String status);
+    public Bulk setIndex(String name);
+    public Bulk setRefresh(Refresh type);
+  }
+  public interface Count { // _count
+    public Count setIndex (String name);
+    public Count setQuery (String q);
+  }
+  public interface DeleteByQuery { // delete_by_query
+    public DeleteByQuery createFilterQuery(String key, String value);
+    public DeleteByQuery createMatchAllQuery();
+    public DeleteByQuery setIndex (String name);
+    public DeleteByQuery setRefresh(boolean state);
+  }
+  public interface Get { // _doc
+    public Get excludeField (String field);
+    public Get excludeFields (List<String> fields);
+    public Get includeField (String field);
+    public Get includeFields (List<String> fields);
+    public Get setId (String id);
+    public Get setIndex (String index);
+  }
+  public interface Mapping { // _mapping
+    public Mapping buildUpdateFieldSchema (Collection<Tuple> pairs);
+    public Mapping setIndex(String name);
+  }
+  public interface MGet extends Get { // _mget
+    public MGet setIds (Collection<String> ids);
+  }
+  public interface Search { // _search
+    public Search all(String sortField, int size, String searchAfter);
+    public Search all(String filterField, String filterValue, String sortField, int size, String searchAfter);
+    public Search buildAlternativeIds(Collection<String> lids);
+    public Search buildGetField(String field_name, String lidvid);
+    public Search buildLatestLidVids(Collection<String> lids);
+    public Search buildListFields(String dataType);
+    public Search buildListLdds (String namespace);
+    public Search buildTermQuery (String fieldname, String value);
+    public Search buildTheseIds(Collection<String> lids);
+    public Search setIndex (String name);
+    public Search setPretty (boolean pretty);
+    public Search setSize (int hitsperpage);
+  }
+  public interface Setting { // _settings
+    public Setting setIndex (String name);
+  }
+}

src/main/java/gov/nasa/pds/registry/common/Response.java

@@ -0,0 +1,62 @@
+package gov.nasa.pds.registry.common;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import gov.nasa.pds.registry.common.es.dao.dd.DataTypeNotFoundException;
+import gov.nasa.pds.registry.common.es.dao.dd.LddInfo;
+import gov.nasa.pds.registry.common.es.dao.dd.LddVersions;
+import gov.nasa.pds.registry.common.util.Tuple;
+
+public interface Response {
+  public interface Bulk {
+    public interface Item {
+      public boolean error();
+      public String id();
+      public String index();
+      public String operation();
+      public String reason();
+      public String result();
+      public int status();
+    }
+    public boolean errors();
+    public List<Item> items();
+    public void logErrors();
+    public long took();
+  }
+  public interface CreatedIndex {
+    public boolean acknowledge();
+    public boolean acknowledgeShards();
+    public String getIndex();
+  }
+  public interface Get {
+    public interface IdSets {
+      public Set<String> lids();
+      public Set<String> lidvids();
+    }
+    public List<Tuple> dataTypes(boolean stringForMissing) throws IOException, DataTypeNotFoundException;    
+    public IdSets ids(); // returns null if nothing is found in returned content
+    public String productClass(); // returns null if product class not in returned content
+    public List<String> refs(); // returns null if nothing is found in returned content
+  }
+  public interface Mapping {
+    public Set<String> fieldNames();
+  }
+  public interface Search {
+    public Map<String,Set<String>> altIds() throws UnsupportedOperationException, IOException;
+    public List<Object> batch() throws UnsupportedOperationException, IOException;
+    public List<Map<String,Object>> documents();
+    public String field(String name) throws NoSuchFieldException; // null means blob not in document and NoSuchFieldException document not found
+    public Set<String> fields() throws UnsupportedOperationException, IOException;
+    public List<String> latestLidvids(); // returns null if nothing is found in returned content
+    public LddVersions lddInfo() throws UnsupportedOperationException, IOException;
+    public List<LddInfo> ldds() throws UnsupportedOperationException, IOException;
+    public Set<String> nonExistingIds(Collection<String> from_ids) throws UnsupportedOperationException, IOException;
+  }
+  public interface Settings {
+    public int replicas();
+    public int shards();
+  }
+}

src/main/java/gov/nasa/pds/registry/common/ResponseException.java

@@ -0,0 +1,10 @@
+package gov.nasa.pds.registry.common;
+
+import java.io.IOException;
+
+abstract public class ResponseException extends IOException {
+  private static final long serialVersionUID = 8629769947735587642L;
+  abstract public String extractErrorMessage();
+  abstract public int statusCode(); // -1 if not known
+  //abstract public Response getResponse();
+}

src/main/java/gov/nasa/pds/registry/common/RestClient.java

@@ -0,0 +1,74 @@
+package gov.nasa.pds.registry.common;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.TreeMap;
+import com.google.gson.Gson;
+import com.google.gson.stream.JsonWriter;
+import gov.nasa.pds.registry.common.util.CloseUtils;
+
+public interface RestClient extends Closeable {
+  public Request.Bulk createBulkRequest();
+  public Request.Count createCountRequest();
+  public Request.DeleteByQuery createDeleteByQuery();
+  public Request.Get createGetRequest();
+  public Request.Mapping createMappingRequest();
+  public Request.MGet createMGetRequest();
+  public Request.Search createSearchRequest();
+  public Request.Setting createSettingRequest();
+  public Response.CreatedIndex create (String indexName, String configAsJson) throws IOException,ResponseException;
+  public void delete (String indexName) throws IOException,ResponseException;
+  public boolean exists (String indexName) throws IOException,ResponseException;
+  public Response.Bulk performRequest(Request.Bulk request) throws IOException,ResponseException;
+  public long performRequest(Request.Count request) throws IOException,ResponseException;
+  public long performRequest(Request.DeleteByQuery request) throws IOException,ResponseException;
+  public Response.Get performRequest(Request.Get request) throws IOException,ResponseException;
+  public Response.Mapping performRequest(Request.Mapping request) throws IOException,ResponseException;
+  public Response.Search performRequest(Request.Search request) throws IOException,ResponseException;
+  public Response.Settings performRequest(Request.Setting request) throws IOException,ResponseException;
+  /**
+   * Build create index request
+   * 
+   * @param schemaFile index schema file
+   * @param shards number of shards
+   * @param replicas number of replicas
+   * @return JSON
+   * @throws Exception Generic exception
+   */
+  @SuppressWarnings({"rawtypes", "unchecked"})
+  public static String createCreateIndexRequest(File schemaFile, int shards, int replicas)
+      throws Exception {
+    // Read schema template
+    FileReader rd = new FileReader(schemaFile);
+    Gson gson = new Gson();
+    Object rootObj = gson.fromJson(rd, Object.class);
+    CloseUtils.close(rd);
+    Object settingsObj = ((Map) rootObj).get("settings");
+    if (settingsObj == null) {
+      settingsObj = new TreeMap();
+    }
+    Object mappingsObj = ((Map) rootObj).get("mappings");
+    if (mappingsObj == null) {
+      throw new Exception("Missing mappings in schema file " + schemaFile.getAbsolutePath());
+    }
+    StringWriter out = new StringWriter();
+    JsonWriter writer = new JsonWriter(out);
+    writer.beginObject();
+    Map settingsMap = (Map) settingsObj;
+    settingsMap.put("number_of_shards", shards);
+    settingsMap.put("number_of_replicas", replicas);
+    // Settings
+    writer.name("settings");
+    gson.toJson(settingsObj, Object.class, writer);
+    // Mappings
+    writer.name("mappings");
+    gson.toJson(mappingsObj, Object.class, writer);
+    writer.endObject();
+    writer.close();
+    return out.toString();
+  }
+}

src/main/java/gov/nasa/pds/registry/common/app/Handler.java

@@ -0,0 +1,32 @@
+package gov.nasa.pds.registry.common.app;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.net.URLStreamHandler;
+
+public class Handler extends URLStreamHandler {
+  @Override
+  protected void parseURL(URL u, String spec, int start, int limit) {
+    /*
+     * Ugly backwards compatibility. Flip any file separator
+     * characters to be forward slashes. This is a nop on Unix
+     * and "fixes" win32 file paths. According to RFC 2396,
+     * only forward slashes may be used to represent hierarchy
+     * separation in a URL but previous releases unfortunately
+     * performed this "fixup" behavior in the file URL parsing code
+     * rather than forcing this to be fixed in the caller of the URL
+     * class where it belongs. Since backslash is an "unwise"
+     * character that would normally be encoded if literally intended
+     * as a non-seperator character the damage of veering away from the
+     * specification is presumably limited.
+     */
+    super.parseURL(u, spec.replace(File.separatorChar, '/'), start, limit);
+  }
+  @Override
+  protected URLConnection openConnection(URL u) throws IOException {
+    System.out.println("here");
+    return null;
+  }
+}