From 76e092f21940b4fdc073853dfc889a3a88a1d624 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 25 Jan 2024 11:06:48 -0800 Subject: [PATCH 01/41] refactoring has started --- .../nasa/pds/registry/common/ConnectionFactory.java | 11 +++++++++++ .../registry/common/EstablishConnectionFactory.java | 10 ++++++++++ .../cfg/{RegistryCfg.java => RegistryCfgxx.java} | 2 +- .../nasa/pds/registry/common/es/dao/DataLoader.java | 10 ++++------ .../common/es/service/CollectionInventoryWriter.java | 7 +++---- .../pds/registry/common/es/service/JsonLddLoader.java | 6 +++--- .../pds/registry/common/es/service/SchemaUpdater.java | 6 +++--- 7 files changed, 35 insertions(+), 17 deletions(-) create mode 100644 src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java rename src/main/java/gov/nasa/pds/registry/common/cfg/{RegistryCfg.java => RegistryCfgxx.java} (87%) diff --git a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java new file mode 100644 index 0000000..b6aaaa3 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java @@ -0,0 +1,11 @@ +package gov.nasa.pds.registry.common; + +import java.net.HttpURLConnection; + +public interface ConnectionFactory { + public HttpURLConnection createConnection(); + public String getHostName(); + public String getIndexName(); + public ConnectionFactory setAPI (String api); + public ConnectionFactory setIndexName (String idxName); +} diff --git a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java new file mode 100644 index 0000000..8de2820 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java @@ -0,0 +1,10 @@ +package gov.nasa.pds.registry.common; + +public class EstablishConnectionFactory { + public static ConnectionFactory directly (String url, String authfile, boolean trustSelfSigned) { + return null; + } + public static ConnectionFactory viaCognito (String clientID, String authfile, boolean trustSelfSigned) { + return null; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/cfg/RegistryCfg.java b/src/main/java/gov/nasa/pds/registry/common/cfg/RegistryCfgxx.java similarity index 87% rename from src/main/java/gov/nasa/pds/registry/common/cfg/RegistryCfg.java rename to src/main/java/gov/nasa/pds/registry/common/cfg/RegistryCfgxx.java index 6fc610c..5fddd0a 100644 --- a/src/main/java/gov/nasa/pds/registry/common/cfg/RegistryCfg.java +++ b/src/main/java/gov/nasa/pds/registry/common/cfg/RegistryCfgxx.java @@ -4,7 +4,7 @@ * Registry (Elasticsearch) configuration * @author karpenko */ -public class RegistryCfg +public class RegistryCfgxx { public String url; public String indexName; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java index 7d9bd91..5e43c8b 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java @@ -19,9 +19,8 @@ import org.apache.logging.log4j.Logger; import com.google.gson.Gson; - +import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.es.client.EsUtils; -import gov.nasa.pds.registry.common.es.client.HttpConnectionFactory; import gov.nasa.pds.registry.common.util.CloseUtils; @@ -41,7 +40,7 @@ public class DataLoader private int totalRecords; private Logger log; - private HttpConnectionFactory conFactory; + private ConnectionFactory conFactory; /** @@ -52,11 +51,10 @@ public class DataLoader * (see Registry Manager documentation for more info) * @throws Exception an exception */ - public DataLoader(String esUrl, String indexName, String authConfigFile) throws Exception + public DataLoader(ConnectionFactory conFactory) throws Exception { log = LogManager.getLogger(this.getClass()); - conFactory = new HttpConnectionFactory(esUrl, indexName, "_bulk?refresh=wait_for"); - conFactory.initAuth(authConfigFile); + conFactory = conFactory.setAPI("_bulk?refresh=wait_for"); } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/service/CollectionInventoryWriter.java b/src/main/java/gov/nasa/pds/registry/common/es/service/CollectionInventoryWriter.java index a102ade..cee75eb 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/service/CollectionInventoryWriter.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/service/CollectionInventoryWriter.java @@ -6,8 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - -import gov.nasa.pds.registry.common.cfg.RegistryCfg; +import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.es.dao.DataLoader; import gov.nasa.pds.registry.common.meta.InventoryBatchReader; import gov.nasa.pds.registry.common.util.CloseUtils; @@ -44,10 +43,10 @@ public class CollectionInventoryWriter /** * Constructor */ - public CollectionInventoryWriter(RegistryCfg cfg) throws Exception + public CollectionInventoryWriter(ConnectionFactory conFact) throws Exception { log = LogManager.getLogger(this.getClass()); - loader = new DataLoader(cfg.url, cfg.indexName + "-refs", cfg.authFile); + loader = new DataLoader(conFact); } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/service/JsonLddLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/service/JsonLddLoader.java index 136342f..3f93cd4 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/service/JsonLddLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/service/JsonLddLoader.java @@ -7,7 +7,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - +import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.dd.LddEsJsonWriter; import gov.nasa.pds.registry.common.dd.LddUtils; import gov.nasa.pds.registry.common.dd.Pds2EsDataTypeMap; @@ -41,12 +41,12 @@ public class JsonLddLoader * @param authFilePath authentication configuration file * @throws Exception an exception */ - public JsonLddLoader(DataDictionaryDao dao, String esUrl, String indexName, String authFilePath) throws Exception + public JsonLddLoader(DataDictionaryDao dao, ConnectionFactory conFact) throws Exception { log = LogManager.getLogger(this.getClass()); dtMap = new Pds2EsDataTypeMap(); - loader = new DataLoader(esUrl, indexName + "-dd", authFilePath); + loader = new DataLoader(conFact.setIndexName(conFact.getIndexName() + "-dd")); this.dao = dao; } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/service/SchemaUpdater.java b/src/main/java/gov/nasa/pds/registry/common/es/service/SchemaUpdater.java index 04339d0..9273df8 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/service/SchemaUpdater.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/service/SchemaUpdater.java @@ -16,7 +16,7 @@ import gov.nasa.pds.registry.common.util.Tuple; import gov.nasa.pds.registry.common.es.dao.dd.DataDictionaryDao; -import gov.nasa.pds.registry.common.cfg.RegistryCfg; +import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.es.dao.schema.SchemaDao; import gov.nasa.pds.registry.common.es.dao.dd.LddVersions; @@ -42,7 +42,7 @@ public class SchemaUpdater * @param cfg Registry (Elasticsearch) configuration * @throws Exception */ - public SchemaUpdater(RegistryCfg cfg, DataDictionaryDao ddDao, SchemaDao schemaDao) throws Exception + public SchemaUpdater(ConnectionFactory conFact, DataDictionaryDao ddDao, SchemaDao schemaDao) throws Exception { log = LogManager.getLogger(this.getClass()); @@ -51,7 +51,7 @@ public SchemaUpdater(RegistryCfg cfg, DataDictionaryDao ddDao, SchemaDao schemaD fileDownloader = new FileDownloader(true); - lddLoader = new JsonLddLoader(ddDao, cfg.url, cfg.indexName, cfg.authFile); + lddLoader = new JsonLddLoader(ddDao, conFact); lddLoader.loadPds2EsDataTypeMap(LddUtils.getPds2EsDataTypeCfgFile("HARVEST_HOME")); } From 6f6c8049e7792ba297928677b2ba6fea74ddc8f6 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 25 Jan 2024 15:22:38 -0800 Subject: [PATCH 02/41] start of a facade Created ConnectionFactory for building the HTTP connection that is currently being done in a more direct use class. The facade should allow cognito go out and get the signed URL without any other part of the code caring. The big question is will the signed URL really work as well as amazon promises. --- .../registry/common/ConnectionFactory.java | 5 +- .../common/EstablishConnectionFactory.java | 33 ++++- .../registry/common/cfg/RegistryCfgxx.java | 12 -- .../common/connection/AuthContent.java | 57 ++++++++ .../registry/common/connection/Direct.java | 80 +++++++++++ .../common/es/client/EsClientFactory.java | 13 +- .../common/es/client/EsRestClientBld.java | 8 +- .../es/client/HttpConnectionFactory.java | 127 ------------------ .../common/es/dao/schema/SchemaDao.java | 4 - .../common/meta/FileMetadataExtractor.java | 10 +- .../common/util/json/BaseNJsonWriter.java | 1 + 11 files changed, 180 insertions(+), 170 deletions(-) delete mode 100644 src/main/java/gov/nasa/pds/registry/common/cfg/RegistryCfgxx.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/Direct.java delete mode 100644 src/main/java/gov/nasa/pds/registry/common/es/client/HttpConnectionFactory.java diff --git a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java index b6aaaa3..37dbc14 100644 --- a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java @@ -1,9 +1,12 @@ package gov.nasa.pds.registry.common; +import java.io.IOException; import java.net.HttpURLConnection; +import org.apache.http.HttpHost; public interface ConnectionFactory { - public HttpURLConnection createConnection(); + public HttpURLConnection createConnection() throws IOException; + public HttpHost getHost(); public String getHostName(); public String getIndexName(); public ConnectionFactory setAPI (String api); diff --git a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java index 8de2820..c26914d 100644 --- a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java @@ -1,10 +1,37 @@ package gov.nasa.pds.registry.common; +import gov.nasa.pds.registry.common.connection.AuthContent; +import gov.nasa.pds.registry.common.connection.Direct; + public class EstablishConnectionFactory { - public static ConnectionFactory directly (String url, String authfile, boolean trustSelfSigned) { - return null; + public static ConnectionFactory directly (String url) throws Exception { + return directly (url, AuthContent.DEFAULT, false); + } + public static ConnectionFactory directly (String url, String authfile) throws Exception { + return directly (url, AuthContent.from(authfile), false); + } + public static ConnectionFactory directly (String url, boolean trustSelfSigned) throws Exception { + return directly (url, AuthContent.DEFAULT, trustSelfSigned); + } + public static ConnectionFactory directly (String url, String authfile, boolean trustSelfSigned) throws Exception { + return directly (url, AuthContent.from(authfile), trustSelfSigned); + } + private static ConnectionFactory directly (String url, AuthContent auth, boolean trustedSelfSigned) throws Exception { + return Direct.build (url, auth, trustedSelfSigned); + } + public static ConnectionFactory viaCognito (String clientID) throws Exception { + return viaCognito (clientID, AuthContent.DEFAULT, false); + } + public static ConnectionFactory viaCognito (String clientID, String authfile) throws Exception { + return viaCognito (clientID, AuthContent.from(authfile), false); + } + public static ConnectionFactory viaCognito (String clientID, boolean trustSelfSigned) throws Exception { + return viaCognito (clientID, AuthContent.DEFAULT, trustSelfSigned); + } + public static ConnectionFactory viaCognito (String clientID, String authfile, boolean trustSelfSigned) throws Exception { + return viaCognito (clientID, AuthContent.from (authfile), trustSelfSigned); } - public static ConnectionFactory viaCognito (String clientID, String authfile, boolean trustSelfSigned) { + private static ConnectionFactory viaCognito (String clientID, AuthContent auth, boolean trustSelfSigned) throws Exception { return null; } } diff --git a/src/main/java/gov/nasa/pds/registry/common/cfg/RegistryCfgxx.java b/src/main/java/gov/nasa/pds/registry/common/cfg/RegistryCfgxx.java deleted file mode 100644 index 5fddd0a..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/cfg/RegistryCfgxx.java +++ /dev/null @@ -1,12 +0,0 @@ -package gov.nasa.pds.registry.common.cfg; - -/** - * Registry (Elasticsearch) configuration - * @author karpenko - */ -public class RegistryCfgxx -{ - public String url; - public String indexName; - public String authFile; -} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java new file mode 100644 index 0000000..c62fe17 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java @@ -0,0 +1,57 @@ +package gov.nasa.pds.registry.common.connection; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import gov.nasa.pds.registry.common.es.client.ClientConstants; +import gov.nasa.pds.registry.common.util.JavaProps; + +public class AuthContent { + final public static AuthContent DEFAULT = new AuthContent(); + final private static Logger LOG = LogManager.getLogger(AuthContent.class); + final private String password; + final private String user; + private String header = null; + + public static AuthContent from (String authfile) throws Exception { + if(authfile == null) { + throw new IllegalArgumentException("null is not an allowable value for authfile. Use AuthContent.DEFAULT in that case."); + } + + JavaProps props = new JavaProps(authfile); + String user = props.getProperty("user"); + String password = props.getProperty("password"); + + if (props.getProperty(ClientConstants.AUTH_TRUST_SELF_SIGNED) != null) { + LOG.warn("The keyword " + ClientConstants.AUTH_TRUST_SELF_SIGNED + + " is no longer used in the authorizaiton file and is being ignored. Please remove " + + ClientConstants.AUTH_TRUST_SELF_SIGNED + + " warning will turn into an error in the future"); + } + + if (user == null || password == null) { + throw new IllegalArgumentException("Must have both 'user' and 'password' in the authorization file: " + authfile); + } + return new AuthContent(password, user); + } + private AuthContent() { + this("admin", "admin"); + } + private AuthContent(String password, String user) { + this.password = password; + this.user = user; + } + public synchronized String getHeader() { + if (this.header == null) { + this.header = "Basic " + Base64.getEncoder().encodeToString((this.getUser() + ":" + this.getPassword()).getBytes(StandardCharsets.UTF_8)); + } + return this.header; + } + public String getPassword() { + return password; + } + public String getUser() { + return user; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java new file mode 100644 index 0000000..c2ad874 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java @@ -0,0 +1,80 @@ +package gov.nasa.pds.registry.common.connection; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import org.apache.http.HttpHost; +import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.es.client.SSLUtils; + +public class Direct implements Cloneable, ConnectionFactory { + final private int timeout = 5000; + final private AuthContent auth; + final private HttpHost host; + final private URL service; + private String api = null; + private String index = null; + + public static Direct build (String url, AuthContent auth, boolean trustSelfSigned) throws Exception { + URL service = new URL(url); + // Trust self-signed certificates + if(trustSelfSigned) + { + SSLContext sslCtx = SSLUtils.createTrustAllContext(); + HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory()); + } + return new Direct (service, auth); + } + + private Direct (URL service, AuthContent auth) { + this.auth = auth; + this.host = new HttpHost(service.getHost(), service.getPort(), service.getProtocol()); + this.service = service; + } + @Override + public ConnectionFactory clone() { + return new Direct(this.service, this.auth).setAPI(this.api).setIndexName(this.index); + } + @Override + public HttpURLConnection createConnection() throws IOException { + String url = this.service.toString(); + if (this.index != null) url += "/" + this.index; + if (this.api != null) url += "/" + this.api; + HttpURLConnection con = (HttpURLConnection)new URL(url).openConnection(); + con.setConnectTimeout(this.timeout); + con.setReadTimeout(this.timeout); + con.setAllowUserInteraction(false); + con.setRequestProperty("Authorization", this.auth.getHeader()); + return con; + } + + @Override + public HttpHost getHost() { + return this.host; + } + + @Override + public String getHostName() { + return this.host.getHostName(); + } + + @Override + public String getIndexName() { + return this.index; + } + + @Override + public ConnectionFactory setAPI(String api) { + this.api = api; + return this; + } + + @Override + public ConnectionFactory setIndexName(String idxName) { + this.index = idxName; + return this; + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/EsClientFactory.java b/src/main/java/gov/nasa/pds/registry/common/es/client/EsClientFactory.java index e6730ba..4388352 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/EsClientFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/client/EsClientFactory.java @@ -2,7 +2,7 @@ import org.elasticsearch.client.RestClient; -import gov.nasa.pds.registry.common.util.JavaProps; +import gov.nasa.pds.registry.common.ConnectionFactory; /** @@ -19,16 +19,9 @@ public class EsClientFactory * @return Elasticsearch rest client instance. * @throws Exception an exception */ - public static RestClient createRestClient(String esUrl, String authPath) throws Exception + public static RestClient createRestClient(ConnectionFactory conFact) throws Exception { - EsRestClientBld bld = new EsRestClientBld(esUrl); - - if(authPath != null) - { - JavaProps props = new JavaProps(authPath); - bld.configureAuth(props); - } - + EsRestClientBld bld = new EsRestClientBld(conFact); return bld.build(); } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java b/src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java index 791d542..354fbe6 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java @@ -1,9 +1,8 @@ package gov.nasa.pds.registry.common.es.client; -import org.apache.http.HttpHost; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; - +import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.util.JavaProps; @@ -24,10 +23,9 @@ public class EsRestClientBld * @param url Elasticsearch URL, e.g., "http://localhost:9200" * @throws Exception an exception */ - public EsRestClientBld(String url) throws Exception + public EsRestClientBld(ConnectionFactory conFact) throws Exception { - HttpHost host = EsUtils.parseEsUrl(url); - bld = RestClient.builder(host); + bld = RestClient.builder(conFact.getHost()); clientCB = new ClientConfigCB(); reqCB = new RequestConfigCB(); diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/HttpConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/es/client/HttpConnectionFactory.java deleted file mode 100644 index 991d7f7..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/HttpConnectionFactory.java +++ /dev/null @@ -1,127 +0,0 @@ -package gov.nasa.pds.registry.common.es.client; - - -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.Base64; - -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; - -import org.apache.http.HttpHost; - -import gov.nasa.pds.registry.common.util.JavaProps; - - -/** - * Factory class to create HTTP connections. - * - * @author karpenko - */ -public class HttpConnectionFactory -{ - private int timeout = 5000; - private URL url; - private HttpHost host; - private String authHeader; - - - /** - * Constructor. - * @param esUrl Elasticsearch URL, e.g., "http://localhost:9200" - * @param indexName Elasticsearch index name. - * @param api API name, e.g., "_bulk". - * @throws Exception an exception - */ - public HttpConnectionFactory(String esUrl, String indexName, String api) throws Exception - { - HttpHost host = EsUtils.parseEsUrl(esUrl); - this.url = new URL(host.toURI() + "/" + indexName + "/" + api); - } - - - /** - * Create HTTP connection - * @return HTTP connection - * @throws Exception an exception - */ - public HttpURLConnection createConnection() throws Exception - { - HttpURLConnection con = (HttpURLConnection)url.openConnection(); - con.setConnectTimeout(timeout); - con.setReadTimeout(timeout); - con.setAllowUserInteraction(false); - - if(authHeader != null) - { - con.setRequestProperty("Authorization", authHeader); - } - - return con; - } - - - /** - * Set connection timeout in seconds. - * - * @param timeoutSec timeout in seconds - */ - public void setTimeoutSec(int timeoutSec) - { - if(timeoutSec <= 0) throw new IllegalArgumentException("Timeout should be > 0"); - this.timeout = timeoutSec * 1000; - } - - - /** - * Get host name - * @return host name - */ - public String getHostName() - { - return host.getHostName(); - } - - - /** - * Set user name and password for basic authentication - * @param user user name - * @param pass password - */ - public void setBasicAuthentication(String user, String pass) - { - String auth = user + ":" + pass; - String b64auth = Base64.getEncoder().encodeToString(auth.getBytes(StandardCharsets.UTF_8)); - this.authHeader = "Basic " + b64auth; - } - - - /** - * Setup authentication parameters and TLS/SSL. - * @param authConfigFile Authentication configuration file. - * @throws Exception an exception - */ - public void initAuth(String authConfigFile) throws Exception - { - if(authConfigFile == null) return; - - JavaProps props = new JavaProps(authConfigFile); - - // Trust self-signed certificates - if(Boolean.TRUE.equals(props.getBoolean(ClientConstants.AUTH_TRUST_SELF_SIGNED))) - { - SSLContext sslCtx = SSLUtils.createTrustAllContext(); - HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory()); - } - - // Basic authentication - String user = props.getProperty("user"); - String pass = props.getProperty("password"); - if(user != null && pass != null) - { - setBasicAuthentication(user, pass); - } - } - -} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java index 6b87295..9b11d47 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java @@ -3,8 +3,6 @@ import java.util.List; import java.util.Set; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; @@ -20,7 +18,6 @@ */ public class SchemaDao { - private Logger log; private RestClient client; private String indexName; @@ -32,7 +29,6 @@ public class SchemaDao */ public SchemaDao(RestClient client, String indexName) { - log = LogManager.getLogger(this.getClass()); this.client = client; this.indexName = indexName; } diff --git a/src/main/java/gov/nasa/pds/registry/common/meta/FileMetadataExtractor.java b/src/main/java/gov/nasa/pds/registry/common/meta/FileMetadataExtractor.java index c7b3288..6f79040 100644 --- a/src/main/java/gov/nasa/pds/registry/common/meta/FileMetadataExtractor.java +++ b/src/main/java/gov/nasa/pds/registry/common/meta/FileMetadataExtractor.java @@ -16,8 +16,6 @@ import java.util.zip.DeflaterOutputStream; import org.apache.commons.codec.binary.Hex; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.tika.Tika; import org.json.JSONObject; import org.json.XML; @@ -30,9 +28,7 @@ * @author karpenko */ public class FileMetadataExtractor -{ - private Logger log; - +{ private MessageDigest md5Digest; private byte[] buf; private Tika tika; @@ -47,9 +43,7 @@ public class FileMetadataExtractor * @throws Exception and exception */ public FileMetadataExtractor() throws Exception - { - log = LogManager.getLogger(this.getClass()); - + { md5Digest = MessageDigest.getInstance("MD5"); buf = new byte[1024 * 16]; tika = new Tika(); diff --git a/src/main/java/gov/nasa/pds/registry/common/util/json/BaseNJsonWriter.java b/src/main/java/gov/nasa/pds/registry/common/util/json/BaseNJsonWriter.java index 339dc4d..31346c1 100644 --- a/src/main/java/gov/nasa/pds/registry/common/util/json/BaseNJsonWriter.java +++ b/src/main/java/gov/nasa/pds/registry/common/util/json/BaseNJsonWriter.java @@ -18,6 +18,7 @@ * * @param A data record to write. */ +@SuppressWarnings("hiding") public abstract class BaseNJsonWriter implements Closeable { protected FileWriter writer; From 7bd3eb293a29aaa3905adde5e0b857e7822864ca Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 25 Jan 2024 15:31:58 -0800 Subject: [PATCH 03/41] clean up tests --- src/test/java/dao/TestDataDictionaryDao.java | 10 +++++----- src/test/java/dao/TestProductDao.java | 4 ++-- src/test/java/tt/TestRestClient.java | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/java/dao/TestDataDictionaryDao.java b/src/test/java/dao/TestDataDictionaryDao.java index e4e7597..93bcb0f 100644 --- a/src/test/java/dao/TestDataDictionaryDao.java +++ b/src/test/java/dao/TestDataDictionaryDao.java @@ -5,7 +5,7 @@ import java.util.Set; import org.elasticsearch.client.RestClient; - +import gov.nasa.pds.registry.common.EstablishConnectionFactory; import gov.nasa.pds.registry.common.es.client.EsClientFactory; import gov.nasa.pds.registry.common.es.dao.dd.DataDictionaryDao; import gov.nasa.pds.registry.common.es.dao.dd.LddInfo; @@ -25,7 +25,7 @@ public static void main(String[] args) throws Exception private static void testListBooleanFields() throws Exception { - RestClient esClient = EsClientFactory.createRestClient("http://localhost:9200", null); + RestClient esClient = EsClientFactory.createRestClient(EstablishConnectionFactory.directly("http://localhost:9200")); try { @@ -44,7 +44,7 @@ private static void testListBooleanFields() throws Exception private static void testListDateFields() throws Exception { - RestClient esClient = EsClientFactory.createRestClient("http://localhost:9200", null); + RestClient esClient = EsClientFactory.createRestClient(EstablishConnectionFactory.directly("http://localhost:9200")); try { @@ -63,7 +63,7 @@ private static void testListDateFields() throws Exception private static void testListLdds() throws Exception { - RestClient esClient = EsClientFactory.createRestClient("http://localhost:9200", null); + RestClient esClient = EsClientFactory.createRestClient(EstablishConnectionFactory.directly("http://localhost:9200")); try { @@ -86,7 +86,7 @@ private static void testListLdds() throws Exception private static void testGetLddInfo() throws Exception { - RestClient esClient = EsClientFactory.createRestClient("http://localhost:9200", null); + RestClient esClient = EsClientFactory.createRestClient(EstablishConnectionFactory.directly("http://localhost:9200")); try { diff --git a/src/test/java/dao/TestProductDao.java b/src/test/java/dao/TestProductDao.java index 7a71c93..5854f8b 100644 --- a/src/test/java/dao/TestProductDao.java +++ b/src/test/java/dao/TestProductDao.java @@ -4,7 +4,7 @@ import java.util.List; import org.elasticsearch.client.RestClient; - +import gov.nasa.pds.registry.common.EstablishConnectionFactory; import gov.nasa.pds.registry.common.es.client.EsClientFactory; import gov.nasa.pds.registry.common.es.dao.ProductDao; import gov.nasa.pds.registry.common.util.CloseUtils; @@ -22,7 +22,7 @@ public static void main(String[] args) throws Exception try { - esClient = EsClientFactory.createRestClient("localhost", null); + esClient = EsClientFactory.createRestClient(EstablishConnectionFactory.directly("localhost")); ProductDao dao = new ProductDao(esClient, "registry"); //testUpdateArchiveStatus(dao); diff --git a/src/test/java/tt/TestRestClient.java b/src/test/java/tt/TestRestClient.java index 9f0d1db..3a42aaa 100644 --- a/src/test/java/tt/TestRestClient.java +++ b/src/test/java/tt/TestRestClient.java @@ -1,7 +1,7 @@ package tt; import org.elasticsearch.client.RestClient; - +import gov.nasa.pds.registry.common.EstablishConnectionFactory; import gov.nasa.pds.registry.common.es.client.EsClientFactory; @@ -10,7 +10,7 @@ public class TestRestClient public static void main(String[] args) throws Exception { - RestClient client = EsClientFactory.createRestClient("localhost", null); + RestClient client = EsClientFactory.createRestClient(EstablishConnectionFactory.directly("localhost")); client.close(); } From e29de52dbd3ed7d5cd0b34d55e925e7a043ade87 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Fri, 26 Jan 2024 10:38:08 -0800 Subject: [PATCH 04/41] some fixes for ESClient authentication Had to extend the ConnectionFactory just a bit to support ESClient authentication. In do so, it seems like creatng the ECClient may become the ConnectionFactory job as we move into cognito. For now, it sufficed to keep it where is. Probably need a common.connection.es, common.connection.os, common.connection.cognito to complete the facade an move code around some more. Really should junk es since it cannot live side-by-side with os (opensearch). --- .../registry/common/ConnectionFactory.java | 3 ++ .../common/connection/AuthContent.java | 10 ++++++ .../registry/common/connection/Direct.java | 31 +++++++++++----- .../common/es/client/ClientConfigCB.java | 22 +++--------- .../common/es/client/EsRestClientBld.java | 36 +++---------------- .../registry/common/es/dao/DataLoader.java | 2 +- 6 files changed, 45 insertions(+), 59 deletions(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java index 37dbc14..2e09e41 100644 --- a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java @@ -3,12 +3,15 @@ import java.io.IOException; import java.net.HttpURLConnection; import org.apache.http.HttpHost; +import org.apache.http.client.CredentialsProvider; public interface ConnectionFactory { public HttpURLConnection createConnection() throws IOException; + public CredentialsProvider getCredentials(); public HttpHost getHost(); public String getHostName(); public String getIndexName(); + public boolean isTrustingSelfSigned(); public ConnectionFactory setAPI (String api); public ConnectionFactory setIndexName (String idxName); } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java index c62fe17..daffb5c 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java @@ -2,6 +2,10 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import gov.nasa.pds.registry.common.es.client.ClientConstants; @@ -48,6 +52,12 @@ public synchronized String getHeader() { } return this.header; } + public CredentialsProvider getCredentials() { + CredentialsProvider creds; + creds = new BasicCredentialsProvider(); + creds.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(this.getUser(), this.getPassword())); + return creds; + } public String getPassword() { return password; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java index c2ad874..5ebc4c2 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java @@ -6,10 +6,12 @@ import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; import org.apache.http.HttpHost; +import org.apache.http.client.CredentialsProvider; import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.es.client.SSLUtils; public class Direct implements Cloneable, ConnectionFactory { + final private boolean veryTrusting; final private int timeout = 5000; final private AuthContent auth; final private HttpHost host; @@ -25,17 +27,18 @@ public static Direct build (String url, AuthContent auth, boolean trustSelfSigne SSLContext sslCtx = SSLUtils.createTrustAllContext(); HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory()); } - return new Direct (service, auth); + return new Direct (service, auth, trustSelfSigned); } - private Direct (URL service, AuthContent auth) { + private Direct (URL service, AuthContent auth, boolean trustSelfSigned) { this.auth = auth; this.host = new HttpHost(service.getHost(), service.getPort(), service.getProtocol()); this.service = service; + this.veryTrusting = trustSelfSigned; } @Override public ConnectionFactory clone() { - return new Direct(this.service, this.auth).setAPI(this.api).setIndexName(this.index); + return new Direct(this.service, this.auth, this.veryTrusting).setAPI(this.api).setIndexName(this.index); } @Override public HttpURLConnection createConnection() throws IOException { @@ -49,32 +52,42 @@ public HttpURLConnection createConnection() throws IOException { con.setRequestProperty("Authorization", this.auth.getHeader()); return con; } - @Override public HttpHost getHost() { return this.host; } - @Override public String getHostName() { return this.host.getHostName(); } - @Override public String getIndexName() { return this.index; } - @Override public ConnectionFactory setAPI(String api) { this.api = api; return this; } - @Override public ConnectionFactory setIndexName(String idxName) { this.index = idxName; return this; } - + @Override + public String toString() { + String me = "Direct to " + this.service.getProtocol() + "://" + this.service.getHost(); + if (0 <= this.service.getPort()) me += ":" + this.service.getPort(); + me += " using index '" + String.valueOf(this.index) + "'"; + me += " to call API '" + String.valueOf(this.api) + "'"; + return me; + } + @Override + public CredentialsProvider getCredentials() { + return this.auth.getCredentials(); + } + @Override + public boolean isTrustingSelfSigned() { + return this.veryTrusting; + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/ClientConfigCB.java b/src/main/java/gov/nasa/pds/registry/common/es/client/ClientConfigCB.java index fb563e2..eeb06ff 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/ClientConfigCB.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/client/ClientConfigCB.java @@ -2,11 +2,8 @@ import javax.net.ssl.SSLContext; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; @@ -25,6 +22,11 @@ public class ClientConfigCB implements RestClientBuilder.HttpClientConfigCallbac private CredentialsProvider credProvider; + public void setCredProvider(CredentialsProvider credProvider) { + this.credProvider = credProvider; + } + + /** * Constructor */ @@ -43,20 +45,6 @@ public void setTrustSelfSignedCert(boolean b) } - /** - * Set user name and password for basic authentication. - * @param user user name - * @param pass password - */ - public void setUserPass(String user, String pass) - { - if(user == null || pass == null) return; - - credProvider = new BasicCredentialsProvider(); - credProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, pass)); - } - - @Override public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java b/src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java index 354fbe6..c091f74 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java @@ -3,7 +3,6 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import gov.nasa.pds.registry.common.ConnectionFactory; -import gov.nasa.pds.registry.common.util.JavaProps; /** @@ -16,8 +15,6 @@ public class EsRestClientBld private RestClientBuilder bld; private ClientConfigCB clientCB; private RequestConfigCB reqCB; - - /** * Constructor. * @param url Elasticsearch URL, e.g., "http://localhost:9200" @@ -28,10 +25,10 @@ public EsRestClientBld(ConnectionFactory conFact) throws Exception bld = RestClient.builder(conFact.getHost()); clientCB = new ClientConfigCB(); + clientCB.setCredProvider(conFact.getCredentials()); + clientCB.setTrustSelfSignedCert(conFact.isTrustingSelfSigned()); reqCB = new RequestConfigCB(); - } - - + } /** * Build the Elasticsearch rest client * @return Elasticsearch rest client @@ -42,30 +39,5 @@ public RestClient build() bld.setRequestConfigCallback(reqCB); return bld.build(); - } - - - /** - * Configure authentication - * @param props properties - * @throws Exception an exception - */ - public void configureAuth(JavaProps props) throws Exception - { - if(props == null) return; - - // Trust self-signed certificates - if(Boolean.TRUE.equals(props.getBoolean(ClientConstants.AUTH_TRUST_SELF_SIGNED))) - { - clientCB.setTrustSelfSignedCert(true); - } - - // Basic authentication - String user = props.getProperty("user"); - String pass = props.getProperty("password"); - if(user != null && pass != null) - { - clientCB.setUserPass(user, pass); - } - } + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java index 5e43c8b..06f7066 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java @@ -54,7 +54,7 @@ public class DataLoader public DataLoader(ConnectionFactory conFactory) throws Exception { log = LogManager.getLogger(this.getClass()); - conFactory = conFactory.setAPI("_bulk?refresh=wait_for"); + this.conFactory = conFactory.setAPI("_bulk?refresh=wait_for"); } From 246179a87e11e09020b4900d163168629247c021 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Fri, 26 Jan 2024 10:50:28 -0800 Subject: [PATCH 05/41] refactory es to start hiding it --- .../gov/nasa/pds/registry/common/connection/AuthContent.java | 2 +- .../java/gov/nasa/pds/registry/common/connection/Direct.java | 2 +- .../common/{es/client => connection/es}/ClientConfigCB.java | 2 +- .../common/{es/client => connection/es}/ClientConstants.java | 2 +- .../common/{es/client => connection/es}/DebugUtils.java | 2 +- .../common/{es/client => connection/es}/EsClientFactory.java | 2 +- .../common/{es/client => connection/es}/EsRestClientBld.java | 2 +- .../registry/common/{es/client => connection/es}/EsUtils.java | 2 +- .../common/{es/client => connection/es}/RequestConfigCB.java | 2 +- .../registry/common/{es/client => connection/es}/SSLUtils.java | 2 +- .../{es/client => connection/es}/SearchResponseParser.java | 2 +- .../common/{es/client => connection/es}/TrustAllManager.java | 2 +- .../java/gov/nasa/pds/registry/common/es/dao/DataLoader.java | 2 +- .../nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java | 3 +-- src/test/java/dao/TestDataDictionaryDao.java | 2 +- src/test/java/dao/TestProductDao.java | 2 +- src/test/java/tt/TestRestClient.java | 2 +- 17 files changed, 17 insertions(+), 18 deletions(-) rename src/main/java/gov/nasa/pds/registry/common/{es/client => connection/es}/ClientConfigCB.java (97%) rename src/main/java/gov/nasa/pds/registry/common/{es/client => connection/es}/ClientConstants.java (77%) rename src/main/java/gov/nasa/pds/registry/common/{es/client => connection/es}/DebugUtils.java (94%) rename src/main/java/gov/nasa/pds/registry/common/{es/client => connection/es}/EsClientFactory.java (93%) rename src/main/java/gov/nasa/pds/registry/common/{es/client => connection/es}/EsRestClientBld.java (95%) rename src/main/java/gov/nasa/pds/registry/common/{es/client => connection/es}/EsUtils.java (98%) rename src/main/java/gov/nasa/pds/registry/common/{es/client => connection/es}/RequestConfigCB.java (95%) rename src/main/java/gov/nasa/pds/registry/common/{es/client => connection/es}/SSLUtils.java (92%) rename src/main/java/gov/nasa/pds/registry/common/{es/client => connection/es}/SearchResponseParser.java (98%) rename src/main/java/gov/nasa/pds/registry/common/{es/client => connection/es}/TrustAllManager.java (94%) diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java index daffb5c..7047cdf 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java @@ -8,7 +8,7 @@ import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import gov.nasa.pds.registry.common.es.client.ClientConstants; +import gov.nasa.pds.registry.common.connection.es.ClientConstants; import gov.nasa.pds.registry.common.util.JavaProps; public class AuthContent { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java index 5ebc4c2..0dc4d2f 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java @@ -8,7 +8,7 @@ import org.apache.http.HttpHost; import org.apache.http.client.CredentialsProvider; import gov.nasa.pds.registry.common.ConnectionFactory; -import gov.nasa.pds.registry.common.es.client.SSLUtils; +import gov.nasa.pds.registry.common.connection.es.SSLUtils; public class Direct implements Cloneable, ConnectionFactory { final private boolean veryTrusting; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/ClientConfigCB.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConfigCB.java similarity index 97% rename from src/main/java/gov/nasa/pds/registry/common/es/client/ClientConfigCB.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConfigCB.java index eeb06ff..12d9eaa 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/ClientConfigCB.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConfigCB.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; import javax.net.ssl.SSLContext; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/ClientConstants.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConstants.java similarity index 77% rename from src/main/java/gov/nasa/pds/registry/common/es/client/ClientConstants.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConstants.java index 2e41f94..73f952a 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/ClientConstants.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConstants.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; /** * Constants used by different classes. diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/DebugUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/DebugUtils.java similarity index 94% rename from src/main/java/gov/nasa/pds/registry/common/es/client/DebugUtils.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/DebugUtils.java index ffc1e5d..246a5f1 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/DebugUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/DebugUtils.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; import java.io.BufferedReader; import java.io.IOException; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/EsClientFactory.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsClientFactory.java similarity index 93% rename from src/main/java/gov/nasa/pds/registry/common/es/client/EsClientFactory.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/EsClientFactory.java index 4388352..8d71889 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/EsClientFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsClientFactory.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; import org.elasticsearch.client.RestClient; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsRestClientBld.java similarity index 95% rename from src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/EsRestClientBld.java index c091f74..53e83f3 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsRestClientBld.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/EsUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java similarity index 98% rename from src/main/java/gov/nasa/pds/registry/common/es/client/EsUtils.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java index 5aefcdf..3044051 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/EsUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; import java.util.List; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/RequestConfigCB.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestConfigCB.java similarity index 95% rename from src/main/java/gov/nasa/pds/registry/common/es/client/RequestConfigCB.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/RequestConfigCB.java index 9ed11b3..bd86dc5 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/RequestConfigCB.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestConfigCB.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; import org.apache.http.client.config.RequestConfig; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/SSLUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SSLUtils.java similarity index 92% rename from src/main/java/gov/nasa/pds/registry/common/es/client/SSLUtils.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/SSLUtils.java index 193b3ff..ca16ca5 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/SSLUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SSLUtils.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; import java.security.SecureRandom; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/SearchResponseParser.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchResponseParser.java similarity index 98% rename from src/main/java/gov/nasa/pds/registry/common/es/client/SearchResponseParser.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/SearchResponseParser.java index 21e9f48..3e9835f 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/SearchResponseParser.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchResponseParser.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; import java.io.InputStream; import java.io.InputStreamReader; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/TrustAllManager.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/TrustAllManager.java similarity index 94% rename from src/main/java/gov/nasa/pds/registry/common/es/client/TrustAllManager.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/TrustAllManager.java index 2c0ecc2..fea7094 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/TrustAllManager.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/TrustAllManager.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; import java.security.cert.CertificateException; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java index 06f7066..1e2f544 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java @@ -20,7 +20,7 @@ import com.google.gson.Gson; import gov.nasa.pds.registry.common.ConnectionFactory; -import gov.nasa.pds.registry.common.es.client.EsUtils; +import gov.nasa.pds.registry.common.connection.es.EsUtils; import gov.nasa.pds.registry.common.util.CloseUtils; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java index 719e57f..9a6343a 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java @@ -13,8 +13,7 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; - -import gov.nasa.pds.registry.common.es.client.SearchResponseParser; +import gov.nasa.pds.registry.common.connection.es.SearchResponseParser; import gov.nasa.pds.registry.common.es.dao.schema.SchemaRequestBuilder; import gov.nasa.pds.registry.common.util.Tuple; diff --git a/src/test/java/dao/TestDataDictionaryDao.java b/src/test/java/dao/TestDataDictionaryDao.java index 93bcb0f..6041d59 100644 --- a/src/test/java/dao/TestDataDictionaryDao.java +++ b/src/test/java/dao/TestDataDictionaryDao.java @@ -6,7 +6,7 @@ import org.elasticsearch.client.RestClient; import gov.nasa.pds.registry.common.EstablishConnectionFactory; -import gov.nasa.pds.registry.common.es.client.EsClientFactory; +import gov.nasa.pds.registry.common.connection.es.EsClientFactory; import gov.nasa.pds.registry.common.es.dao.dd.DataDictionaryDao; import gov.nasa.pds.registry.common.es.dao.dd.LddInfo; import gov.nasa.pds.registry.common.es.dao.dd.LddVersions; diff --git a/src/test/java/dao/TestProductDao.java b/src/test/java/dao/TestProductDao.java index 5854f8b..56d001f 100644 --- a/src/test/java/dao/TestProductDao.java +++ b/src/test/java/dao/TestProductDao.java @@ -5,7 +5,7 @@ import org.elasticsearch.client.RestClient; import gov.nasa.pds.registry.common.EstablishConnectionFactory; -import gov.nasa.pds.registry.common.es.client.EsClientFactory; +import gov.nasa.pds.registry.common.connection.es.EsClientFactory; import gov.nasa.pds.registry.common.es.dao.ProductDao; import gov.nasa.pds.registry.common.util.CloseUtils; import tt.TestLogConfigurator; diff --git a/src/test/java/tt/TestRestClient.java b/src/test/java/tt/TestRestClient.java index 3a42aaa..54e2fc1 100644 --- a/src/test/java/tt/TestRestClient.java +++ b/src/test/java/tt/TestRestClient.java @@ -2,7 +2,7 @@ import org.elasticsearch.client.RestClient; import gov.nasa.pds.registry.common.EstablishConnectionFactory; -import gov.nasa.pds.registry.common.es.client.EsClientFactory; +import gov.nasa.pds.registry.common.connection.es.EsClientFactory; public class TestRestClient From 3723706ce06b0bd440e36f474fa13c6b9561a95a Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 30 Jan 2024 10:01:06 -0800 Subject: [PATCH 06/41] refactor code to isolate elasticsearch dependencies --- .../common/connection/AuthContent.java | 1 - .../connection/{es => }/ClientConstants.java | 4 +- .../registry/common/connection/Direct.java | 1 - .../common/connection/{es => }/SSLUtils.java | 4 +- .../connection/{es => }/TrustAllManager.java | 4 +- .../common/connection/es/ClientConfigCB.java | 2 +- .../common/connection/es/DebugUtils.java | 2 +- .../common/connection/es/EsClientFactory.java | 2 +- .../common/connection/es/EsRestClientBld.java | 2 +- .../common/connection/es/EsUtils.java | 42 ++----------------- .../common/connection/es/RequestConfigCB.java | 2 +- .../registry/common/es/dao/DataLoader.java | 6 +-- .../common/es/dao/dd/DataDictionaryDao.java | 2 +- .../es => util}/SearchResponseParser.java | 38 ++++++++++++++++- 14 files changed, 54 insertions(+), 58 deletions(-) rename src/main/java/gov/nasa/pds/registry/common/connection/{es => }/ClientConstants.java (65%) rename src/main/java/gov/nasa/pds/registry/common/connection/{es => }/SSLUtils.java (90%) rename src/main/java/gov/nasa/pds/registry/common/connection/{es => }/TrustAllManager.java (87%) rename src/main/java/gov/nasa/pds/registry/common/{connection/es => util}/SearchResponseParser.java (81%) diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java index 7047cdf..b8a90ec 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java @@ -8,7 +8,6 @@ import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import gov.nasa.pds.registry.common.connection.es.ClientConstants; import gov.nasa.pds.registry.common.util.JavaProps; public class AuthContent { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConstants.java b/src/main/java/gov/nasa/pds/registry/common/connection/ClientConstants.java similarity index 65% rename from src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConstants.java rename to src/main/java/gov/nasa/pds/registry/common/connection/ClientConstants.java index 73f952a..c8a5db5 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConstants.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/ClientConstants.java @@ -1,11 +1,11 @@ -package gov.nasa.pds.registry.common.connection.es; +package gov.nasa.pds.registry.common.connection; /** * Constants used by different classes. * * @author karpenko */ -public class ClientConstants +class ClientConstants { public static final String AUTH_TRUST_SELF_SIGNED = "trust.self-signed"; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java index 0dc4d2f..5d00bbd 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java @@ -8,7 +8,6 @@ import org.apache.http.HttpHost; import org.apache.http.client.CredentialsProvider; import gov.nasa.pds.registry.common.ConnectionFactory; -import gov.nasa.pds.registry.common.connection.es.SSLUtils; public class Direct implements Cloneable, ConnectionFactory { final private boolean veryTrusting; diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SSLUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/SSLUtils.java similarity index 90% rename from src/main/java/gov/nasa/pds/registry/common/connection/es/SSLUtils.java rename to src/main/java/gov/nasa/pds/registry/common/connection/SSLUtils.java index ca16ca5..eaddc09 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/SSLUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/SSLUtils.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.connection.es; +package gov.nasa.pds.registry.common.connection; import java.security.SecureRandom; @@ -12,7 +12,7 @@ * * @author karpenko */ -public class SSLUtils +class SSLUtils { /** * Create "trust all" SSL context to support self-signed certificates. diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/TrustAllManager.java b/src/main/java/gov/nasa/pds/registry/common/connection/TrustAllManager.java similarity index 87% rename from src/main/java/gov/nasa/pds/registry/common/connection/es/TrustAllManager.java rename to src/main/java/gov/nasa/pds/registry/common/connection/TrustAllManager.java index fea7094..6bce074 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/TrustAllManager.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/TrustAllManager.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.connection.es; +package gov.nasa.pds.registry.common.connection; import java.security.cert.CertificateException; @@ -12,7 +12,7 @@ * * @author karpenko */ -public class TrustAllManager implements X509TrustManager +class TrustAllManager implements X509TrustManager { private X509Certificate[] certs; diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConfigCB.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConfigCB.java index 12d9eaa..5065daa 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConfigCB.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConfigCB.java @@ -16,7 +16,7 @@ * * @author karpenko */ -public class ClientConfigCB implements RestClientBuilder.HttpClientConfigCallback +class ClientConfigCB implements RestClientBuilder.HttpClientConfigCallback { private boolean trustSelfSignedCert = false; private CredentialsProvider credProvider; diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/DebugUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/DebugUtils.java index 246a5f1..9796b0d 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/DebugUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/DebugUtils.java @@ -13,7 +13,7 @@ * * @author karpenko */ -public class DebugUtils +class DebugUtils { /** * Print Elasticsearch API response. diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsClientFactory.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsClientFactory.java index 8d71889..2ba27b0 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsClientFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsClientFactory.java @@ -10,7 +10,7 @@ * * @author karpenko */ -public class EsClientFactory +class EsClientFactory { /** * Create Elasticsearch rest client. diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsRestClientBld.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsRestClientBld.java index 53e83f3..3ceb6c6 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsRestClientBld.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsRestClientBld.java @@ -10,7 +10,7 @@ * * @author karpenko */ -public class EsRestClientBld +class EsRestClientBld { private RestClientBuilder bld; private ClientConfigCB clientCB; diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java index 3044051..939f486 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java @@ -2,21 +2,18 @@ import java.util.List; -import java.util.Map; import org.apache.http.HttpHost; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; - -import com.google.gson.Gson; - +import gov.nasa.pds.registry.common.util.SearchResponseParser; /** * Elasticsearch utility methods. * * @author karpenko */ -public class EsUtils +class EsUtils { /** * Parse Elasticsearch URL @@ -99,46 +96,13 @@ public static String extractErrorMessage(ResponseException ex) String lines[] = msg.split("\n"); if(lines.length < 2) return msg; - String reason = extractReasonFromJson(lines[1]); + String reason = SearchResponseParser.extractReasonFromJson(lines[1]); if(reason == null) return msg; return reason; } - /** - * Extract error message from Elasticsearch response JSON. - * @param json response JSON - * @return error message - */ - @SuppressWarnings("rawtypes") - public static String extractReasonFromJson(String json) - { - try - { - Gson gson = new Gson(); - Object obj = gson.fromJson(json, Object.class); - - obj = ((Map)obj).get("error"); - - Object rc = ((Map)obj).get("root_cause"); - if(rc != null) - { - List list = (List)rc; - obj = ((Map)list.get(0)).get("reason"); - } - else - { - obj = ((Map)obj).get("reason"); - } - - return obj.toString(); - } - catch(Exception ex) - { - return null; - } - } /** diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestConfigCB.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestConfigCB.java index bd86dc5..460ac60 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestConfigCB.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestConfigCB.java @@ -11,7 +11,7 @@ * * @author karpenko */ -public class RequestConfigCB implements RestClientBuilder.RequestConfigCallback +class RequestConfigCB implements RestClientBuilder.RequestConfigCallback { private int connectTimeoutSec = 5; private int socketTimeoutSec = 10; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java index 1e2f544..a6b457c 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java @@ -20,8 +20,8 @@ import com.google.gson.Gson; import gov.nasa.pds.registry.common.ConnectionFactory; -import gov.nasa.pds.registry.common.connection.es.EsUtils; import gov.nasa.pds.registry.common.util.CloseUtils; +import gov.nasa.pds.registry.common.util.SearchResponseParser; /** @@ -244,7 +244,7 @@ private String loadBatch(BufferedReader fileReader, String firstLine, int retrie if(json == null) throw ex; // Parse error JSON to extract reason. - String msg = EsUtils.extractReasonFromJson(json); + String msg = SearchResponseParser.extractReasonFromJson(json); if(msg == null) msg = json; throw new Exception(msg); @@ -337,7 +337,7 @@ public int loadBatch(List data, Set errorLidvids, int retries) t if(json == null) throw ex; // Parse error JSON to extract reason. - String msg = EsUtils.extractReasonFromJson(json); + String msg = SearchResponseParser.extractReasonFromJson(json); if(msg == null) msg = json; throw new Exception(msg); diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java index 9a6343a..f3b7838 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java @@ -13,8 +13,8 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; -import gov.nasa.pds.registry.common.connection.es.SearchResponseParser; import gov.nasa.pds.registry.common.es.dao.schema.SchemaRequestBuilder; +import gov.nasa.pds.registry.common.util.SearchResponseParser; import gov.nasa.pds.registry.common.util.Tuple; diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchResponseParser.java b/src/main/java/gov/nasa/pds/registry/common/util/SearchResponseParser.java similarity index 81% rename from src/main/java/gov/nasa/pds/registry/common/connection/es/SearchResponseParser.java rename to src/main/java/gov/nasa/pds/registry/common/util/SearchResponseParser.java index 3e9835f..c19ab98 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchResponseParser.java +++ b/src/main/java/gov/nasa/pds/registry/common/util/SearchResponseParser.java @@ -1,8 +1,9 @@ -package gov.nasa.pds.registry.common.connection.es; +package gov.nasa.pds.registry.common.util; import java.io.InputStream; import java.io.InputStreamReader; - +import java.util.List; +import java.util.Map; import org.elasticsearch.client.Response; import com.google.gson.Gson; @@ -172,4 +173,37 @@ else if("_source".equals(name)) cb.onRecord(lastId, src); } + /** + * Extract error message from search response JSON. + * @param json response JSON + * @return error message + */ + @SuppressWarnings("rawtypes") + public static String extractReasonFromJson(String json) + { + try + { + Gson gson = new Gson(); + Object obj = gson.fromJson(json, Object.class); + + obj = ((Map)obj).get("error"); + + Object rc = ((Map)obj).get("root_cause"); + if(rc != null) + { + List list = (List)rc; + obj = ((Map)list.get(0)).get("reason"); + } + else + { + obj = ((Map)obj).get("reason"); + } + + return obj.toString(); + } + catch(Exception ex) + { + return null; + } + } } From 951eec97d858b9352c606c7f2b4e95b4e6529e39 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 30 Jan 2024 12:40:47 -0800 Subject: [PATCH 07/41] back together again but need wrappers --- .../registry/common/ConnectionFactory.java | 1 + .../gov/nasa/pds/registry/common/Request.java | 6 +++++ .../nasa/pds/registry/common/Response.java | 9 +++++++ .../registry/common/ResponseException.java | 7 +++++ .../nasa/pds/registry/common/RestClient.java | 9 +++++++ .../registry/common/connection/Direct.java | 6 +++++ .../common/connection/es/EsUtils.java | 7 ++++- .../registry/common/es/dao/ProductDao.java | 21 +++++++-------- .../common/es/dao/dd/DataDictionaryDao.java | 14 +++++----- .../common/es/dao/schema/SchemaDao.java | 12 ++++----- .../common/util/SearchResponseParser.java | 2 +- src/test/java/dao/TestDataDictionaryDao.java | 27 +++++++++---------- src/test/java/dao/TestProductDao.java | 11 ++++---- src/test/java/tt/TestRestClient.java | 5 ++-- 14 files changed, 87 insertions(+), 50 deletions(-) create mode 100644 src/main/java/gov/nasa/pds/registry/common/Request.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/Response.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/ResponseException.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/RestClient.java diff --git a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java index 2e09e41..e6ca832 100644 --- a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java @@ -7,6 +7,7 @@ public interface ConnectionFactory { public HttpURLConnection createConnection() throws IOException; + public RestClient createRestClient(); public CredentialsProvider getCredentials(); public HttpHost getHost(); public String getHostName(); diff --git a/src/main/java/gov/nasa/pds/registry/common/Request.java b/src/main/java/gov/nasa/pds/registry/common/Request.java new file mode 100644 index 0000000..9cc0085 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/Request.java @@ -0,0 +1,6 @@ +package gov.nasa.pds.registry.common; + +public interface Request { + enum Method { GET, POST, PUT }; + public void setJsonEntity(String entity); +} diff --git a/src/main/java/gov/nasa/pds/registry/common/Response.java b/src/main/java/gov/nasa/pds/registry/common/Response.java new file mode 100644 index 0000000..674fc56 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/Response.java @@ -0,0 +1,9 @@ +package gov.nasa.pds.registry.common; + +import org.apache.http.HttpEntity; +import org.apache.http.StatusLine; + +public interface Response { + public HttpEntity getEntity(); + public StatusLine getStatusLine(); +} diff --git a/src/main/java/gov/nasa/pds/registry/common/ResponseException.java b/src/main/java/gov/nasa/pds/registry/common/ResponseException.java new file mode 100644 index 0000000..eaab13c --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/ResponseException.java @@ -0,0 +1,7 @@ +package gov.nasa.pds.registry.common; + +import java.io.IOException; + +abstract public class ResponseException extends IOException { + abstract public Response getResponse(); +} diff --git a/src/main/java/gov/nasa/pds/registry/common/RestClient.java b/src/main/java/gov/nasa/pds/registry/common/RestClient.java new file mode 100644 index 0000000..890b130 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/RestClient.java @@ -0,0 +1,9 @@ +package gov.nasa.pds.registry.common; + +import java.io.Closeable; +import java.io.IOException; + +public interface RestClient extends Closeable { + public Request createRequest(Request.Method method, String endpoint); + public Response performRequest(Request request) throws IOException; +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java index 5d00bbd..05b130e 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java @@ -8,6 +8,7 @@ import org.apache.http.HttpHost; import org.apache.http.client.CredentialsProvider; import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.RestClient; public class Direct implements Cloneable, ConnectionFactory { final private boolean veryTrusting; @@ -52,6 +53,11 @@ public HttpURLConnection createConnection() throws IOException { return con; } @Override + public RestClient createRestClient() { + // FIXME: need an implementation + return null; + } + @Override public HttpHost getHost() { return this.host; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java index 939f486..a1144d5 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java @@ -4,6 +4,7 @@ import java.util.List; import org.apache.http.HttpHost; +import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import gov.nasa.pds.registry.common.util.SearchResponseParser; @@ -102,7 +103,11 @@ public static String extractErrorMessage(ResponseException ex) return reason; } - + public static String blahblah (String a) { + String result = a + "snafu"; + Request req; + return result; + } /** diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java index d8c3dee..a3abd73 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java @@ -8,14 +8,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; - +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.ResponseException; +import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.common.util.CloseUtils; import gov.nasa.pds.registry.common.util.LidVidUtils; @@ -56,7 +55,7 @@ public String getProductClass(String lidvid) throws Exception if(lidvid == null) return null; String reqUrl = "/" + indexName + "/_doc/" + lidvid + "?_source=product_class"; - Request req = new Request("GET", reqUrl); + Request req = client.createRequest(Request.Method.GET, reqUrl); Response resp = null; try @@ -142,7 +141,7 @@ else if(type == 'S') // Request URL String reqUrl = "/" + indexName + "-refs/_count?q=" + query; - Request req = new Request("GET", reqUrl); + Request req = client.createRequest(Request.Method.GET, reqUrl); Response resp = null; try @@ -212,7 +211,7 @@ public List getRefs(String collectionLidVid, char type, int page) throws String docId = collectionLidVid + "::" + type + page; String reqUrl = "/" + indexName + "-refs/_doc/" + docId + "?_source=product_lidvid"; - Request req = new Request("GET", reqUrl); + Request req = client.createRequest(Request.Method.GET, reqUrl); Response resp = null; try @@ -281,7 +280,7 @@ public void updateArchiveStatus(Collection lidvids, String status) throw log.debug("Request:\n" + json); String reqUrl = "/" + indexName + "/_bulk"; //?refresh=wait_for"; - Request req = new Request("POST", reqUrl); + Request req = client.createRequest(Request.Method.POST, reqUrl); req.setJsonEntity(json); Response resp = client.performRequest(req); @@ -362,7 +361,7 @@ public LidvidSet getCollectionIds(String bundleLidvid) throws Exception if(bundleLidvid == null) return null; String reqUrl = "/" + indexName + "/_doc/" + bundleLidvid + "?_source=ref_lidvid_collection,ref_lid_collection"; - Request req = new Request("GET", reqUrl); + Request req = client.createRequest(Request.Method.GET, reqUrl); Response resp = null; try @@ -477,7 +476,7 @@ public List getLatestLidVids(Collection lids) throws Exception if(json == null) return null; String reqUrl = "/" + indexName + "/_search/"; - Request req = new Request("GET", reqUrl); + Request req = client.createRequest(Request.Method.GET, reqUrl); req.setJsonEntity(json); Response resp = null; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java index f3b7838..f563728 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java @@ -10,9 +10,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.common.es.dao.schema.SchemaRequestBuilder; import gov.nasa.pds.registry.common.util.SearchResponseParser; import gov.nasa.pds.registry.common.util.Tuple; @@ -87,7 +87,7 @@ public LddVersions getLddInfo(String namespace) throws Exception DDRequestBuilder bld = new DDRequestBuilder(); String json = bld.createListLddsRequest(namespace); - Request req = new Request("GET", "/" + indexName + "-dd/_search"); + Request req = client.createRequest(Request.Method.GET, "/" + indexName + "-dd/_search"); req.setJsonEntity(json); Response resp = client.performRequest(req); @@ -153,7 +153,7 @@ public List listLdds(String namespace) throws Exception DDRequestBuilder bld = new DDRequestBuilder(); String json = bld.createListLddsRequest(namespace); - Request req = new Request("GET", "/" + indexName + "-dd/_search"); + Request req = client.createRequest(Request.Method.GET, "/" + indexName + "-dd/_search"); req.setJsonEntity(json); Response resp = client.performRequest(req); @@ -202,7 +202,7 @@ public Set getFieldNamesByEsType(String esType) throws Exception DDRequestBuilder bld = new DDRequestBuilder(); String json = bld.createListFieldsRequest(esType); - Request req = new Request("GET", "/" + indexName + "-dd/_search"); + Request req = client.createRequest(Request.Method.GET, "/" + indexName + "-dd/_search"); req.setJsonEntity(json); Response resp = client.performRequest(req); @@ -229,7 +229,7 @@ public List getDataTypes(Collection ids, boolean stringForMissing List dtInfo = new ArrayList(); // Create request - Request req = new Request("GET", "/" + indexName + "-dd/_mget?_source=es_data_type"); + Request req = client.createRequest(Request.Method.GET, "/" + indexName + "-dd/_mget?_source=es_data_type"); // Create request body SchemaRequestBuilder bld = new SchemaRequestBuilder(); diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java index 9b11d47..be280a5 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java @@ -2,11 +2,9 @@ import java.util.List; import java.util.Set; - -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; - +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.common.util.Tuple; @@ -41,7 +39,7 @@ public SchemaDao(RestClient client, String indexName) */ public Set getFieldNames() throws Exception { - Request req = new Request("GET", "/" + indexName + "/_mappings"); + Request req = client.createRequest(Request.Method.GET, "/" + indexName + "/_mappings"); Response resp = client.performRequest(req); MappingsParser parser = new MappingsParser(indexName); @@ -61,7 +59,7 @@ public void updateSchema(List fields) throws Exception SchemaRequestBuilder bld = new SchemaRequestBuilder(); String json = bld.createUpdateSchemaRequest(fields); - Request req = new Request("PUT", "/" + indexName + "/_mapping"); + Request req = client.createRequest(Request.Method.PUT, "/" + indexName + "/_mapping"); req.setJsonEntity(json); client.performRequest(req); } diff --git a/src/main/java/gov/nasa/pds/registry/common/util/SearchResponseParser.java b/src/main/java/gov/nasa/pds/registry/common/util/SearchResponseParser.java index c19ab98..b1f32be 100644 --- a/src/main/java/gov/nasa/pds/registry/common/util/SearchResponseParser.java +++ b/src/main/java/gov/nasa/pds/registry/common/util/SearchResponseParser.java @@ -4,11 +4,11 @@ import java.io.InputStreamReader; import java.util.List; import java.util.Map; -import org.elasticsearch.client.Response; import com.google.gson.Gson; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; +import gov.nasa.pds.registry.common.Response; /** diff --git a/src/test/java/dao/TestDataDictionaryDao.java b/src/test/java/dao/TestDataDictionaryDao.java index 6041d59..f62aee3 100644 --- a/src/test/java/dao/TestDataDictionaryDao.java +++ b/src/test/java/dao/TestDataDictionaryDao.java @@ -4,9 +4,8 @@ import java.util.List; import java.util.Set; -import org.elasticsearch.client.RestClient; import gov.nasa.pds.registry.common.EstablishConnectionFactory; -import gov.nasa.pds.registry.common.connection.es.EsClientFactory; +import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.common.es.dao.dd.DataDictionaryDao; import gov.nasa.pds.registry.common.es.dao.dd.LddInfo; import gov.nasa.pds.registry.common.es.dao.dd.LddVersions; @@ -25,11 +24,11 @@ public static void main(String[] args) throws Exception private static void testListBooleanFields() throws Exception { - RestClient esClient = EsClientFactory.createRestClient(EstablishConnectionFactory.directly("http://localhost:9200")); + RestClient client = EstablishConnectionFactory.directly("http://localhost:9200").createRestClient(); try { - DataDictionaryDao dao = new DataDictionaryDao(esClient, "registry"); + DataDictionaryDao dao = new DataDictionaryDao(client, "registry"); Set list = dao.getFieldNamesByEsType("boolean"); System.out.println("Boolean fields count = " + list.size()); @@ -37,18 +36,18 @@ private static void testListBooleanFields() throws Exception } finally { - esClient.close(); + client.close(); } } private static void testListDateFields() throws Exception { - RestClient esClient = EsClientFactory.createRestClient(EstablishConnectionFactory.directly("http://localhost:9200")); + RestClient client = EstablishConnectionFactory.directly("http://localhost:9200").createRestClient(); try { - DataDictionaryDao dao = new DataDictionaryDao(esClient, "registry"); + DataDictionaryDao dao = new DataDictionaryDao(client, "registry"); Set list = dao.getFieldNamesByEsType("date"); System.out.println("Date fields count = " + list.size()); @@ -56,18 +55,18 @@ private static void testListDateFields() throws Exception } finally { - esClient.close(); + client.close(); } } private static void testListLdds() throws Exception { - RestClient esClient = EsClientFactory.createRestClient(EstablishConnectionFactory.directly("http://localhost:9200")); + RestClient client = EstablishConnectionFactory.directly("http://localhost:9200").createRestClient(); try { - DataDictionaryDao dao = new DataDictionaryDao(esClient, "registry"); + DataDictionaryDao dao = new DataDictionaryDao(client, "registry"); List list = dao.listLdds(null); Collections.sort(list); @@ -79,24 +78,24 @@ private static void testListLdds() throws Exception } finally { - esClient.close(); + client.close(); } } private static void testGetLddInfo() throws Exception { - RestClient esClient = EsClientFactory.createRestClient(EstablishConnectionFactory.directly("http://localhost:9200")); + RestClient client = EstablishConnectionFactory.directly("http://localhost:9200").createRestClient(); try { - DataDictionaryDao dao = new DataDictionaryDao(esClient, "registry"); + DataDictionaryDao dao = new DataDictionaryDao(client, "registry"); LddVersions info = dao.getLddInfo("pds"); info.debug(); } finally { - esClient.close(); + client.close(); } } diff --git a/src/test/java/dao/TestProductDao.java b/src/test/java/dao/TestProductDao.java index 56d001f..12bdcc2 100644 --- a/src/test/java/dao/TestProductDao.java +++ b/src/test/java/dao/TestProductDao.java @@ -3,9 +3,8 @@ import java.util.Arrays; import java.util.List; -import org.elasticsearch.client.RestClient; import gov.nasa.pds.registry.common.EstablishConnectionFactory; -import gov.nasa.pds.registry.common.connection.es.EsClientFactory; +import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.common.es.dao.ProductDao; import gov.nasa.pds.registry.common.util.CloseUtils; import tt.TestLogConfigurator; @@ -18,19 +17,19 @@ public static void main(String[] args) throws Exception TestLogConfigurator.configureLogger(); - RestClient esClient = null; + RestClient client = null; try { - esClient = EsClientFactory.createRestClient(EstablishConnectionFactory.directly("localhost")); - ProductDao dao = new ProductDao(esClient, "registry"); + client = EstablishConnectionFactory.directly("localhost").createRestClient(); + ProductDao dao = new ProductDao(client, "registry"); //testUpdateArchiveStatus(dao); testGetLatestLidVids(dao); } finally { - CloseUtils.close(esClient); + CloseUtils.close(client); } } diff --git a/src/test/java/tt/TestRestClient.java b/src/test/java/tt/TestRestClient.java index 54e2fc1..79eea49 100644 --- a/src/test/java/tt/TestRestClient.java +++ b/src/test/java/tt/TestRestClient.java @@ -1,8 +1,7 @@ package tt; -import org.elasticsearch.client.RestClient; import gov.nasa.pds.registry.common.EstablishConnectionFactory; -import gov.nasa.pds.registry.common.connection.es.EsClientFactory; +import gov.nasa.pds.registry.common.RestClient; public class TestRestClient @@ -10,7 +9,7 @@ public class TestRestClient public static void main(String[] args) throws Exception { - RestClient client = EsClientFactory.createRestClient(EstablishConnectionFactory.directly("localhost")); + RestClient client = EstablishConnectionFactory.directly("localhost").createRestClient(); client.close(); } From 5c1dadb3d90d4a0159ddd36ace37d8ae8aa4ff8f Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 30 Jan 2024 13:16:27 -0800 Subject: [PATCH 08/41] wrappers are now in place as well --- .../registry/common/ConnectionFactory.java | 2 +- .../registry/common/ResponseException.java | 1 + .../registry/common/connection/Direct.java | 6 +- .../common/connection/es/DebugUtils.java | 46 -------------- .../common/connection/es/EsClientFactory.java | 28 --------- .../common/connection/es/EsRestClientBld.java | 43 ------------- .../common/connection/es/EsUtils.java | 8 --- .../common/connection/es/RequestWrapper.java | 25 ++++++++ .../es/ResponseExceptionWrapper.java | 63 +++++++++++++++++++ .../common/connection/es/ResponseWrapper.java | 20 ++++++ .../connection/es/RestClientWrapper.java | 52 +++++++++++++++ 11 files changed, 165 insertions(+), 129 deletions(-) delete mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/DebugUtils.java delete mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/EsClientFactory.java delete mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/EsRestClientBld.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/RequestWrapper.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseWrapper.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java diff --git a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java index e6ca832..0446e19 100644 --- a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java @@ -7,7 +7,7 @@ public interface ConnectionFactory { public HttpURLConnection createConnection() throws IOException; - public RestClient createRestClient(); + public RestClient createRestClient() throws Exception; public CredentialsProvider getCredentials(); public HttpHost getHost(); public String getHostName(); diff --git a/src/main/java/gov/nasa/pds/registry/common/ResponseException.java b/src/main/java/gov/nasa/pds/registry/common/ResponseException.java index eaab13c..bf26760 100644 --- a/src/main/java/gov/nasa/pds/registry/common/ResponseException.java +++ b/src/main/java/gov/nasa/pds/registry/common/ResponseException.java @@ -3,5 +3,6 @@ import java.io.IOException; abstract public class ResponseException extends IOException { + private static final long serialVersionUID = 8629769947735587642L; abstract public Response getResponse(); } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java index 05b130e..316cf89 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java @@ -9,6 +9,7 @@ import org.apache.http.client.CredentialsProvider; import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.RestClient; +import gov.nasa.pds.registry.common.connection.es.RestClientWrapper; public class Direct implements Cloneable, ConnectionFactory { final private boolean veryTrusting; @@ -53,9 +54,8 @@ public HttpURLConnection createConnection() throws IOException { return con; } @Override - public RestClient createRestClient() { - // FIXME: need an implementation - return null; + public RestClient createRestClient() throws Exception { + return new RestClientWrapper(this); } @Override public HttpHost getHost() { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/DebugUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/DebugUtils.java deleted file mode 100644 index 9796b0d..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/DebugUtils.java +++ /dev/null @@ -1,46 +0,0 @@ -package gov.nasa.pds.registry.common.connection.es; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; - -import org.elasticsearch.client.Response; - - -/** - * Debug utilities. - * - * @author karpenko - */ -class DebugUtils -{ - /** - * Print Elasticsearch API response. - * @param resp HTTP response - * @throws IOException an exception - */ - public static void dumpResponseBody(Response resp) throws IOException - { - InputStream is = resp.getEntity().getContent(); - dump(is); - is.close(); - } - - - /** - * Print content of an input stream. - * @param is input stream - * @throws IOException an exception - */ - public static void dump(InputStream is) throws IOException - { - BufferedReader rd = new BufferedReader(new InputStreamReader(is)); - - String line; - while((line = rd.readLine()) != null) - { - System.out.println(line); - } - } -} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsClientFactory.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsClientFactory.java deleted file mode 100644 index 2ba27b0..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsClientFactory.java +++ /dev/null @@ -1,28 +0,0 @@ -package gov.nasa.pds.registry.common.connection.es; - - -import org.elasticsearch.client.RestClient; -import gov.nasa.pds.registry.common.ConnectionFactory; - - -/** - * A factory class to create Elasticsearch Rest client instances. - * - * @author karpenko - */ -class EsClientFactory -{ - /** - * Create Elasticsearch rest client. - * @param esUrl Elasticsearch URL, e.g., "http://localhost:9200" - * @param authPath Path to authentication configuration file. - * @return Elasticsearch rest client instance. - * @throws Exception an exception - */ - public static RestClient createRestClient(ConnectionFactory conFact) throws Exception - { - EsRestClientBld bld = new EsRestClientBld(conFact); - return bld.build(); - } - -} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsRestClientBld.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsRestClientBld.java deleted file mode 100644 index 3ceb6c6..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsRestClientBld.java +++ /dev/null @@ -1,43 +0,0 @@ -package gov.nasa.pds.registry.common.connection.es; - -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; -import gov.nasa.pds.registry.common.ConnectionFactory; - - -/** - * Utility class to build Elasticsearch rest client. - * - * @author karpenko - */ -class EsRestClientBld -{ - private RestClientBuilder bld; - private ClientConfigCB clientCB; - private RequestConfigCB reqCB; - /** - * Constructor. - * @param url Elasticsearch URL, e.g., "http://localhost:9200" - * @throws Exception an exception - */ - public EsRestClientBld(ConnectionFactory conFact) throws Exception - { - bld = RestClient.builder(conFact.getHost()); - - clientCB = new ClientConfigCB(); - clientCB.setCredProvider(conFact.getCredentials()); - clientCB.setTrustSelfSignedCert(conFact.isTrustingSelfSigned()); - reqCB = new RequestConfigCB(); - } - /** - * Build the Elasticsearch rest client - * @return Elasticsearch rest client - */ - public RestClient build() - { - bld.setHttpClientConfigCallback(clientCB); - bld.setRequestConfigCallback(reqCB); - - return bld.build(); - } -} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java index a1144d5..9cb2f32 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java @@ -4,7 +4,6 @@ import java.util.List; import org.apache.http.HttpHost; -import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import gov.nasa.pds.registry.common.util.SearchResponseParser; @@ -103,13 +102,6 @@ public static String extractErrorMessage(ResponseException ex) return reason; } - public static String blahblah (String a) { - String result = a + "snafu"; - Request req; - return result; - } - - /** * Print Elasticsearch response warnings. * @param resp HTTP response diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestWrapper.java new file mode 100644 index 0000000..7e20b2f --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestWrapper.java @@ -0,0 +1,25 @@ +package gov.nasa.pds.registry.common.connection.es; + +import gov.nasa.pds.registry.common.Request; + +final class RequestWrapper implements Request { + final org.elasticsearch.client.Request real_request; + RequestWrapper (org.elasticsearch.client.Request real_request) { + this.real_request = real_request; + } + RequestWrapper (Request.Method method, String endpoint) { + this(new org.elasticsearch.client.Request(RequestWrapper.methodToString(method), endpoint)); + } + @Override + public void setJsonEntity(String entity) { + this.real_request.setJsonEntity(entity); + } + private static String methodToString (Request.Method method) { + switch (method) { + case GET: return "GET"; + case POST: return "POST"; + case PUT: return "PUT"; + } + throw new RuntimeException ("Request.Method not fully enumerated"); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java new file mode 100644 index 0000000..8b2ba2e --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java @@ -0,0 +1,63 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.PrintStream; +import java.io.PrintWriter; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.ResponseException; + +final class ResponseExceptionWrapper extends ResponseException { + private static final long serialVersionUID = -5116172984798822803L; + final org.elasticsearch.client.ResponseException real_exception; + ResponseExceptionWrapper (org.elasticsearch.client.ResponseException real_exception){ + this.real_exception = real_exception; + } + @Override + public String getMessage() { + return this.real_exception.getMessage(); + } + @Override + public String getLocalizedMessage() { + return this.real_exception.getLocalizedMessage(); + } + @Override + public synchronized Throwable getCause() { + return this.real_exception.getCause(); + } + @Override + public synchronized Throwable initCause(Throwable cause) { + return this.real_exception.initCause(cause); + } + @Override + public String toString() { + return this.real_exception.toString(); + } + @Override + public void printStackTrace() { + this.real_exception.printStackTrace(); + } + @Override + public void printStackTrace(PrintStream s) { + this.real_exception.printStackTrace(s); + } + @Override + public void printStackTrace(PrintWriter s) { + this.real_exception.printStackTrace(s); + } + @Override + public synchronized Throwable fillInStackTrace() { + return this.real_exception.fillInStackTrace(); + } + @Override + public StackTraceElement[] getStackTrace() { + return this.real_exception.getStackTrace(); + } + @Override + public void setStackTrace(StackTraceElement[] stackTrace) { + this.real_exception.setStackTrace(stackTrace); + } + @Override + public Response getResponse() { + return new ResponseWrapper(this.real_exception.getResponse()); + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseWrapper.java new file mode 100644 index 0000000..1681dbd --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseWrapper.java @@ -0,0 +1,20 @@ +package gov.nasa.pds.registry.common.connection.es; + +import org.apache.http.HttpEntity; +import org.apache.http.StatusLine; +import gov.nasa.pds.registry.common.Response; + +class ResponseWrapper implements Response { + final org.elasticsearch.client.Response real_response; + ResponseWrapper (org.elasticsearch.client.Response real_response) { + this.real_response = real_response; + } + @Override + public HttpEntity getEntity() { + return this.real_response.getEntity(); + } + @Override + public StatusLine getStatusLine() { + return this.real_response.getStatusLine(); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java new file mode 100644 index 0000000..c8b9e8b --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java @@ -0,0 +1,52 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import org.elasticsearch.client.RestClientBuilder; +import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.Request.Method; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.RestClient; + + +/** + * Utility class to build Elasticsearch rest client. + * + * @author karpenko + */ +public class RestClientWrapper implements RestClient +{ + final org.elasticsearch.client.RestClient real_client; + /** + * Constructor. + * @param url Elasticsearch URL, e.g., "http://localhost:9200" + * @throws Exception an exception + */ + public RestClientWrapper(ConnectionFactory conFact) throws Exception + { + ClientConfigCB clientCB = new ClientConfigCB(); + RequestConfigCB reqCB = new RequestConfigCB(); + RestClientBuilder bld = org.elasticsearch.client.RestClient.builder(conFact.getHost()); + clientCB.setCredProvider(conFact.getCredentials()); + clientCB.setTrustSelfSignedCert(conFact.isTrustingSelfSigned()); + bld.setHttpClientConfigCallback(clientCB); + bld.setRequestConfigCallback(reqCB); + this.real_client = bld.build(); + } + @Override + public void close() throws IOException { + this.real_client.close(); + } + @Override + public Request createRequest(Method method, String endpoint) { + return new RequestWrapper(method, endpoint); + } + @Override + public Response performRequest(Request request) throws IOException { + try { + return this.performRequest((RequestWrapper)request); + } catch (org.elasticsearch.client.ResponseException e) { + throw new ResponseExceptionWrapper(e); + } + } +} From 8709e60af27deb1f86a30e9ed0503e880fb00217 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 30 Jan 2024 14:50:48 -0800 Subject: [PATCH 09/41] fixes for harvest to work --- .../pds/registry/common/connection/es/RestClientWrapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java index c8b9e8b..e390fd0 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java @@ -44,7 +44,7 @@ public Request createRequest(Method method, String endpoint) { @Override public Response performRequest(Request request) throws IOException { try { - return this.performRequest((RequestWrapper)request); + return new ResponseWrapper(this.real_client.performRequest(((RequestWrapper)request).real_request)); } catch (org.elasticsearch.client.ResponseException e) { throw new ResponseExceptionWrapper(e); } From 7f5a2e1ca6336c7f71d1c2956ec054dea57098a1 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 30 Jan 2024 15:46:34 -0800 Subject: [PATCH 10/41] new functionality for registry-mgr --- .../gov/nasa/pds/registry/common/Request.java | 2 +- .../gov/nasa/pds/registry/common/Response.java | 1 + .../pds/registry/common/ResponseException.java | 1 + .../common/connection/es/RequestWrapper.java | 2 ++ .../connection/es/ResponseExceptionWrapper.java | 15 ++++++++++++++- .../common/connection/es/ResponseWrapper.java | 12 ++++++++++++ 6 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/Request.java b/src/main/java/gov/nasa/pds/registry/common/Request.java index 9cc0085..71bd2ec 100644 --- a/src/main/java/gov/nasa/pds/registry/common/Request.java +++ b/src/main/java/gov/nasa/pds/registry/common/Request.java @@ -1,6 +1,6 @@ package gov.nasa.pds.registry.common; public interface Request { - enum Method { GET, POST, PUT }; + enum Method { DELETE, GET, HEAD, POST, PUT }; public void setJsonEntity(String entity); } diff --git a/src/main/java/gov/nasa/pds/registry/common/Response.java b/src/main/java/gov/nasa/pds/registry/common/Response.java index 674fc56..ea79b80 100644 --- a/src/main/java/gov/nasa/pds/registry/common/Response.java +++ b/src/main/java/gov/nasa/pds/registry/common/Response.java @@ -6,4 +6,5 @@ public interface Response { public HttpEntity getEntity(); public StatusLine getStatusLine(); + public void printWarnings(); } diff --git a/src/main/java/gov/nasa/pds/registry/common/ResponseException.java b/src/main/java/gov/nasa/pds/registry/common/ResponseException.java index bf26760..905c709 100644 --- a/src/main/java/gov/nasa/pds/registry/common/ResponseException.java +++ b/src/main/java/gov/nasa/pds/registry/common/ResponseException.java @@ -4,5 +4,6 @@ abstract public class ResponseException extends IOException { private static final long serialVersionUID = 8629769947735587642L; + abstract public String extractErrorMessage(); abstract public Response getResponse(); } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestWrapper.java index 7e20b2f..38411bb 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestWrapper.java @@ -16,7 +16,9 @@ public void setJsonEntity(String entity) { } private static String methodToString (Request.Method method) { switch (method) { + case DELETE: return "DELETE"; case GET: return "GET"; + case HEAD: return "HEAD"; case POST: return "POST"; case PUT: return "PUT"; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java index 8b2ba2e..d132fae 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java @@ -4,6 +4,7 @@ import java.io.PrintWriter; import gov.nasa.pds.registry.common.Response; import gov.nasa.pds.registry.common.ResponseException; +import gov.nasa.pds.registry.common.util.SearchResponseParser; final class ResponseExceptionWrapper extends ResponseException { private static final long serialVersionUID = -5116172984798822803L; @@ -59,5 +60,17 @@ public void setStackTrace(StackTraceElement[] stackTrace) { public Response getResponse() { return new ResponseWrapper(this.real_exception.getResponse()); } - + @Override + public String extractErrorMessage() { + String msg = this.real_exception.getMessage(); + if(msg == null) return "Unknown error"; + + String lines[] = msg.split("\n"); + if(lines.length < 2) return msg; + + String reason = SearchResponseParser.extractReasonFromJson(lines[1]); + if(reason == null) return msg; + + return reason; + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseWrapper.java index 1681dbd..e54693d 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseWrapper.java @@ -1,5 +1,6 @@ package gov.nasa.pds.registry.common.connection.es; +import java.util.List; import org.apache.http.HttpEntity; import org.apache.http.StatusLine; import gov.nasa.pds.registry.common.Response; @@ -17,4 +18,15 @@ public HttpEntity getEntity() { public StatusLine getStatusLine() { return this.real_response.getStatusLine(); } + @Override + public void printWarnings() { + List warnings = this.real_response.getWarnings(); + if(warnings != null) + { + for(String warn: warnings) + { + System.out.println("[WARN] " + warn); + } + } + } } From c701f0dd50069534f25b72e8cbc68a85f01393ae Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Sat, 3 Feb 2024 10:55:09 -0800 Subject: [PATCH 11/41] work in progress The first step of getting past cognito is working. The second step of retrieving a self signed URL is not working. Backing the changes up to this branch. --- .../pds/registry/common/CognitoContent.java | 7 + .../common/EstablishConnectionFactory.java | 19 +-- .../common/connection/MultiTenancy.java | 122 ++++++++++++++++++ 3 files changed, 136 insertions(+), 12 deletions(-) create mode 100644 src/main/java/gov/nasa/pds/registry/common/CognitoContent.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java diff --git a/src/main/java/gov/nasa/pds/registry/common/CognitoContent.java b/src/main/java/gov/nasa/pds/registry/common/CognitoContent.java new file mode 100644 index 0000000..10671c3 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/CognitoContent.java @@ -0,0 +1,7 @@ +package gov.nasa.pds.registry.common; + +public interface CognitoContent { + String getClientID(); + String getGateway(); + String getIDP(); +} diff --git a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java index c26914d..53ba7c5 100644 --- a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java @@ -2,6 +2,7 @@ import gov.nasa.pds.registry.common.connection.AuthContent; import gov.nasa.pds.registry.common.connection.Direct; +import gov.nasa.pds.registry.common.connection.MultiTenancy; public class EstablishConnectionFactory { public static ConnectionFactory directly (String url) throws Exception { @@ -19,19 +20,13 @@ public static ConnectionFactory directly (String url, String authfile, boolean t private static ConnectionFactory directly (String url, AuthContent auth, boolean trustedSelfSigned) throws Exception { return Direct.build (url, auth, trustedSelfSigned); } - public static ConnectionFactory viaCognito (String clientID) throws Exception { - return viaCognito (clientID, AuthContent.DEFAULT, false); + public static ConnectionFactory viaCognito (CognitoContent cog) throws Exception { + return viaCognito (cog, AuthContent.DEFAULT); } - public static ConnectionFactory viaCognito (String clientID, String authfile) throws Exception { - return viaCognito (clientID, AuthContent.from(authfile), false); + public static ConnectionFactory viaCognito (CognitoContent cog, String authfile) throws Exception { + return viaCognito (cog, AuthContent.from(authfile)); } - public static ConnectionFactory viaCognito (String clientID, boolean trustSelfSigned) throws Exception { - return viaCognito (clientID, AuthContent.DEFAULT, trustSelfSigned); - } - public static ConnectionFactory viaCognito (String clientID, String authfile, boolean trustSelfSigned) throws Exception { - return viaCognito (clientID, AuthContent.from (authfile), trustSelfSigned); - } - private static ConnectionFactory viaCognito (String clientID, AuthContent auth, boolean trustSelfSigned) throws Exception { - return null; + private static ConnectionFactory viaCognito (CognitoContent cog, AuthContent auth) throws Exception { + return MultiTenancy.build(cog, auth); } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java new file mode 100644 index 0000000..7b5b956 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java @@ -0,0 +1,122 @@ +package gov.nasa.pds.registry.common.connection; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse; +import java.util.Map; +import org.apache.http.HttpHost; +import org.apache.http.client.CredentialsProvider; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import gov.nasa.pds.registry.common.CognitoContent; +import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.RestClient; + +public final class MultiTenancy implements ConnectionFactory { + final private AuthContent auth; + final private HttpHost host; + final private URL signed; + private String api = null; + private String index = null; + public static MultiTenancy build (CognitoContent cog, AuthContent auth) throws IOException, InterruptedException { + boolean expectedContent = true; + Gson gson = new Gson(); + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(cog.getIDP())) + .POST(BodyPublishers.ofString("{\"AuthFlow\":\"USER_PASSWORD_AUTH\",\"AuthParameters\":{" + + "\"USERNAME\":\"" + auth.getUser() + "\"," + + "\"PASSWORD\":\"" + auth.getPassword() + "\"" + + "},\"ClientId\":\"" + cog.getClientID() + "\"" + + "}")) + .setHeader("X-Amz-Target", "AWSCognitoIdentityProviderService.InitiateAuth") + .setHeader("Content-Type", "application/x-amz-json-1.1") + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + Map> content; + Type contentType = new TypeToken>>(){}.getType(); + + expectedContent &= response.body().contains("AuthenticationResult"); + expectedContent &= response.body().contains("AccessToken"); + expectedContent &= response.body().contains("ExpiresIn"); + expectedContent &= response.body().contains("IdToken"); + expectedContent &= response.body().contains("RefreshToken"); + expectedContent &= response.body().contains("TokenType"); + expectedContent &= response.body().contains("ChallengeParameters"); + if (!expectedContent) { + throw new IOException("Received an unexpected response of: " + response.toString() + + " ->\n" + response.body()); + } + content = gson.fromJson(response.body(), contentType); + client = HttpClient.newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .build(); + request = HttpRequest.newBuilder() + .uri(URI.create(cog.getGateway())) + .GET() + .setHeader("Authorization", content.get("AuthenticationResult").get("AccessToken")) + .setHeader("IDToken", content.get("AuthenticationResult").get("IdToken")) + .build(); + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + System.out.println (response); + System.out.println (response.body()); + URL signed = null; + return new MultiTenancy(auth, signed); + } + + private MultiTenancy (AuthContent auth, URL signed) { + this.auth = auth; + this.host = new HttpHost(signed.getHost(), signed.getPort(), signed.getProtocol()); + this.signed = signed; + } + @Override + public ConnectionFactory clone() { + return new MultiTenancy(this.auth, this.signed).setAPI(this.api).setIndexName(this.index); + } + @Override + public HttpURLConnection createConnection() throws IOException { + // TODO Auto-generated method stub + return null; + } + @Override + public RestClient createRestClient() throws Exception { + // TODO Auto-generated method stub + return null; + } + @Override + public CredentialsProvider getCredentials() { + return this.auth.getCredentials(); + } + @Override + public HttpHost getHost() { + return this.host; + } + @Override + public String getHostName() { + return this.host.getHostName(); + } + @Override + public String getIndexName() { + return this.index; + } + @Override + public boolean isTrustingSelfSigned() { + return false; + } + @Override + public ConnectionFactory setAPI(String api) { + this.api = api; + return this; + } + @Override + public ConnectionFactory setIndexName(String idxName) { + this.index = idxName; + return this; + } +} From 06079fd7b14a0078d7b82235d8d9918bb0da2fef Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Fri, 9 Feb 2024 15:02:15 -0800 Subject: [PATCH 12/41] new code for new arg handling --- pom.xml | 16 ++ .../pds/registry/common/CognitoContent.java | 7 - .../common/EstablishConnectionFactory.java | 37 ++--- .../registry/common/connection/Direct.java | 9 +- .../common/connection/MultiTenancy.java | 8 +- .../connection/RegistryConnectionContent.java | 43 +++++ .../common/connection/config/CognitoType.java | 149 ++++++++++++++++++ .../common/connection/config/DirectType.java | 113 +++++++++++++ .../connection/config/ObjectFactory.java | 68 ++++++++ .../connection/config/RegistryConnection.java | 142 +++++++++++++++++ src/main/resources/registry-connection.xsd | 105 ++++++++++++ src/test/java/dao/TestDataDictionaryDao.java | 10 +- src/test/java/dao/TestProductDao.java | 2 +- src/test/java/tt/TestRestClient.java | 2 +- 14 files changed, 666 insertions(+), 45 deletions(-) delete mode 100644 src/main/java/gov/nasa/pds/registry/common/CognitoContent.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/config/CognitoType.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/config/DirectType.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/config/ObjectFactory.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/config/RegistryConnection.java create mode 100644 src/main/resources/registry-connection.xsd diff --git a/pom.xml b/pom.xml index 7bb2099..5823903 100644 --- a/pom.xml +++ b/pom.xml @@ -116,6 +116,22 @@ POSSIBILITY OF SUCH DAMAGE. tika-core 1.28.3 + + + jakarta.xml.bind + jakarta.xml.bind-api + 4.0.1 + + + com.sun.xml.bind + jaxb-impl + 4.0.4 + + + jakarta.activation + jakarta.activation-api + 2.1.2 + diff --git a/src/main/java/gov/nasa/pds/registry/common/CognitoContent.java b/src/main/java/gov/nasa/pds/registry/common/CognitoContent.java deleted file mode 100644 index 10671c3..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/CognitoContent.java +++ /dev/null @@ -1,7 +0,0 @@ -package gov.nasa.pds.registry.common; - -public interface CognitoContent { - String getClientID(); - String getGateway(); - String getIDP(); -} diff --git a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java index 53ba7c5..f1e13fe 100644 --- a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java @@ -1,32 +1,23 @@ package gov.nasa.pds.registry.common; +import java.net.URL; import gov.nasa.pds.registry.common.connection.AuthContent; import gov.nasa.pds.registry.common.connection.Direct; import gov.nasa.pds.registry.common.connection.MultiTenancy; +import gov.nasa.pds.registry.common.connection.RegistryConnectionContent; public class EstablishConnectionFactory { - public static ConnectionFactory directly (String url) throws Exception { - return directly (url, AuthContent.DEFAULT, false); - } - public static ConnectionFactory directly (String url, String authfile) throws Exception { - return directly (url, AuthContent.from(authfile), false); - } - public static ConnectionFactory directly (String url, boolean trustSelfSigned) throws Exception { - return directly (url, AuthContent.DEFAULT, trustSelfSigned); - } - public static ConnectionFactory directly (String url, String authfile, boolean trustSelfSigned) throws Exception { - return directly (url, AuthContent.from(authfile), trustSelfSigned); - } - private static ConnectionFactory directly (String url, AuthContent auth, boolean trustedSelfSigned) throws Exception { - return Direct.build (url, auth, trustedSelfSigned); - } - public static ConnectionFactory viaCognito (CognitoContent cog) throws Exception { - return viaCognito (cog, AuthContent.DEFAULT); - } - public static ConnectionFactory viaCognito (CognitoContent cog, String authfile) throws Exception { - return viaCognito (cog, AuthContent.from(authfile)); - } - private static ConnectionFactory viaCognito (CognitoContent cog, AuthContent auth) throws Exception { - return MultiTenancy.build(cog, auth); + public static ConnectionFactory from (String urlToRegistryConnection) throws Exception { + return EstablishConnectionFactory.from (urlToRegistryConnection, AuthContent.DEFAULT); + } + public static ConnectionFactory from (String urlToRegistryConnection, String authFile) throws Exception { + return EstablishConnectionFactory.from (urlToRegistryConnection, AuthContent.from(authFile)); + } + private static ConnectionFactory from (String urlToRegistryConnection, AuthContent auth) throws Exception { + RegistryConnectionContent conn = RegistryConnectionContent.from (new URL(urlToRegistryConnection)); + + if (conn.isDirectConnection()) return Direct.build(conn.getServerUrl(), auth); + if (conn.isCognitoConnection()) return MultiTenancy.build(conn.getCognitoClientId(), auth); + throw new RuntimeException("New XML/Java choices in src/main/resources/registry_connection.xsd that are not handled."); } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java index 316cf89..913de94 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java @@ -9,6 +9,7 @@ import org.apache.http.client.CredentialsProvider; import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.RestClient; +import gov.nasa.pds.registry.common.connection.config.DirectType; import gov.nasa.pds.registry.common.connection.es.RestClientWrapper; public class Direct implements Cloneable, ConnectionFactory { @@ -20,15 +21,15 @@ public class Direct implements Cloneable, ConnectionFactory { private String api = null; private String index = null; - public static Direct build (String url, AuthContent auth, boolean trustSelfSigned) throws Exception { - URL service = new URL(url); + public static Direct build (DirectType url, AuthContent auth) throws Exception { + URL service = new URL(url.getValue()); // Trust self-signed certificates - if(trustSelfSigned) + if(url.isTrustSelfSigned()) { SSLContext sslCtx = SSLUtils.createTrustAllContext(); HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory()); } - return new Direct (service, auth, trustSelfSigned); + return new Direct (service, auth, url.isTrustSelfSigned()); } private Direct (URL service, AuthContent auth, boolean trustSelfSigned) { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java index 7b5b956..003f463 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java @@ -14,9 +14,9 @@ import org.apache.http.client.CredentialsProvider; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; -import gov.nasa.pds.registry.common.CognitoContent; import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.RestClient; +import gov.nasa.pds.registry.common.connection.config.CognitoType; public final class MultiTenancy implements ConnectionFactory { final private AuthContent auth; @@ -24,7 +24,7 @@ public final class MultiTenancy implements ConnectionFactory { final private URL signed; private String api = null; private String index = null; - public static MultiTenancy build (CognitoContent cog, AuthContent auth) throws IOException, InterruptedException { + public static MultiTenancy build (CognitoType cog, AuthContent auth) throws IOException, InterruptedException { boolean expectedContent = true; Gson gson = new Gson(); HttpClient client = HttpClient.newHttpClient(); @@ -33,7 +33,7 @@ public static MultiTenancy build (CognitoContent cog, AuthContent auth) throws I .POST(BodyPublishers.ofString("{\"AuthFlow\":\"USER_PASSWORD_AUTH\",\"AuthParameters\":{" + "\"USERNAME\":\"" + auth.getUser() + "\"," + "\"PASSWORD\":\"" + auth.getPassword() + "\"" - + "},\"ClientId\":\"" + cog.getClientID() + "\"" + + "},\"ClientId\":\"" + cog.getValue() + "\"" + "}")) .setHeader("X-Amz-Target", "AWSCognitoIdentityProviderService.InitiateAuth") .setHeader("Content-Type", "application/x-amz-json-1.1") @@ -60,7 +60,7 @@ public static MultiTenancy build (CognitoContent cog, AuthContent auth) throws I request = HttpRequest.newBuilder() .uri(URI.create(cog.getGateway())) .GET() - .setHeader("Authorization", content.get("AuthenticationResult").get("AccessToken")) + .setHeader("Authorization", content.get("AuthenticationResult").get("TokenType") + " " + content.get("AuthenticationResult").get("AccessToken")) .setHeader("IDToken", content.get("AuthenticationResult").get("IdToken")) .build(); response = client.send(request, HttpResponse.BodyHandlers.ofString()); diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java b/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java new file mode 100644 index 0000000..8601dbf --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java @@ -0,0 +1,43 @@ +package gov.nasa.pds.registry.common.connection; + +import java.io.IOException; +import java.net.URL; +import java.util.Arrays; +import org.glassfish.jaxb.runtime.v2.JAXBContextFactory; +import gov.nasa.pds.registry.common.connection.config.CognitoType; +import gov.nasa.pds.registry.common.connection.config.DirectType; +import gov.nasa.pds.registry.common.connection.config.RegistryConnection; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; + +public final class RegistryConnectionContent { + public static RegistryConnectionContent from (URL registry_connection) throws IOException, JAXBException { + JAXBContext jaxbContext = new JAXBContextFactory().createContext(new Class[]{RegistryConnection.class}, null); + RegistryConnection xml; + String acceptable[] = { "app", "file", "http", "https" }; + if (!Arrays.stream(acceptable).anyMatch(registry_connection.getProtocol()::equalsIgnoreCase)) { + throw new IllegalArgumentException("URL protocol must be one of " + acceptable + " not the one specified: " + registry_connection); + } + xml = (RegistryConnection)jaxbContext.createUnmarshaller().unmarshal(registry_connection); + return new RegistryConnectionContent(xml); + } + final private RegistryConnection content; + private RegistryConnectionContent (RegistryConnection xml) { + this.content = xml; + } + public CognitoType getCognitoClientId() { + return this.content.getCognitoClientId(); + } + public String getIndex() { + return this.content.getIndex(); + } + public DirectType getServerUrl() { + return this.content.getServerUrl(); + } + public boolean isCognitoConnection() { + return this.content.getCognitoClientId() != null; + } + public boolean isDirectConnection() { + return this.content.getServerUrl() != null; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/config/CognitoType.java b/src/main/java/gov/nasa/pds/registry/common/connection/config/CognitoType.java new file mode 100644 index 0000000..a07bc4a --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/config/CognitoType.java @@ -0,0 +1,149 @@ +// +// This file was generated by the Eclipse Implementation of JAXB, v4.0.3 +// See https://eclipse-ee4j.github.io/jaxb-ri +// Any modifications to this file will be lost upon recompilation of the source schema. +// + + +package gov.nasa.pds.registry.common.connection.config; + +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlSchemaType; +import jakarta.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlValue; +import jakarta.xml.bind.annotation.adapters.NormalizedStringAdapter; +import jakarta.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + + +/** + * + * Currently, cognito to serverless opensearh requires three bits of + * information: + * 1. clientID: magic ID which must be given and is not a secret but + * sure looks like one + * 2. @gateway: an endpoint that exchanges the cognito authentication + * to a signed URL that can be used for communicating with + * opensearch on AWS + * 3. @IDP: the cognito authentication endpoint that exchanges the + * username/password to a set of tokens used by the gateway. + * + * + *

Java class for cognito_type complex type. + * + *

The following schema fragment specifies the expected content contained within this class. + * + * 

{@code
+ * 
+ *   
+ *     
+ *       
+ *       
+ *     
+ *   
+ * 
+ * }
+ * + * + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "cognito_type", propOrder = { + "value" +}) +public class CognitoType { + + @XmlValue + @XmlJavaTypeAdapter(NormalizedStringAdapter.class) + @XmlSchemaType(name = "normalizedString") + protected String value; + @XmlAttribute(name = "gateway") + @XmlJavaTypeAdapter(NormalizedStringAdapter.class) + @XmlSchemaType(name = "normalizedString") + protected String gateway; + @XmlAttribute(name = "IDP") + @XmlJavaTypeAdapter(NormalizedStringAdapter.class) + @XmlSchemaType(name = "normalizedString") + protected String idp; + + /** + * Gets the value of the value property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getValue() { + return value; + } + + /** + * Sets the value of the value property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setValue(String value) { + this.value = value; + } + + /** + * Gets the value of the gateway property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getGateway() { + if (gateway == null) { + return "https://c8u1zk30u5.execute-api.us-west-2.amazonaws.com/dev/signed-url"; + } else { + return gateway; + } + } + + /** + * Sets the value of the gateway property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setGateway(String value) { + this.gateway = value; + } + + /** + * Gets the value of the idp property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getIDP() { + if (idp == null) { + return "cognito-idp.us-west-2.amazonaws.com"; + } else { + return idp; + } + } + + /** + * Sets the value of the idp property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setIDP(String value) { + this.idp = value; + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/config/DirectType.java b/src/main/java/gov/nasa/pds/registry/common/connection/config/DirectType.java new file mode 100644 index 0000000..a343574 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/config/DirectType.java @@ -0,0 +1,113 @@ +// +// This file was generated by the Eclipse Implementation of JAXB, v4.0.3 +// See https://eclipse-ee4j.github.io/jaxb-ri +// Any modifications to this file will be lost upon recompilation of the source schema. +// + + +package gov.nasa.pds.registry.common.connection.config; + +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlSchemaType; +import jakarta.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlValue; +import jakarta.xml.bind.annotation.adapters.NormalizedStringAdapter; +import jakarta.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + + +/** + * + * Currently, connecting to an opensearch service directly requires just + * two pieces of information: + * 1. URL: the URL to directly contact an opensearch instance and its + * API to include authentication + * 2. @trustSelfSigned: when running locally for testing or development + * it is often necessary to use self signed + * certificates. Setting this option to true + * will all for self signed certificates. + * + * + *

Java class for direct_type complex type. + * + *

The following schema fragment specifies the expected content contained within this class. + * + * 

{@code
+ * 
+ *   
+ *     
+ *       
+ *     
+ *   
+ * 
+ * }
+ * + * + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "direct_type", propOrder = { + "value" +}) +public class DirectType { + + @XmlValue + @XmlJavaTypeAdapter(NormalizedStringAdapter.class) + @XmlSchemaType(name = "normalizedString") + protected String value; + @XmlAttribute(name = "trust_self_signed") + protected Boolean trustSelfSigned; + + /** + * Gets the value of the value property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getValue() { + return value; + } + + /** + * Sets the value of the value property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setValue(String value) { + this.value = value; + } + + /** + * Gets the value of the trustSelfSigned property. + * + * @return + * possible object is + * {@link Boolean } + * + */ + public boolean isTrustSelfSigned() { + if (trustSelfSigned == null) { + return false; + } else { + return trustSelfSigned; + } + } + + /** + * Sets the value of the trustSelfSigned property. + * + * @param value + * allowed object is + * {@link Boolean } + * + */ + public void setTrustSelfSigned(Boolean value) { + this.trustSelfSigned = value; + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/config/ObjectFactory.java b/src/main/java/gov/nasa/pds/registry/common/connection/config/ObjectFactory.java new file mode 100644 index 0000000..f25274b --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/config/ObjectFactory.java @@ -0,0 +1,68 @@ +// +// This file was generated by the Eclipse Implementation of JAXB, v4.0.3 +// See https://eclipse-ee4j.github.io/jaxb-ri +// Any modifications to this file will be lost upon recompilation of the source schema. +// + + +package gov.nasa.pds.registry.common.connection.config; + +import jakarta.xml.bind.annotation.XmlRegistry; + + +/** + * This object contains factory methods for each + * Java content interface and Java element interface + * generated in the gov.nasa.pds.registry.common.connection.config package. + *

An ObjectFactory allows you to programmatically + * construct new instances of the Java representation + * for XML content. The Java representation of XML + * content can consist of schema derived interfaces + * and classes representing the binding of schema + * type definitions, element declarations and model + * groups. Factory methods for each of these are + * provided in this class. + * + */ +@XmlRegistry +public class ObjectFactory { + + + /** + * Create a new ObjectFactory that can be used to create new instances of schema derived classes for package: gov.nasa.pds.registry.common.connection.config + * + */ + public ObjectFactory() { + } + + /** + * Create an instance of {@link RegistryConnection } + * + * @return + * the new instance of {@link RegistryConnection } + */ + public RegistryConnection createRegistryConnection() { + return new RegistryConnection(); + } + + /** + * Create an instance of {@link CognitoType } + * + * @return + * the new instance of {@link CognitoType } + */ + public CognitoType createCognitoType() { + return new CognitoType(); + } + + /** + * Create an instance of {@link DirectType } + * + * @return + * the new instance of {@link DirectType } + */ + public DirectType createDirectType() { + return new DirectType(); + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/config/RegistryConnection.java b/src/main/java/gov/nasa/pds/registry/common/connection/config/RegistryConnection.java new file mode 100644 index 0000000..3fe0462 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/config/RegistryConnection.java @@ -0,0 +1,142 @@ +// +// This file was generated by the Eclipse Implementation of JAXB, v4.0.3 +// See https://eclipse-ee4j.github.io/jaxb-ri +// Any modifications to this file will be lost upon recompilation of the source schema. +// + + +package gov.nasa.pds.registry.common.connection.config; + +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlRootElement; +import jakarta.xml.bind.annotation.XmlSchemaType; +import jakarta.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.adapters.NormalizedStringAdapter; +import jakarta.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + + +/** + * + * This terrible construct is so that xjc can autodetect this as the + * root node for processing. Many things would be better but this is + * the most workable solution especially if the making of the binding + * code is automated in the pom. The only other real solution is to + * modify one of the classes generated by hand. + * + * + *

Java class for anonymous complex type. + * + *

The following schema fragment specifies the expected content contained within this class. + * + * 

{@code
+ * 
+ *   
+ *     
+ *       
+ *         
+ *         
+ *       
+ *       
+ *     
+ *   
+ * 
+ * }
+ * + * + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "", propOrder = { + "cognitoClientId", + "serverUrl" +}) +@XmlRootElement(name = "registry_connection") +public class RegistryConnection { + + protected CognitoType cognitoClientId; + @XmlElement(name = "server_url") + protected DirectType serverUrl; + @XmlAttribute(name = "index") + @XmlJavaTypeAdapter(NormalizedStringAdapter.class) + @XmlSchemaType(name = "normalizedString") + protected String index; + + /** + * Gets the value of the cognitoClientId property. + * + * @return + * possible object is + * {@link CognitoType } + * + */ + public CognitoType getCognitoClientId() { + return cognitoClientId; + } + + /** + * Sets the value of the cognitoClientId property. + * + * @param value + * allowed object is + * {@link CognitoType } + * + */ + public void setCognitoClientId(CognitoType value) { + this.cognitoClientId = value; + } + + /** + * Gets the value of the serverUrl property. + * + * @return + * possible object is + * {@link DirectType } + * + */ + public DirectType getServerUrl() { + return serverUrl; + } + + /** + * Sets the value of the serverUrl property. + * + * @param value + * allowed object is + * {@link DirectType } + * + */ + public void setServerUrl(DirectType value) { + this.serverUrl = value; + } + + /** + * Gets the value of the index property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getIndex() { + if (index == null) { + return "registry"; + } else { + return index; + } + } + + /** + * Sets the value of the index property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setIndex(String value) { + this.index = value; + } + +} diff --git a/src/main/resources/registry-connection.xsd b/src/main/resources/registry-connection.xsd new file mode 100644 index 0000000..5184956 --- /dev/null +++ b/src/main/resources/registry-connection.xsd @@ -0,0 +1,105 @@ + + + + + + + + + Currently, cognito to serverless opensearh requires three bits of + information: + 1. clientID: magic ID which must be given and is not a secret but + sure looks like one + 2. @gateway: an endpoint that exchanges the cognito authentication + to a signed URL that can be used for communicating with + opensearch on AWS + 3. @IDP: the cognito authentication endpoint that exchanges the + username/password to a set of tokens used by the gateway. + + + + + + + + + + + + + + Currently, connecting to an opensearch service directly requires just + two pieces of information: + 1. URL: the URL to directly contact an opensearch instance and its + API to include authentication + 2. @trustSelfSigned: when running locally for testing or development + it is often necessary to use self signed + certificates. Setting this option to true + will all for self signed certificates. + + + + + + + + + + + + + + + This terrible construct is so that xjc can autodetect this as the + root node for processing. Many things would be better but this is + the most workable solution especially if the making of the binding + code is automated in the pom. The only other real solution is to + modify one of the classes generated by hand. + + + + + + Define the connection to the registry, security for the connection, + and the index within the registry. + + @index: the index to be used by harvest whose default is registry + + cognito: the cognito client ID, IDP and gateway for AWS based + instances of opensearch + server_url: the opensearch URL when not using AWS services + + + + + + + + + + diff --git a/src/test/java/dao/TestDataDictionaryDao.java b/src/test/java/dao/TestDataDictionaryDao.java index f62aee3..47ae1a1 100644 --- a/src/test/java/dao/TestDataDictionaryDao.java +++ b/src/test/java/dao/TestDataDictionaryDao.java @@ -24,13 +24,13 @@ public static void main(String[] args) throws Exception private static void testListBooleanFields() throws Exception { - RestClient client = EstablishConnectionFactory.directly("http://localhost:9200").createRestClient(); + RestClient client = EstablishConnectionFactory.from("http://localhost:9200").createRestClient(); try { DataDictionaryDao dao = new DataDictionaryDao(client, "registry"); Set list = dao.getFieldNamesByEsType("boolean"); - + System.out.println("Boolean fields count = " + list.size()); list.forEach((name) -> { System.out.println(name); }); } @@ -43,7 +43,7 @@ private static void testListBooleanFields() throws Exception private static void testListDateFields() throws Exception { - RestClient client = EstablishConnectionFactory.directly("http://localhost:9200").createRestClient(); + RestClient client = EstablishConnectionFactory.from("http://localhost:9200").createRestClient(); try { @@ -62,7 +62,7 @@ private static void testListDateFields() throws Exception private static void testListLdds() throws Exception { - RestClient client = EstablishConnectionFactory.directly("http://localhost:9200").createRestClient(); + RestClient client = EstablishConnectionFactory.from("http://localhost:9200").createRestClient(); try { @@ -85,7 +85,7 @@ private static void testListLdds() throws Exception private static void testGetLddInfo() throws Exception { - RestClient client = EstablishConnectionFactory.directly("http://localhost:9200").createRestClient(); + RestClient client = EstablishConnectionFactory.from("http://localhost:9200").createRestClient(); try { diff --git a/src/test/java/dao/TestProductDao.java b/src/test/java/dao/TestProductDao.java index 12bdcc2..66397c7 100644 --- a/src/test/java/dao/TestProductDao.java +++ b/src/test/java/dao/TestProductDao.java @@ -21,7 +21,7 @@ public static void main(String[] args) throws Exception try { - client = EstablishConnectionFactory.directly("localhost").createRestClient(); + client = EstablishConnectionFactory.from("localhost").createRestClient(); ProductDao dao = new ProductDao(client, "registry"); //testUpdateArchiveStatus(dao); diff --git a/src/test/java/tt/TestRestClient.java b/src/test/java/tt/TestRestClient.java index 79eea49..dd624f8 100644 --- a/src/test/java/tt/TestRestClient.java +++ b/src/test/java/tt/TestRestClient.java @@ -9,7 +9,7 @@ public class TestRestClient public static void main(String[] args) throws Exception { - RestClient client = EstablishConnectionFactory.directly("localhost").createRestClient(); + RestClient client = EstablishConnectionFactory.from("localhost").createRestClient(); client.close(); } From 138f0bcf50293835696dfde859f03b0e861a6b7a Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 13 Feb 2024 13:47:02 -0800 Subject: [PATCH 13/41] add new registrty connection Moved to an XML file for registry connection information. Also added some support for various URL protocols to shorten the names (app). Internally, always use the jar: protcol because it is naturally supported by Java. Did an operation test then removed code as it is good for development but not regression. --- .../nasa/pds/registry/common/app/Handler.java | 16 ++++++ .../connection/KnownRegistryConnections.java | 56 +++++++++++++++++++ .../connection/RegistryConnectionContent.java | 6 +- .../connections/direct/localhost.xml | 4 ++ 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 src/main/java/gov/nasa/pds/registry/common/app/Handler.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/KnownRegistryConnections.java create mode 100644 src/main/resources/connections/direct/localhost.xml diff --git a/src/main/java/gov/nasa/pds/registry/common/app/Handler.java b/src/main/java/gov/nasa/pds/registry/common/app/Handler.java new file mode 100644 index 0000000..ecc3554 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/app/Handler.java @@ -0,0 +1,16 @@ +package gov.nasa.pds.registry.common.app; + +import java.io.IOException; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; + +public class Handler extends URLStreamHandler { + + @Override + protected URLConnection openConnection(URL u) throws IOException { + System.out.println("here"); + return null; + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/KnownRegistryConnections.java b/src/main/java/gov/nasa/pds/registry/common/connection/KnownRegistryConnections.java new file mode 100644 index 0000000..3dbc3b8 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/KnownRegistryConnections.java @@ -0,0 +1,56 @@ +package gov.nasa.pds.registry.common.connection; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Stream; + +public final class KnownRegistryConnections { + static { initialzeAppHandler(); } + static public void initialzeAppHandler() { + String handlers = System.getProperty("java.protocol.handler.pkgs"); + if (handlers == null) { + handlers = "gov.nasa.pds.registry.common"; + } else if (!handlers.contains("gov.nasa.pds.registry.common")) { + handlers += "|gov.nasa.pds.registry.common"; + } + System.setProperty("java.protocol.handler.pkgs", handlers); + } + static public List list() throws URISyntaxException, IOException { + return KnownRegistryConnections.list(KnownRegistryConnections.class); + } + static public List list(@SuppressWarnings("rawtypes") Class cls) throws URISyntaxException, IOException { + ArrayList result = new ArrayList(); + Path root; + URL resource = cls.getResource("/connections"); + if (resource == null) { + throw new RuntimeException("Resource files are not packaged in the jar file containing class " + cls.getName()); + } + if (resource.getProtocol().equalsIgnoreCase("jar")) { + FileSystem jarfiles = FileSystems.newFileSystem(resource.toURI(), Collections.emptyMap()); + root = jarfiles.getPath("/connections"); + } else { + root = Paths.get(resource.toURI()); + } + try (Stream walk = Files.walk(root, Integer.MAX_VALUE)) { + for (Iterator i = walk.iterator() ; i.hasNext();) { + String candidate = i.next().toString(); + candidate = candidate.substring(candidate.indexOf("/connections")); + if (candidate.endsWith(".xml")) { + result.add(new URL("app:" + candidate)); + } + } + } + for (URL url : result) System.out.println(url.toString()); + return result; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java b/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java index 8601dbf..8fd4177 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java @@ -11,13 +11,17 @@ import jakarta.xml.bind.JAXBException; public final class RegistryConnectionContent { + static { KnownRegistryConnections.initialzeAppHandler(); } public static RegistryConnectionContent from (URL registry_connection) throws IOException, JAXBException { JAXBContext jaxbContext = new JAXBContextFactory().createContext(new Class[]{RegistryConnection.class}, null); RegistryConnection xml; - String acceptable[] = { "app", "file", "http", "https" }; + String acceptable[] = { "app", "file", "jar", "http", "https" }; if (!Arrays.stream(acceptable).anyMatch(registry_connection.getProtocol()::equalsIgnoreCase)) { throw new IllegalArgumentException("URL protocol must be one of " + acceptable + " not the one specified: " + registry_connection); } + if (registry_connection.getProtocol().equalsIgnoreCase("app")) { + registry_connection = RegistryConnectionContent.class.getResource(registry_connection.getPath()); + } xml = (RegistryConnection)jaxbContext.createUnmarshaller().unmarshal(registry_connection); return new RegistryConnectionContent(xml); } diff --git a/src/main/resources/connections/direct/localhost.xml b/src/main/resources/connections/direct/localhost.xml new file mode 100644 index 0000000..ae8ab14 --- /dev/null +++ b/src/main/resources/connections/direct/localhost.xml @@ -0,0 +1,4 @@ + + + https://localhost:9200 + From 5fac3ccf8128813c3f3a979d20dc30cfc207f960 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Mon, 19 Feb 2024 12:12:27 -0800 Subject: [PATCH 14/41] debug help --- .../gov/nasa/pds/registry/common/connection/MultiTenancy.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java index 003f463..00b6774 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java @@ -63,6 +63,8 @@ public static MultiTenancy build (CognitoType cog, AuthContent auth) throws IOEx .setHeader("Authorization", content.get("AuthenticationResult").get("TokenType") + " " + content.get("AuthenticationResult").get("AccessToken")) .setHeader("IDToken", content.get("AuthenticationResult").get("IdToken")) .build(); + System.out.println ("Authorization : " + content.get("AuthenticationResult").get("TokenType") + " " + content.get("AuthenticationResult").get("AccessToken")); + System.out.println ("IDToken : " + content.get("AuthenticationResult").get("IdToken")); response = client.send(request, HttpResponse.BodyHandlers.ofString()); System.out.println (response); System.out.println (response.body()); From 669fa723a2a48788a35181b31d0824656520e627 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Mon, 19 Feb 2024 12:16:36 -0800 Subject: [PATCH 15/41] update to new connection URL --- .../nasa/pds/registry/common/connection/es/EsUtils.java | 2 +- .../registry/common/connection/es/RestClientWrapper.java | 2 +- .../gov/nasa/pds/registry/common/es/dao/DataLoader.java | 2 +- src/test/java/dao/TestDataDictionaryDao.java | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java index 9cb2f32..0407f9d 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java @@ -17,7 +17,7 @@ class EsUtils { /** * Parse Elasticsearch URL - * @param url Elasticsearch URL, e.g., "http://localhost:9200" + * @param url Elasticsearch URL, e.g., "app:/connections/direct/localhost.xml" * @return HTTP host information * @throws Exception an exception */ diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java index e390fd0..5fe27ad 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java @@ -19,7 +19,7 @@ public class RestClientWrapper implements RestClient final org.elasticsearch.client.RestClient real_client; /** * Constructor. - * @param url Elasticsearch URL, e.g., "http://localhost:9200" + * @param url Elasticsearch URL, e.g., "app:/connections/direct/localhost.xml" * @throws Exception an exception */ public RestClientWrapper(ConnectionFactory conFact) throws Exception diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java index a6b457c..9038d1c 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java @@ -45,7 +45,7 @@ public class DataLoader /** * Constructor - * @param esUrl Elasticsearch URL, e.g., "http://localhost:9200" + * @param esUrl Elasticsearch URL, e.g., "app:/connections/direct/localhost.xml" * @param indexName Elasticsearch index name * @param authConfigFile Elasticsearch authentication configuration file * (see Registry Manager documentation for more info) diff --git a/src/test/java/dao/TestDataDictionaryDao.java b/src/test/java/dao/TestDataDictionaryDao.java index 47ae1a1..fef177c 100644 --- a/src/test/java/dao/TestDataDictionaryDao.java +++ b/src/test/java/dao/TestDataDictionaryDao.java @@ -24,7 +24,7 @@ public static void main(String[] args) throws Exception private static void testListBooleanFields() throws Exception { - RestClient client = EstablishConnectionFactory.from("http://localhost:9200").createRestClient(); + RestClient client = EstablishConnectionFactory.from("app:/connections/direct/localhost.xml").createRestClient(); try { @@ -43,7 +43,7 @@ private static void testListBooleanFields() throws Exception private static void testListDateFields() throws Exception { - RestClient client = EstablishConnectionFactory.from("http://localhost:9200").createRestClient(); + RestClient client = EstablishConnectionFactory.from("app:/connections/direct/localhost.xml").createRestClient(); try { @@ -62,7 +62,7 @@ private static void testListDateFields() throws Exception private static void testListLdds() throws Exception { - RestClient client = EstablishConnectionFactory.from("http://localhost:9200").createRestClient(); + RestClient client = EstablishConnectionFactory.from("app:/connections/direct/localhost.xml").createRestClient(); try { @@ -85,7 +85,7 @@ private static void testListLdds() throws Exception private static void testGetLddInfo() throws Exception { - RestClient client = EstablishConnectionFactory.from("http://localhost:9200").createRestClient(); + RestClient client = EstablishConnectionFactory.from("app:/connections/direct/localhost.xml").createRestClient(); try { From 08a15dc041533ef4597843c9209f08fcef291f57 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Fri, 23 Feb 2024 12:22:28 -0800 Subject: [PATCH 16/41] testing in progress --- .../common/connection/MultiTenancy.java | 25 ++++++++++++------- .../es/ResponseExceptionWrapper.java | 22 ++++++++-------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java index 00b6774..c86a5f5 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java @@ -17,8 +17,10 @@ import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.common.connection.config.CognitoType; +import gov.nasa.pds.registry.common.connection.es.RestClientWrapper; public final class MultiTenancy implements ConnectionFactory { + final private int timeout = 5000; final private AuthContent auth; final private HttpHost host; final private URL signed; @@ -63,12 +65,11 @@ public static MultiTenancy build (CognitoType cog, AuthContent auth) throws IOEx .setHeader("Authorization", content.get("AuthenticationResult").get("TokenType") + " " + content.get("AuthenticationResult").get("AccessToken")) .setHeader("IDToken", content.get("AuthenticationResult").get("IdToken")) .build(); - System.out.println ("Authorization : " + content.get("AuthenticationResult").get("TokenType") + " " + content.get("AuthenticationResult").get("AccessToken")); - System.out.println ("IDToken : " + content.get("AuthenticationResult").get("IdToken")); response = client.send(request, HttpResponse.BodyHandlers.ofString()); - System.out.println (response); - System.out.println (response.body()); - URL signed = null; + if (299 < response.statusCode()) { + throw new IOException("Could not obtain signed URL: " + response.toString()); + } + URL signed = new URL(response.body().substring(response.body().indexOf("https://"))); return new MultiTenancy(auth, signed); } @@ -83,13 +84,19 @@ public ConnectionFactory clone() { } @Override public HttpURLConnection createConnection() throws IOException { - // TODO Auto-generated method stub - return null; + String url = this.signed.toString(); + if (this.index != null) url += "/" + this.index; + if (this.api != null) url += "/" + this.api; + HttpURLConnection con = (HttpURLConnection)new URL(url).openConnection(); + con.setConnectTimeout(this.timeout); + con.setReadTimeout(this.timeout); + con.setAllowUserInteraction(false); + con.setRequestProperty("Authorization", this.auth.getHeader()); + return con; } @Override public RestClient createRestClient() throws Exception { - // TODO Auto-generated method stub - return null; + return new RestClientWrapper(this); } @Override public CredentialsProvider getCredentials() { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java index d132fae..fcf1824 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java @@ -14,47 +14,47 @@ final class ResponseExceptionWrapper extends ResponseException { } @Override public String getMessage() { - return this.real_exception.getMessage(); + return this.real_exception == null ? super.getMessage() : this.real_exception.getMessage(); } @Override public String getLocalizedMessage() { - return this.real_exception.getLocalizedMessage(); + return this.real_exception == null ? super.getLocalizedMessage() : this.real_exception.getLocalizedMessage(); } @Override public synchronized Throwable getCause() { - return this.real_exception.getCause(); + return this.real_exception == null ? super.getCause() : this.real_exception.getCause(); } @Override public synchronized Throwable initCause(Throwable cause) { - return this.real_exception.initCause(cause); + return this.real_exception == null ? super.initCause(cause) : this.real_exception.initCause(cause); } @Override public String toString() { - return this.real_exception.toString(); + return this.real_exception == null ? super.toString() : this.real_exception.toString(); } @Override public void printStackTrace() { - this.real_exception.printStackTrace(); + if (this.real_exception == null) super.printStackTrace(); else this.real_exception.printStackTrace(); } @Override public void printStackTrace(PrintStream s) { - this.real_exception.printStackTrace(s); + if (this.real_exception == null) super.printStackTrace(s); else this.real_exception.printStackTrace(s); } @Override public void printStackTrace(PrintWriter s) { - this.real_exception.printStackTrace(s); + if (this.real_exception == null) super.printStackTrace(s); else this.real_exception.printStackTrace(s); } @Override public synchronized Throwable fillInStackTrace() { - return this.real_exception.fillInStackTrace(); + return this.real_exception == null ? super.fillInStackTrace() : this.real_exception.fillInStackTrace(); } @Override public StackTraceElement[] getStackTrace() { - return this.real_exception.getStackTrace(); + return this.real_exception == null ? super.getStackTrace() : this.real_exception.getStackTrace(); } @Override public void setStackTrace(StackTraceElement[] stackTrace) { - this.real_exception.setStackTrace(stackTrace); + if (this.real_exception == null) super.setStackTrace(stackTrace); else this.real_exception.setStackTrace(stackTrace); } @Override public Response getResponse() { From 0e57b33edd58fe5a404a97e06f9ad57e0606a5d6 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 27 Mar 2024 07:06:07 -0700 Subject: [PATCH 17/41] wip --- pom.xml | 21 ++++++++++++++++ .../common/connection/MultiTenancy.java | 24 +++++++++++++++--- .../common/connection/aws/RequestWrapper.java | 13 ++++++++++ .../connection/aws/ResponseWrapper.java | 23 +++++++++++++++++ .../connection/aws/RestClientWrapper.java | 25 +++++++++++++++++++ 5 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/RequestWrapper.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/ResponseWrapper.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java diff --git a/pom.xml b/pom.xml index 5823903..899594f 100644 --- a/pom.xml +++ b/pom.xml @@ -132,6 +132,27 @@ POSSIBILITY OF SUCH DAMAGE. jakarta.activation-api 2.1.2 + + + org.opensearch.client + opensearch-java + 2.9.0 + + + org.apache.httpcomponents.client5 + httpclient5 + 5.3.1 + + + software.amazon.awssdk + opensearch + 2.25.6 + + + software.amazon.awssdk + sts + 2.25.13 + diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java index c86a5f5..924baf3 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java @@ -10,14 +10,15 @@ import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse; import java.util.Map; +import java.util.Properties; import org.apache.http.HttpHost; import org.apache.http.client.CredentialsProvider; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.RestClient; +import gov.nasa.pds.registry.common.connection.aws.RestClientWrapper; import gov.nasa.pds.registry.common.connection.config.CognitoType; -import gov.nasa.pds.registry.common.connection.es.RestClientWrapper; public final class MultiTenancy implements ConnectionFactory { final private int timeout = 5000; @@ -42,6 +43,7 @@ public static MultiTenancy build (CognitoType cog, AuthContent auth) throws IOEx .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); Map> content; + Properties awsCreds = new Properties(System.getProperties()); Type contentType = new TypeToken>>(){}.getType(); expectedContent &= response.body().contains("AuthenticationResult"); @@ -69,7 +71,23 @@ public static MultiTenancy build (CognitoType cog, AuthContent auth) throws IOEx if (299 < response.statusCode()) { throw new IOException("Could not obtain signed URL: " + response.toString()); } - URL signed = new URL(response.body().substring(response.body().indexOf("https://"))); + expectedContent &= response.body().contains("IdentityId"); + expectedContent &= response.body().contains("Credentials"); + expectedContent &= response.body().contains("AccessKeyId"); + expectedContent &= response.body().contains("SecretKey"); + expectedContent &= response.body().contains("SessionToken"); + expectedContent &= response.body().contains("Expiration"); + expectedContent &= response.body().contains("ResponseMetadata"); + contentType = new TypeToken>(){}.getType(); + content = gson.fromJson(response.body(), contentType); + awsCreds.setProperty("aws.accessKeyId", content.get("Credentials").get("AccessKeyId")); + awsCreds.setProperty("aws.secretAccessKey", content.get("Credentials").get("SecretKey")); + awsCreds.setProperty("aws.sessionToken", content.get("Credentials").get("SessionToken")); + System.out.println (content.get("Credentials").get("AccessKeyId")); + System.out.println (content.get("Credentials").get("SecretKey")); + System.out.println (content.get("Credentials").get("SessionToken")); + System.setProperties(awsCreds); + URL signed = new URL("https://p5qmxrldysl1gy759hqf.us-west-2.aoss.amazonaws.com"); // move to config or lambda?? return new MultiTenancy(auth, signed); } @@ -96,7 +114,7 @@ public HttpURLConnection createConnection() throws IOException { } @Override public RestClient createRestClient() throws Exception { - return new RestClientWrapper(this); + return new RestClientWrapper(); } @Override public CredentialsProvider getCredentials() { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RequestWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RequestWrapper.java new file mode 100644 index 0000000..5c5e4d7 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RequestWrapper.java @@ -0,0 +1,13 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import gov.nasa.pds.registry.common.Request; + +class RequestWrapper implements Request { + + @Override + public void setJsonEntity(String entity) { + // TODO Auto-generated method stub + + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/ResponseWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/ResponseWrapper.java new file mode 100644 index 0000000..405a224 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/ResponseWrapper.java @@ -0,0 +1,23 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.apache.http.HttpEntity; +import org.apache.http.StatusLine; +import gov.nasa.pds.registry.common.Response; + +public class ResponseWrapper implements Response { + @Override + public HttpEntity getEntity() { + // TODO Auto-generated method stub + return null; + } + @Override + public StatusLine getStatusLine() { + // TODO Auto-generated method stub + return null; + } + @Override + public void printWarnings() { + // TODO Auto-generated method stub + + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java new file mode 100644 index 0000000..b20d5be --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java @@ -0,0 +1,25 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.io.IOException; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.Request.Method; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.RestClient; + +public class RestClientWrapper implements RestClient { + @Override + public void close() throws IOException { + // TODO Auto-generated method stub + } + @Override + public Request createRequest(Method method, String endpoint) { + // TODO Auto-generated method stub + return null; + } + @Override + public Response performRequest(Request request) throws IOException { + // TODO Auto-generated method stub + return null; + } + +} From 16566e61ae1af2712fcd8bceb737e897db10c50d Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Sat, 30 Mar 2024 16:51:54 -0700 Subject: [PATCH 18/41] add a bulk wrapper for serverless and harvest --- pom.xml | 18 +- .../registry/common/ConnectionFactory.java | 3 - .../gov/nasa/pds/registry/common/Request.java | 5 + .../nasa/pds/registry/common/RestClient.java | 2 + .../registry/common/connection/Direct.java | 9 +- .../common/connection/MultiTenancy.java | 32 +-- .../connection/aws/RestClientWrapper.java | 35 ++- .../common/connection/es/BulkResponse.java | 27 +++ .../common/connection/es/BulkWrapper.java | 25 ++ .../connection/es/RestClientWrapper.java | 19 +- .../registry/common/es/dao/DataLoader.java | 227 +++--------------- 11 files changed, 162 insertions(+), 240 deletions(-) create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponse.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/BulkWrapper.java diff --git a/pom.xml b/pom.xml index 899594f..06c06c0 100644 --- a/pom.xml +++ b/pom.xml @@ -132,26 +132,26 @@ POSSIBILITY OF SUCH DAMAGE. jakarta.activation-api 2.1.2 - - - org.opensearch.client - opensearch-java - 2.9.0 - + org.apache.httpcomponents.client5 httpclient5 5.3.1 + + org.opensearch.client + opensearch-java + 2.9.1 + software.amazon.awssdk opensearch - 2.25.6 + 2.25.21 software.amazon.awssdk - sts - 2.25.13 + apache-client + 2.25.21 diff --git a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java index 0446e19..37e1784 100644 --- a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java @@ -1,12 +1,9 @@ package gov.nasa.pds.registry.common; -import java.io.IOException; -import java.net.HttpURLConnection; import org.apache.http.HttpHost; import org.apache.http.client.CredentialsProvider; public interface ConnectionFactory { - public HttpURLConnection createConnection() throws IOException; public RestClient createRestClient() throws Exception; public CredentialsProvider getCredentials(); public HttpHost getHost(); diff --git a/src/main/java/gov/nasa/pds/registry/common/Request.java b/src/main/java/gov/nasa/pds/registry/common/Request.java index 71bd2ec..8859c43 100644 --- a/src/main/java/gov/nasa/pds/registry/common/Request.java +++ b/src/main/java/gov/nasa/pds/registry/common/Request.java @@ -1,6 +1,11 @@ package gov.nasa.pds.registry.common; +import java.io.IOException; + public interface Request { + public interface Bulk { + public void add (String statement) throws IOException; + } enum Method { DELETE, GET, HEAD, POST, PUT }; public void setJsonEntity(String entity); } diff --git a/src/main/java/gov/nasa/pds/registry/common/RestClient.java b/src/main/java/gov/nasa/pds/registry/common/RestClient.java index 890b130..9d68687 100644 --- a/src/main/java/gov/nasa/pds/registry/common/RestClient.java +++ b/src/main/java/gov/nasa/pds/registry/common/RestClient.java @@ -4,6 +4,8 @@ import java.io.IOException; public interface RestClient extends Closeable { + public Request.Bulk createBulkRequest() throws IOException; public Request createRequest(Request.Method method, String endpoint); public Response performRequest(Request request) throws IOException; + public Response performRequest(Request.Bulk request) throws IOException; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java index 913de94..799da09 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java @@ -38,11 +38,6 @@ private Direct (URL service, AuthContent auth, boolean trustSelfSigned) { this.service = service; this.veryTrusting = trustSelfSigned; } - @Override - public ConnectionFactory clone() { - return new Direct(this.service, this.auth, this.veryTrusting).setAPI(this.api).setIndexName(this.index); - } - @Override public HttpURLConnection createConnection() throws IOException { String url = this.service.toString(); if (this.index != null) url += "/" + this.index; @@ -55,6 +50,10 @@ public HttpURLConnection createConnection() throws IOException { return con; } @Override + public ConnectionFactory clone() { + return new Direct(this.service, this.auth, this.veryTrusting).setAPI(this.api).setIndexName(this.index); + } + @Override public RestClient createRestClient() throws Exception { return new RestClientWrapper(this); } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java index 924baf3..026b64d 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.lang.reflect.Type; -import java.net.HttpURLConnection; import java.net.URI; import java.net.URL; import java.net.http.HttpClient; @@ -21,10 +20,9 @@ import gov.nasa.pds.registry.common.connection.config.CognitoType; public final class MultiTenancy implements ConnectionFactory { - final private int timeout = 5000; final private AuthContent auth; final private HttpHost host; - final private URL signed; + final private URL endpoint; private String api = null; private String index = null; public static MultiTenancy build (CognitoType cog, AuthContent auth) throws IOException, InterruptedException { @@ -43,7 +41,7 @@ public static MultiTenancy build (CognitoType cog, AuthContent auth) throws IOEx .build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); Map> content; - Properties awsCreds = new Properties(System.getProperties()); + Properties awsCreds = new Properties(System.getProperties()); // initialize properties as oracle suggests Type contentType = new TypeToken>>(){}.getType(); expectedContent &= response.body().contains("AuthenticationResult"); @@ -80,41 +78,27 @@ public static MultiTenancy build (CognitoType cog, AuthContent auth) throws IOEx expectedContent &= response.body().contains("ResponseMetadata"); contentType = new TypeToken>(){}.getType(); content = gson.fromJson(response.body(), contentType); + // fill then set system properties as oracle suggests (init happened above) awsCreds.setProperty("aws.accessKeyId", content.get("Credentials").get("AccessKeyId")); awsCreds.setProperty("aws.secretAccessKey", content.get("Credentials").get("SecretKey")); awsCreds.setProperty("aws.sessionToken", content.get("Credentials").get("SessionToken")); - System.out.println (content.get("Credentials").get("AccessKeyId")); - System.out.println (content.get("Credentials").get("SecretKey")); - System.out.println (content.get("Credentials").get("SessionToken")); System.setProperties(awsCreds); URL signed = new URL("https://p5qmxrldysl1gy759hqf.us-west-2.aoss.amazonaws.com"); // move to config or lambda?? return new MultiTenancy(auth, signed); } - private MultiTenancy (AuthContent auth, URL signed) { + private MultiTenancy (AuthContent auth, URL opensearchEndpoint) { this.auth = auth; - this.host = new HttpHost(signed.getHost(), signed.getPort(), signed.getProtocol()); - this.signed = signed; + this.endpoint = opensearchEndpoint; + this.host = new HttpHost(this.endpoint.getHost(), this.endpoint.getPort(), this.endpoint.getProtocol()); } @Override public ConnectionFactory clone() { - return new MultiTenancy(this.auth, this.signed).setAPI(this.api).setIndexName(this.index); - } - @Override - public HttpURLConnection createConnection() throws IOException { - String url = this.signed.toString(); - if (this.index != null) url += "/" + this.index; - if (this.api != null) url += "/" + this.api; - HttpURLConnection con = (HttpURLConnection)new URL(url).openConnection(); - con.setConnectTimeout(this.timeout); - con.setReadTimeout(this.timeout); - con.setAllowUserInteraction(false); - con.setRequestProperty("Authorization", this.auth.getHeader()); - return con; + return new MultiTenancy(this.auth, this.endpoint).setAPI(this.api).setIndexName(this.index); } @Override public RestClient createRestClient() throws Exception { - return new RestClientWrapper(); + return new RestClientWrapper(this); } @Override public CredentialsProvider getCredentials() { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java index b20d5be..e384a47 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java @@ -1,15 +1,38 @@ package gov.nasa.pds.registry.common.connection.aws; import java.io.IOException; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.transport.aws.AwsSdk2Transport; +import org.opensearch.client.transport.aws.AwsSdk2TransportOptions; +import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.Request.Bulk; import gov.nasa.pds.registry.common.Request.Method; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.regions.Region; import gov.nasa.pds.registry.common.Response; import gov.nasa.pds.registry.common.RestClient; public class RestClientWrapper implements RestClient { + final private OpenSearchClient client; + final private SdkHttpClient httpClient; + public RestClientWrapper(ConnectionFactory conFact) { + this.httpClient = ApacheHttpClient.builder().build(); + this.client = new OpenSearchClient( + new AwsSdk2Transport( + httpClient, + conFact.getHostName(), + "aoss", + Region.US_WEST_2, // signing service region that we should probably get from host name?? + AwsSdk2TransportOptions.builder().build() + ) + ); + } @Override public void close() throws IOException { - // TODO Auto-generated method stub + this.client.shutdown(); + this.httpClient.close(); } @Override public Request createRequest(Method method, String endpoint) { @@ -21,5 +44,15 @@ public Response performRequest(Request request) throws IOException { // TODO Auto-generated method stub return null; } + @Override + public Bulk createBulkRequest() throws IOException { + // TODO Auto-generated method stub + return null; + } + @Override + public Response performRequest(Bulk request) throws IOException { + // TODO Auto-generated method stub + return null; + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponse.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponse.java new file mode 100644 index 0000000..48c579b --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponse.java @@ -0,0 +1,27 @@ +package gov.nasa.pds.registry.common.connection.es; + +import org.apache.http.HttpEntity; +import org.apache.http.StatusLine; +import gov.nasa.pds.registry.common.Response; + +class BulkResponse implements Response { + final String lastLine; + BulkResponse(String lastLine) { + this.lastLine = lastLine; + } + @Override + public HttpEntity getEntity() { + return null; + } + @Override + public StatusLine getStatusLine() { + return null; + } + @Override + public void printWarnings() { + } + @Override + public String toString() { + return this.lastLine; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkWrapper.java new file mode 100644 index 0000000..579d8bd --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkWrapper.java @@ -0,0 +1,25 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.net.HttpURLConnection; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.connection.Direct; + +class BulkWrapper implements Request.Bulk { + final HttpURLConnection con; + final OutputStreamWriter writer; + BulkWrapper(Direct conFactory) throws IOException { + this.con = conFactory.createConnection(); + this.con.setDoInput(true); + this.con.setDoOutput(true); + this.con.setRequestMethod("POST"); + this.con.setRequestProperty("content-type", "application/x-ndjson; charset=utf-8"); + this.writer = new OutputStreamWriter(con.getOutputStream(), "UTF-8"); + } + @Override + public void add(String statement) throws IOException { + this.writer.write(statement); + this.writer.write("\n"); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java index 5fe27ad..e690948 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java @@ -2,11 +2,13 @@ import java.io.IOException; import org.elasticsearch.client.RestClientBuilder; -import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.Request.Bulk; import gov.nasa.pds.registry.common.Request.Method; import gov.nasa.pds.registry.common.Response; import gov.nasa.pds.registry.common.RestClient; +import gov.nasa.pds.registry.common.connection.Direct; +import gov.nasa.pds.registry.common.es.dao.DaoUtils; /** @@ -16,13 +18,13 @@ */ public class RestClientWrapper implements RestClient { + final Direct conFactory; final org.elasticsearch.client.RestClient real_client; /** * Constructor. - * @param url Elasticsearch URL, e.g., "app:/connections/direct/localhost.xml" * @throws Exception an exception */ - public RestClientWrapper(ConnectionFactory conFact) throws Exception + public RestClientWrapper(Direct conFact) throws Exception { ClientConfigCB clientCB = new ClientConfigCB(); RequestConfigCB reqCB = new RequestConfigCB(); @@ -31,6 +33,7 @@ public RestClientWrapper(ConnectionFactory conFact) throws Exception clientCB.setTrustSelfSignedCert(conFact.isTrustingSelfSigned()); bld.setHttpClientConfigCallback(clientCB); bld.setRequestConfigCallback(reqCB); + this.conFactory = conFact; this.real_client = bld.build(); } @Override @@ -48,5 +51,15 @@ public Response performRequest(Request request) throws IOException { } catch (org.elasticsearch.client.ResponseException e) { throw new ResponseExceptionWrapper(e); } + } + @Override + public Bulk createBulkRequest() throws IOException { + return new BulkWrapper(this.conFactory); + } + @Override + public Response performRequest(Bulk request) throws IOException { + ((BulkWrapper)request).writer.close(); + DaoUtils.getLastLine(((BulkWrapper)request).con.getInputStream()); + return null; } } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java index 9038d1c..8e570e7 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java @@ -4,11 +4,9 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.HttpURLConnection; import java.net.UnknownHostException; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Set; @@ -20,8 +18,9 @@ import com.google.gson.Gson; import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.Response; import gov.nasa.pds.registry.common.util.CloseUtils; -import gov.nasa.pds.registry.common.util.SearchResponseParser; /** @@ -160,43 +159,25 @@ private String loadBatch(BufferedReader fileReader, String firstLine) throws Exc */ private String loadBatch(BufferedReader fileReader, String firstLine, int retries) throws Exception { - HttpURLConnection con = null; - OutputStreamWriter writer = null; - + ArrayList statements = new ArrayList(); try { - con = conFactory.createConnection(); - con.setDoInput(true); - con.setDoOutput(true); - con.setRequestMethod("POST"); - con.setRequestProperty("content-type", "application/x-ndjson; charset=utf-8"); - - writer = new OutputStreamWriter(con.getOutputStream(), "UTF-8"); - // First record String line1 = firstLine; String line2 = fileReader.readLine(); if(line2 == null) throw new Exception("Premature end of file"); - - writer.write(line1); - writer.write("\n"); - writer.write(line2); - writer.write("\n"); + statements.add(line1); + statements.add(line2); int numRecords = 1; while(numRecords < batchSize) { line1 = fileReader.readLine(); if(line1 == null) break; - line2 = fileReader.readLine(); if(line2 == null) throw new Exception("Premature end of file"); - - writer.write(line1); - writer.write("\n"); - writer.write(line2); - writer.write("\n"); - + statements.add(line1); + statements.add(line2); numRecords++; } @@ -206,53 +187,18 @@ private String loadBatch(BufferedReader fileReader, String firstLine, int retrie line1 = fileReader.readLine(); if(line1 != null && line1.isEmpty()) line1 = null; } - - writer.flush(); - writer.close(); - - // Check for Elasticsearch errors. - String respJson = getLastLine(con.getInputStream()); - log.debug(respJson); - - if(responseHasErrors(respJson)) - { - throw new Exception("Could not load data."); + int uploaded = this.loadBatch(statements); + totalRecords += uploaded; + + if (uploaded != numRecords) { + throw new Exception ("failed to upload all documents"); } - - totalRecords += numRecords; - return line1; } catch(UnknownHostException ex) { throw new Exception("Unknown host " + conFactory.getHostName()); } - catch(IOException ex) - { - if (retries > 0) { - String msg = ex.getMessage(); - log.warn("DataLoader.loadBatch() request failed due to \"" + msg + "\" ("+ retries +" retries remaining)"); - return loadBatch(fileReader, firstLine, retries - 1); - } - - // Get HTTP response code - int respCode = getResponseCode(con); - if(respCode <= 0) throw ex; - - // Try extracting JSON from multi-line error response (last line) - String json = getLastLine(con.getErrorStream()); - if(json == null) throw ex; - - // Parse error JSON to extract reason. - String msg = SearchResponseParser.extractReasonFromJson(json); - if(msg == null) msg = json; - - throw new Exception(msg); - } - finally - { - CloseUtils.close(writer); - } } @@ -281,40 +227,25 @@ public int loadBatch(List data, Set errorLidvids, int retries) t if(data == null || data.isEmpty()) return 0; if(data.size() % 2 != 0) throw new Exception("Data list size should be an even number."); - HttpURLConnection con = null; - OutputStreamWriter writer = null; + Request.Bulk bulk = null; try { - con = conFactory.createConnection(); - con.setDoInput(true); - con.setDoOutput(true); - con.setRequestMethod("POST"); - con.setRequestProperty("content-type", "application/x-ndjson; charset=utf-8"); - - writer = new OutputStreamWriter(con.getOutputStream(), "UTF-8"); - - for(int i = 0; i < data.size(); i+=2) - { - writer.write(data.get(i)); - writer.write("\n"); - writer.write(data.get(i+1)); - writer.write("\n"); - } - - writer.flush(); - writer.close(); - - // Read Elasticsearch response. - String respJson = DaoUtils.getLastLine(con.getInputStream()); - log.debug(respJson); - - // Check for Elasticsearch errors. - int failedCount = processErrors(respJson, errorLidvids); - // Calculate number of successfully saved records - // NOTE: data list has two lines per record (primary key + data) - int loadedCount = data.size() / 2 - failedCount; - return loadedCount; + bulk = this.conFactory.createRestClient().createBulkRequest(); + for (String statement : data) { + bulk.add(statement); + } + Response response = this.conFactory.createRestClient().performRequest(bulk); + // Read Elasticsearch response. + String respJson = response.toString(); + log.debug(respJson); + + // Check for Elasticsearch errors. + int failedCount = processErrors(respJson, errorLidvids); + // Calculate number of successfully saved records + // NOTE: data list has two lines per record (primary key + data) + int loadedCount = data.size() / 2 - failedCount; + return loadedCount; } catch(UnknownHostException ex) { @@ -327,7 +258,8 @@ public int loadBatch(List data, Set errorLidvids, int retries) t log.warn("DataLoader.loadBatch() request failed due to \"" + msg + "\" ("+ retries +" retries remaining)"); return loadBatch(data, errorLidvids, retries - 1); } - + throw ex; +/* // Get HTTP response code int respCode = getResponseCode(con); if(respCode <= 0) throw ex; @@ -341,10 +273,7 @@ public int loadBatch(List data, Set errorLidvids, int retries) t if(msg == null) msg = json; throw new Exception(msg); - } - finally - { - CloseUtils.close(writer); + */ } } @@ -425,96 +354,4 @@ private int processErrors(String resp, Set errorLidvids) return 0; } } - - - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private boolean responseHasErrors(String resp) - { - try - { - // Parse JSON response - Gson gson = new Gson(); - Map json = (Map)gson.fromJson(resp, Object.class); - - Boolean hasErrors = (Boolean)json.get("errors"); - if(hasErrors) - { - List list = (List)json.get("items"); - - // List size = batch size (one item per document) - // NOTE: Only few items in the list could have errors - for(Object item: list) - { - Map index = (Map)((Map)item).get("index"); - Map error = (Map)index.get("error"); - if(error != null) - { - String message = (String)error.get("reason"); - log.error(message); - return true; - } - } - } - - return false; - } - catch(Exception ex) - { - return false; - } - } - - - /** - * Get HTTP response code, e.g., 200 (OK) - * @param con HTTP connection - * @return HTTP response code, e.g., 200 (OK) - */ - private static int getResponseCode(HttpURLConnection con) - { - if(con == null) return -1; - - try - { - return con.getResponseCode(); - } - catch(Exception ex) - { - return -1; - } - } - - - /** - * This method is used to parse multi-line Elasticsearch error responses. - * JSON error response is on the last line of a message. - * @param is input stream - * @return Last line - */ - private String getLastLine(InputStream is) - { - String lastLine = null; - - try - { - BufferedReader rd = new BufferedReader(new InputStreamReader(is)); - - String line; - while((line = rd.readLine()) != null) - { - lastLine = line; - } - } - catch(Exception ex) - { - log.info("Exception thrown in DataLoader.getLastLine() - please inform developer", ex); - } - finally - { - CloseUtils.close(is); - } - - return lastLine; - } } From e744a5adb33dca07595b11fd279f5d4500536001 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Sun, 31 Mar 2024 13:12:00 -0700 Subject: [PATCH 19/41] major refactor Really broke the ES knowledge from out of the existing Java code. Left some because it is as good as a configuration language as anything else. It does mean it will have to be parsed in the serverless half of the house. There are now interfaces that separate the commands and what needs to be done with those commands. Moved all of the turning of business data into ES JSON into its own utility and buried. Then fixed all of the broken bits. It is now more cleanly divided and ready for Java v2 implementation for serverless. --- .../registry/common/ConnectionFactory.java | 1 - .../gov/nasa/pds/registry/common/Request.java | 45 ++- .../nasa/pds/registry/common/RestClient.java | 20 +- .../registry/common/connection/Direct.java | 23 +- .../common/connection/MultiTenancy.java | 8 +- .../connection/aws/RestClientWrapper.java | 6 +- .../common/connection/es/BulkImpl.java | 40 ++ .../common/connection/es/BulkResponse.java | 27 -- .../common/connection/es/BulkWrapper.java | 25 -- .../common/connection/es/CountImpl.java | 22 ++ .../common/connection/es/GetImpl.java | 62 +++ .../common/connection/es/JsonHelper.java | 363 ++++++++++++++++++ .../common/connection/es/MappingImpl.java | 24 ++ .../common/connection/es/RequestWrapper.java | 27 -- .../connection/es/RestClientWrapper.java | 99 +++-- .../common/connection/es/SearchImpl.java | 49 +++ .../common/connection/es/SettingImpl.java | 16 + .../common/es/dao/BaseRequestBuilder.java | 68 ---- .../registry/common/es/dao/DataLoader.java | 9 +- .../registry/common/es/dao/ProductDao.java | 55 ++- .../common/es/dao/ProductRequestBuilder.java | 134 ------- .../common/es/dao/dd/DDRequestBuilder.java | 178 --------- .../common/es/dao/dd/DataDictionaryDao.java | 35 +- .../common/es/dao/schema/SchemaDao.java | 10 +- .../es/dao/schema/SchemaRequestBuilder.java | 67 ---- .../nasa/pds/registry/common/util/Tuple.java | 4 +- 26 files changed, 761 insertions(+), 656 deletions(-) create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/BulkImpl.java delete mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponse.java delete mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/BulkWrapper.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/CountImpl.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/GetImpl.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/JsonHelper.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/MappingImpl.java delete mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/RequestWrapper.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/SettingImpl.java delete mode 100644 src/main/java/gov/nasa/pds/registry/common/es/dao/BaseRequestBuilder.java delete mode 100644 src/main/java/gov/nasa/pds/registry/common/es/dao/ProductRequestBuilder.java delete mode 100644 src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DDRequestBuilder.java delete mode 100644 src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaRequestBuilder.java diff --git a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java index 37e1784..5d3a540 100644 --- a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java @@ -10,6 +10,5 @@ public interface ConnectionFactory { public String getHostName(); public String getIndexName(); public boolean isTrustingSelfSigned(); - public ConnectionFactory setAPI (String api); public ConnectionFactory setIndexName (String idxName); } diff --git a/src/main/java/gov/nasa/pds/registry/common/Request.java b/src/main/java/gov/nasa/pds/registry/common/Request.java index 8859c43..5a55652 100644 --- a/src/main/java/gov/nasa/pds/registry/common/Request.java +++ b/src/main/java/gov/nasa/pds/registry/common/Request.java @@ -1,11 +1,46 @@ package gov.nasa.pds.registry.common; -import java.io.IOException; +import java.util.Collection; +import java.util.List; +import gov.nasa.pds.registry.common.util.Tuple; public interface Request { - public interface Bulk { - public void add (String statement) throws IOException; + public interface Bulk { // _bulk + //public void add (String statement); // delete + public void add (String statement, String document); // create, index, update + public Bulk buildUpdateStatus(Collection lidvids, String status); + public Bulk setIndex(String name); + public Bulk setRefresh(String type); + } + public interface Count { // _count + public Count setIndex (String name); + public Count setQuery (String q); + } + public interface Get { // _doc + public Get excludeField (String field); + public Get excludeFields (List fields); + public Get includeField (String field); + public Get includeFields (List fields); + public Get setId (String id); + public Get setIndex (String index); + } + public interface Mapping { // _mapping + public Mapping buildUpdateFieldSchema (Collection pairs); + public Mapping setIndex(String name); + } + public interface MGet extends Get { // _mget + public MGet setIds (Collection ids); + } + public interface Search { // _search + public Search buildAlternativeIds(Collection lids); + public Search buildLatestLidVids(Collection lids); + public Search buildListFields(String dataType); + public Search buildListLdds (String namespace); + public Search buildTheseIds(Collection lids); + public Search setIndex (String name); + public Search setPretty (boolean pretty); + } + public interface Setting { // _settings + public Setting setIndex (String name); } - enum Method { DELETE, GET, HEAD, POST, PUT }; - public void setJsonEntity(String entity); } diff --git a/src/main/java/gov/nasa/pds/registry/common/RestClient.java b/src/main/java/gov/nasa/pds/registry/common/RestClient.java index 9d68687..a630a4e 100644 --- a/src/main/java/gov/nasa/pds/registry/common/RestClient.java +++ b/src/main/java/gov/nasa/pds/registry/common/RestClient.java @@ -4,8 +4,20 @@ import java.io.IOException; public interface RestClient extends Closeable { - public Request.Bulk createBulkRequest() throws IOException; - public Request createRequest(Request.Method method, String endpoint); - public Response performRequest(Request request) throws IOException; - public Response performRequest(Request.Bulk request) throws IOException; + public Request.Bulk createBulkRequest(); + public Request.Count createCountRequest(); + public Request.Get createGetRequest(); + public Request.Mapping createMappingRequest(); + public Request.MGet createMGetRequest(); + public Request.Search createSearchRequest(); + public Request.Setting createSettingRequest(); + public Response create (String indexName, String configAsJson) throws IOException,ResponseException; + public Response delete (String indexName) throws IOException,ResponseException; + public Response exists (String indexName) throws IOException,ResponseException; + public Response performRequest(Request.Bulk request) throws IOException,ResponseException; + public Response performRequest(Request.Count request) throws IOException,ResponseException; + public Response performRequest(Request.Get request) throws IOException,ResponseException; + public Response performRequest(Request.Mapping request) throws IOException,ResponseException; + public Response performRequest(Request.Search request) throws IOException,ResponseException; + public Response performRequest(Request.Setting request) throws IOException,ResponseException; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java index 799da09..ef5d078 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java @@ -1,7 +1,5 @@ package gov.nasa.pds.registry.common.connection; -import java.io.IOException; -import java.net.HttpURLConnection; import java.net.URL; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; @@ -14,11 +12,9 @@ public class Direct implements Cloneable, ConnectionFactory { final private boolean veryTrusting; - final private int timeout = 5000; final private AuthContent auth; final private HttpHost host; final private URL service; - private String api = null; private String index = null; public static Direct build (DirectType url, AuthContent auth) throws Exception { @@ -38,20 +34,9 @@ private Direct (URL service, AuthContent auth, boolean trustSelfSigned) { this.service = service; this.veryTrusting = trustSelfSigned; } - public HttpURLConnection createConnection() throws IOException { - String url = this.service.toString(); - if (this.index != null) url += "/" + this.index; - if (this.api != null) url += "/" + this.api; - HttpURLConnection con = (HttpURLConnection)new URL(url).openConnection(); - con.setConnectTimeout(this.timeout); - con.setReadTimeout(this.timeout); - con.setAllowUserInteraction(false); - con.setRequestProperty("Authorization", this.auth.getHeader()); - return con; - } @Override public ConnectionFactory clone() { - return new Direct(this.service, this.auth, this.veryTrusting).setAPI(this.api).setIndexName(this.index); + return new Direct(this.service, this.auth, this.veryTrusting).setIndexName(this.index); } @Override public RestClient createRestClient() throws Exception { @@ -70,11 +55,6 @@ public String getIndexName() { return this.index; } @Override - public ConnectionFactory setAPI(String api) { - this.api = api; - return this; - } - @Override public ConnectionFactory setIndexName(String idxName) { this.index = idxName; return this; @@ -84,7 +64,6 @@ public String toString() { String me = "Direct to " + this.service.getProtocol() + "://" + this.service.getHost(); if (0 <= this.service.getPort()) me += ":" + this.service.getPort(); me += " using index '" + String.valueOf(this.index) + "'"; - me += " to call API '" + String.valueOf(this.api) + "'"; return me; } @Override diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java index 026b64d..54f1a91 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java @@ -23,7 +23,6 @@ public final class MultiTenancy implements ConnectionFactory { final private AuthContent auth; final private HttpHost host; final private URL endpoint; - private String api = null; private String index = null; public static MultiTenancy build (CognitoType cog, AuthContent auth) throws IOException, InterruptedException { boolean expectedContent = true; @@ -94,7 +93,7 @@ private MultiTenancy (AuthContent auth, URL opensearchEndpoint) { } @Override public ConnectionFactory clone() { - return new MultiTenancy(this.auth, this.endpoint).setAPI(this.api).setIndexName(this.index); + return new MultiTenancy(this.auth, this.endpoint).setIndexName(this.index); } @Override public RestClient createRestClient() throws Exception { @@ -121,11 +120,6 @@ public boolean isTrustingSelfSigned() { return false; } @Override - public ConnectionFactory setAPI(String api) { - this.api = api; - return this; - } - @Override public ConnectionFactory setIndexName(String idxName) { this.index = idxName; return this; diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java index e384a47..3af91e4 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java @@ -35,17 +35,17 @@ public void close() throws IOException { this.httpClient.close(); } @Override - public Request createRequest(Method method, String endpoint) { + public Bulk createBulkRequest() throws IOException { // TODO Auto-generated method stub return null; } @Override - public Response performRequest(Request request) throws IOException { + public Request createRequest(Method method, String endpoint) { // TODO Auto-generated method stub return null; } @Override - public Bulk createBulkRequest() throws IOException { + public Response performRequest(Request request) throws IOException { // TODO Auto-generated method stub return null; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkImpl.java new file mode 100644 index 0000000..b1f81f5 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkImpl.java @@ -0,0 +1,40 @@ +package gov.nasa.pds.registry.common.connection.es; + + +import java.util.Collection; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.Request.Bulk; + +class BulkImpl implements Request.Bulk { + private String index = null; + private String refresh = null; + String json = ""; + //@Override + public void add(String statement) { + this.json += statement + "\n"; + } + @Override + public void add(String statement, String document) { + this.json += statement + "\n"; + this.json += document + "\n"; + } + @Override + public Request.Bulk buildUpdateStatus(Collection lidvids, String status) { + this.json = JsonHelper.buildUpdateStatusJson(lidvids, status); + return this; + } + @Override + public Bulk setIndex(String name) { + this.index = name; + return this; + } + @Override + public Bulk setRefresh(String type) { + this.refresh = type; + return this; + } + @Override + public String toString() { + return (this.index == null ? "" : "/" + this.index) + "/_bulk" + (this.refresh == null ? "" : "?refresh=" + this.refresh); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponse.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponse.java deleted file mode 100644 index 48c579b..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponse.java +++ /dev/null @@ -1,27 +0,0 @@ -package gov.nasa.pds.registry.common.connection.es; - -import org.apache.http.HttpEntity; -import org.apache.http.StatusLine; -import gov.nasa.pds.registry.common.Response; - -class BulkResponse implements Response { - final String lastLine; - BulkResponse(String lastLine) { - this.lastLine = lastLine; - } - @Override - public HttpEntity getEntity() { - return null; - } - @Override - public StatusLine getStatusLine() { - return null; - } - @Override - public void printWarnings() { - } - @Override - public String toString() { - return this.lastLine; - } -} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkWrapper.java deleted file mode 100644 index 579d8bd..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkWrapper.java +++ /dev/null @@ -1,25 +0,0 @@ -package gov.nasa.pds.registry.common.connection.es; - -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.net.HttpURLConnection; -import gov.nasa.pds.registry.common.Request; -import gov.nasa.pds.registry.common.connection.Direct; - -class BulkWrapper implements Request.Bulk { - final HttpURLConnection con; - final OutputStreamWriter writer; - BulkWrapper(Direct conFactory) throws IOException { - this.con = conFactory.createConnection(); - this.con.setDoInput(true); - this.con.setDoOutput(true); - this.con.setRequestMethod("POST"); - this.con.setRequestProperty("content-type", "application/x-ndjson; charset=utf-8"); - this.writer = new OutputStreamWriter(con.getOutputStream(), "UTF-8"); - } - @Override - public void add(String statement) throws IOException { - this.writer.write(statement); - this.writer.write("\n"); - } -} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/CountImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/CountImpl.java new file mode 100644 index 0000000..b8e3e3a --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/CountImpl.java @@ -0,0 +1,22 @@ +package gov.nasa.pds.registry.common.connection.es; + +import gov.nasa.pds.registry.common.Request.Count; + +class CountImpl implements Count { + private String index; + private String query; + @Override + public Count setIndex(String name) { + this.index = name; + return this; + } + @Override + public Count setQuery(String q) { + this.query = q; + return this; + } + @Override + public String toString() { + return "/" + this.index + "/_count?q=" + this.query; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/GetImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/GetImpl.java new file mode 100644 index 0000000..f715aa2 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/GetImpl.java @@ -0,0 +1,62 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import gov.nasa.pds.registry.common.Request.Get; +import gov.nasa.pds.registry.common.Request.MGet; + +class GetImpl implements MGet { + final private ArrayList excludes = new ArrayList(); + final private ArrayList includes = new ArrayList(); + private String id = null; + private String index; + String json = null; + @Override + public Get excludeField(String field) { + this.excludes.add(field); + return this; + } + @Override + public Get excludeFields(List fields) { + this.excludes.addAll(fields); + return this; + } + @Override + public Get includeField(String field) { + this.includes.add(field); + return this; + } + @Override + public Get includeFields(List fields) { + this.includes.addAll(fields); + return this; + } + @Override + public Get setId(String id) { + this.setId(id); + return this; + } + @Override + public MGet setIds(Collection ids) { + this.json = JsonHelper.buildIdList(ids); + return this; + } + @Override + public Get setIndex(String index) { + this.setIndex(index); + return this; + } + @Override + public String toString() { + String constraints = ""; + if (!this.excludes.isEmpty() || !this.includes.isEmpty()) { + constraints += "?_source="; + for (String field : this.includes) { + constraints += field + ","; + } + constraints = constraints.substring(0, constraints.length()-1); + } + return "/" + this.index + "/_doc" + (this.id == null ? "" : "/" + this.id) + constraints; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/JsonHelper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/JsonHelper.java new file mode 100644 index 0000000..57ed6b4 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/JsonHelper.java @@ -0,0 +1,363 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.Collection; +import com.google.gson.stream.JsonWriter; +import gov.nasa.pds.registry.common.meta.Metadata; +import gov.nasa.pds.registry.common.util.Tuple; + +/** + * Builds Elasticsearch JSON queries + * + * @author karpenko + */ +class JsonHelper { + /** + * Build update product archive status JSON request + * + * @param lidvids list of LIDVIDs to update + * @param status new status + * @return JSON + */ + static String buildUpdateStatusJson(Collection lidvids, String status) { + if (lidvids == null || lidvids.isEmpty()) + return null; + if (status == null || status.isEmpty()) + throw new IllegalArgumentException("Status could not be null or empty."); + + StringBuilder bld = new StringBuilder(); + String dataLine = + "{ \"doc\" : {\"" + Metadata.FLD_ARCHIVE_STATUS + "\" : \"" + status + "\"} }\n"; + + // Build NJSON (new-line delimited JSON) + for (String lidvid : lidvids) { + // Line 1: Elasticsearch document ID + bld.append("{ \"update\" : {\"_id\" : \"" + lidvid + "\" } }\n"); + // Line 2: Data + bld.append(dataLine); + } + + return bld.toString(); + } + + + /** + * Build aggregation query to select latest versions of lids + * + * @param lids list of LIDs + * @return JSON + */ + static String buildGetLatestLidVidsJson(Collection lids) { + if (lids == null || lids.isEmpty()) + return null; + + StringWriter strWriter = new StringWriter(); + try (JsonWriter jw = new JsonWriter(strWriter)) { + jw.beginObject(); + + jw.name("_source").value(false); + jw.name("size").value(0); + + // Query + jw.name("query"); + jw.beginObject(); + + jw.name("terms"); + jw.beginObject(); + + jw.name("lid"); + jw.beginArray(); + for (String lid : lids) { + jw.value(lid); + } + jw.endArray(); + + jw.endObject(); // terms + jw.endObject(); // query + + // Aggs + jw.name("aggs"); + jw.beginObject(); + + jw.name("lids"); + jw.beginObject(); + + jw.name("terms"); + jw.beginObject(); + jw.name("field").value("lid"); + jw.name("size").value(5000); + jw.endObject(); + + jw.name("aggs"); + jw.beginObject(); + jw.name("latest"); + jw.beginObject(); + jw.name("top_hits"); + jw.beginObject(); + + jw.name("sort"); + jw.beginArray(); + jw.beginObject(); + jw.name("vid"); + jw.beginObject(); + jw.name("order").value("desc"); + jw.endObject(); + jw.endObject(); + jw.endArray(); + + jw.name("_source").value(false); + jw.name("size").value(1); + + jw.endObject(); // top_hits + jw.endObject(); // latest + jw.endObject(); // aggs + + jw.endObject(); // lids + jw.endObject(); // aggs + + jw.endObject(); + + return strWriter.toString(); + } catch (IOException ioe) { + // not sure how we got here but fail in a large way + throw new RuntimeException("some weird internal JSON writing problem"); + } + } + + static String buildSearchIdsRequest(Collection ids, int pageSize, boolean isAlt) { + StringWriter strWriter = new StringWriter(); + try (JsonWriter writer = new JsonWriter(strWriter)) { + // Create ids query + writer.beginObject(); + + // Exclude source from response + JsonWriter src = writer.name("_source"); + if (isAlt) + src.value("alternate_ids"); + else + src.value(false); + writer.name("size").value(pageSize); + + writer.name("query"); + writer.beginObject(); + writer.name("ids"); + writer.beginObject(); + + writer.name("values"); + JsonWriter a = writer.beginArray(); + for (String id : ids) { + writer.value(id); + } + writer.endArray(); + + writer.endObject(); + writer.endObject(); + writer.endObject(); + + a.close(); + writer.close(); + } catch (IOException ioe) { + // cannot imagine ever getting here + throw new RuntimeException( + "some weird JSON serialization problem that should not be happening"); + } + return strWriter.toString(); + } + + static String buildListLddsRequest(String namespace) { + StringWriter strWriter = new StringWriter(); + try (JsonWriter jw = new JsonWriter(strWriter)) { + jw.beginObject(); + // Size (number of records to return) + jw.name("size").value(1000); + + // Start query + jw.name("query"); + jw.beginObject(); + jw.name("bool"); + jw.beginObject(); + + jw.name("must"); + jw.beginArray(); + appendMatch(jw, "class_ns", "registry"); + appendMatch(jw, "class_name", "LDD_Info"); + if (namespace != null) { + appendMatch(jw, "attr_ns", namespace); + } + jw.endArray(); + + jw.endObject(); + jw.endObject(); + // End query + + // Start source + jw.name("_source"); + jw.beginArray(); + jw.value("date").value("attr_name").value("attr_ns").value("im_version"); + jw.endArray(); + // End source + + jw.endObject(); + jw.close(); + } catch (IOException ioe) { + // cannot imagine ever getting here + throw new RuntimeException( + "some weird JSON serialization problem that should not be happening"); + } + + return strWriter.toString(); + } + + static String buildListFieldsRequest(String dataType) { + StringWriter strWriter = new StringWriter(); + try (JsonWriter jw = new JsonWriter(strWriter)) { + jw.beginObject(); + // Size (number of records to return) + jw.name("size").value(1000); + + // Start query + jw.name("query"); + jw.beginObject(); + jw.name("bool"); + jw.beginObject(); + + jw.name("must"); + jw.beginArray(); + appendMatch(jw, "es_data_type", dataType); + jw.endArray(); + + jw.endObject(); + jw.endObject(); + // End query + + // Start source + jw.name("_source"); + jw.beginArray(); + jw.value("es_field_name"); + jw.endArray(); + // End source + + jw.endObject(); + jw.close(); + } catch (IOException ioe) { + // cannot imagine ever getting here + throw new RuntimeException( + "some weird JSON serialization problem that should not be happening"); + } + + return strWriter.toString(); + } + + + /** + * Create get data dictionary (LDD) info request. + * + * @param namespace LDD namespace ID, such as 'pds', 'cart', etc. + * @return Elasticsearch query in JSON format + * @throws IOException an exception + */ + static String buildGetLddInfoRequest(String namespace) throws IOException { + StringWriter strWriter = new StringWriter(); + try (JsonWriter jw = new JsonWriter(strWriter)) { + jw.beginObject(); + // Size (number of records to return) + jw.name("size").value(1000); + + // Start query + jw.name("query"); + jw.beginObject(); + jw.name("bool"); + jw.beginObject(); + + jw.name("must"); + jw.beginArray(); + appendMatch(jw, "class_ns", "registry"); + appendMatch(jw, "class_name", "LDD_Info"); + appendMatch(jw, "attr_ns", namespace); + jw.endArray(); + + jw.endObject(); + jw.endObject(); + // End query + + // Start source + jw.name("_source"); + jw.beginArray(); + jw.value("date").value("attr_name"); + jw.endArray(); + // End source + + jw.endObject(); + jw.close(); + } catch (IOException ioe) { + // cannot imagine ever getting here + throw new RuntimeException( + "some weird JSON serialization problem that should not be happening"); + } + + return strWriter.toString(); + } + + + private static void appendMatch(JsonWriter jw, String field, String value) throws IOException { + jw.beginObject(); + jw.name("match"); + jw.beginObject(); + jw.name(field).value(value); + jw.endObject(); + jw.endObject(); + } + + static String buildIdList(Collection ids) { + StringWriter strWriter = new StringWriter(); + try (JsonWriter jw = new JsonWriter(strWriter)) { + jw.beginObject(); + jw.name("ids"); + + JsonWriter a = jw.beginArray(); + for (String id : ids) { + jw.value(id); + } + jw.endArray(); + + jw.endObject(); + a.close(); + jw.close(); + } catch (IOException ioe) { + // cannot imagine ever getting here + throw new RuntimeException( + "some weird JSON serialization problem that should not be happening"); + } + + return strWriter.toString(); + } + + static String buildUpdateSchemaRequest(Collection fields) { + StringWriter strWriter = new StringWriter(); + try (JsonWriter jw = new JsonWriter(strWriter)) { + jw.beginObject(); + + jw.name("properties"); + JsonWriter o = jw.beginObject(); + for (Tuple field : fields) { + jw.name(field.item1); + jw.beginObject(); + jw.name("type").value(field.item2); + jw.endObject(); + } + jw.endObject(); + + jw.endObject(); + o.close(); + jw.close(); + } catch (IOException ioe) { + // cannot imagine ever getting here + throw new RuntimeException( + "some weird JSON serialization problem that should not be happening"); + } + + return strWriter.toString(); + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingImpl.java new file mode 100644 index 0000000..6d9aae4 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingImpl.java @@ -0,0 +1,24 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.util.Collection; +import gov.nasa.pds.registry.common.Request.Mapping; +import gov.nasa.pds.registry.common.util.Tuple; + +class MappingImpl implements Mapping { + private String index; + String json = null; + @Override + public Mapping buildUpdateFieldSchema(Collection pairs) { + this.json = JsonHelper.buildUpdateSchemaRequest(pairs); + return this; + } + @Override + public Mapping setIndex(String name) { + this.index = name; + return this; + } + @Override + public String toString() { + return "/" + this.index + (this.json == null ? "/_mappings" : "/_mapping"); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestWrapper.java deleted file mode 100644 index 38411bb..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestWrapper.java +++ /dev/null @@ -1,27 +0,0 @@ -package gov.nasa.pds.registry.common.connection.es; - -import gov.nasa.pds.registry.common.Request; - -final class RequestWrapper implements Request { - final org.elasticsearch.client.Request real_request; - RequestWrapper (org.elasticsearch.client.Request real_request) { - this.real_request = real_request; - } - RequestWrapper (Request.Method method, String endpoint) { - this(new org.elasticsearch.client.Request(RequestWrapper.methodToString(method), endpoint)); - } - @Override - public void setJsonEntity(String entity) { - this.real_request.setJsonEntity(entity); - } - private static String methodToString (Request.Method method) { - switch (method) { - case DELETE: return "DELETE"; - case GET: return "GET"; - case HEAD: return "HEAD"; - case POST: return "POST"; - case PUT: return "PUT"; - } - throw new RuntimeException ("Request.Method not fully enumerated"); - } -} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java index e690948..5636100 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java @@ -1,14 +1,19 @@ package gov.nasa.pds.registry.common.connection.es; import java.io.IOException; +import org.elasticsearch.client.Request; import org.elasticsearch.client.RestClientBuilder; -import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.Request.Bulk; -import gov.nasa.pds.registry.common.Request.Method; +import gov.nasa.pds.registry.common.Request.Count; +import gov.nasa.pds.registry.common.Request.Get; +import gov.nasa.pds.registry.common.Request.Mapping; +import gov.nasa.pds.registry.common.Request.MGet; +import gov.nasa.pds.registry.common.Request.Search; +import gov.nasa.pds.registry.common.Request.Setting; import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.ResponseException; import gov.nasa.pds.registry.common.RestClient; -import gov.nasa.pds.registry.common.connection.Direct; -import gov.nasa.pds.registry.common.es.dao.DaoUtils; /** @@ -18,13 +23,12 @@ */ public class RestClientWrapper implements RestClient { - final Direct conFactory; final org.elasticsearch.client.RestClient real_client; /** * Constructor. * @throws Exception an exception */ - public RestClientWrapper(Direct conFact) throws Exception + public RestClientWrapper(ConnectionFactory conFact) throws Exception { ClientConfigCB clientCB = new ClientConfigCB(); RequestConfigCB reqCB = new RequestConfigCB(); @@ -33,33 +37,84 @@ public RestClientWrapper(Direct conFact) throws Exception clientCB.setTrustSelfSignedCert(conFact.isTrustingSelfSigned()); bld.setHttpClientConfigCallback(clientCB); bld.setRequestConfigCallback(reqCB); - this.conFactory = conFact; this.real_client = bld.build(); } + private Response performRequest(String endpoint, String json, String method) throws IOException,ResponseException { + try { + Request request = new Request(method, endpoint); + if (json != null) request.setJsonEntity(json); + return new ResponseWrapper(this.real_client.performRequest(request)); + } catch (org.elasticsearch.client.ResponseException e) { + throw new ResponseExceptionWrapper(e); + } + } @Override public void close() throws IOException { this.real_client.close(); } @Override - public Request createRequest(Method method, String endpoint) { - return new RequestWrapper(method, endpoint); + public Bulk createBulkRequest() { + return new BulkImpl(); } @Override - public Response performRequest(Request request) throws IOException { - try { - return new ResponseWrapper(this.real_client.performRequest(((RequestWrapper)request).real_request)); - } catch (org.elasticsearch.client.ResponseException e) { - throw new ResponseExceptionWrapper(e); - } + public Count createCountRequest() { + return new CountImpl(); + } + @Override + public Get createGetRequest() { + return new GetImpl(); + } + @Override + public Mapping createMappingRequest() { + return new MappingImpl(); + } + @Override + public MGet createMGetRequest() { + return new GetImpl(); + } + @Override + public Search createSearchRequest() { + return new SearchImpl(); } @Override - public Bulk createBulkRequest() throws IOException { - return new BulkWrapper(this.conFactory); + public Setting createSettingRequest() { + return new SettingImpl(); } @Override - public Response performRequest(Bulk request) throws IOException { - ((BulkWrapper)request).writer.close(); - DaoUtils.getLastLine(((BulkWrapper)request).con.getInputStream()); - return null; - } + public Response create (String indexName, String configAsJSON) throws IOException,ResponseException { + return this.performRequest("/" + indexName, configAsJSON, "PUT"); + } + @Override + public Response delete (String indexName) throws IOException,ResponseException { + return this.performRequest(indexName, null, "DELETE"); + } + @Override + public Response exists (String indexName) throws IOException,ResponseException { + return this.performRequest ("/" + indexName, null, "HEAD"); + } + @Override + public Response performRequest(Bulk request) throws IOException,ResponseException { + return this.performRequest(request.toString(), ((BulkImpl)request).json, "POST"); + } + @Override + public Response performRequest(Count request) throws IOException,ResponseException { + return this.performRequest(request.toString(), null, "GET"); + } + @Override + public Response performRequest(Get request) throws IOException,ResponseException { + return this.performRequest(request.toString(), ((GetImpl)request).json, "GET"); + } + @Override + public Response performRequest(Mapping request) throws IOException,ResponseException { + String json = ((MappingImpl)request).json; + return this.performRequest(request.toString(), json, json == null ? "GET" : "PUT"); + } + @Override + public Response performRequest(Search request) throws IOException,ResponseException { + return this.performRequest(request.toString(), ((SearchImpl)request).json, "GET"); + } + @Override + public Response performRequest(Setting request) throws IOException,ResponseException { + return this.performRequest(request.toString(), null, "GET"); + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java new file mode 100644 index 0000000..83f982e --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java @@ -0,0 +1,49 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.util.Collection; +import gov.nasa.pds.registry.common.Request.Search; + +class SearchImpl implements Search { + private boolean pretty = false; + private String index; + String json = null; + @Override + public Search buildAlternativeIds(Collection lids) { + JsonHelper.buildSearchIdsRequest(lids, lids.size(), true); + return this; + } + @Override + public Search buildLatestLidVids(Collection lids) { + this.json = JsonHelper.buildGetLatestLidVidsJson(lids); + return this; + } + @Override + public Search buildListFields(String dataType) { + this.json = JsonHelper.buildListFieldsRequest(dataType); + return this; + } + @Override + public Search buildListLdds(String namespace) { + this.json = JsonHelper.buildListLddsRequest(namespace); + return this; + } + @Override + public Search buildTheseIds(Collection lids) { + JsonHelper.buildSearchIdsRequest(lids, lids.size(), true); + return this; + } + @Override + public Search setIndex(String name) { + this.index = name; + return this; + } + @Override + public String toString() { + return "/" + this.index + "/_search" + (this.pretty ? "?pretty" : ""); + } + @Override + public Search setPretty(boolean pretty) { + this.pretty = pretty; + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SettingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SettingImpl.java new file mode 100644 index 0000000..610c748 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SettingImpl.java @@ -0,0 +1,16 @@ +package gov.nasa.pds.registry.common.connection.es; + +import gov.nasa.pds.registry.common.Request.Setting; + +class SettingImpl implements Setting { + private String index; + @Override + public Setting setIndex(String name) { + this.index = name; + return this; + } + @Override + public String toString() { + return "/" + this.index + "/_settings"; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/BaseRequestBuilder.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/BaseRequestBuilder.java deleted file mode 100644 index b680873..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/BaseRequestBuilder.java +++ /dev/null @@ -1,68 +0,0 @@ -package gov.nasa.pds.registry.common.es.dao; - -import java.io.IOException; -import java.io.StringWriter; -import java.io.Writer; -import java.util.Collection; - -import com.google.gson.stream.JsonWriter; - -/** - * Methods to build JSON requests for Elasticsearch APIs. - * @author karpenko - */ -public class BaseRequestBuilder -{ - protected boolean pretty; - - /** - * Constructor - * @param pretty Format JSON for humans to read. - */ - public BaseRequestBuilder(boolean pretty) - { - this.pretty = pretty; - } - - - protected JsonWriter createJsonWriter(Writer writer) - { - JsonWriter jw = new JsonWriter(writer); - if (pretty) - { - jw.setIndent(" "); - } - - return jw; - } - - - /** - * Create multi get (_mget) request. - * @param ids list of IDs - * @return JSON - * @throws IOException an exception - */ - public String createMgetRequest(Collection ids) throws IOException - { - StringWriter wr = new StringWriter(); - JsonWriter jw = createJsonWriter(wr); - - jw.beginObject(); - jw.name("ids"); - - jw.beginArray(); - for(String id: ids) - { - jw.value(id); - } - jw.endArray(); - - jw.endObject(); - jw.close(); - - return wr.toString(); - } - - -} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java index 8e570e7..83093fc 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java @@ -53,7 +53,7 @@ public class DataLoader public DataLoader(ConnectionFactory conFactory) throws Exception { log = LogManager.getLogger(this.getClass()); - this.conFactory = conFactory.setAPI("_bulk?refresh=wait_for"); + this.conFactory = conFactory; } @@ -231,10 +231,11 @@ public int loadBatch(List data, Set errorLidvids, int retries) t try { - bulk = this.conFactory.createRestClient().createBulkRequest(); - for (String statement : data) { - bulk.add(statement); + bulk = this.conFactory.createRestClient().createBulkRequest().setRefresh("wait_for"); + for (int index = 0 ; index < data.size() ; index++) { + bulk.add(data.get(index), data.get(++index)); } + // FIXME: response has changed so need to debug this Response response = this.conFactory.createRestClient().performRequest(bulk); // Read Elasticsearch response. String respJson = response.toString(); diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java index a3abd73..acfbac9 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java @@ -5,10 +5,6 @@ import java.net.URLEncoder; import java.util.Collection; import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; import gov.nasa.pds.registry.common.Request; @@ -24,9 +20,7 @@ * @author karpenko */ public class ProductDao -{ - private Logger log; - +{ private RestClient client; private String indexName; @@ -37,8 +31,6 @@ public class ProductDao */ public ProductDao(RestClient client, String indexName) { - log = LogManager.getLogger(this.getClass()); - this.client = client; this.indexName = indexName; } @@ -54,8 +46,10 @@ public String getProductClass(String lidvid) throws Exception { if(lidvid == null) return null; - String reqUrl = "/" + indexName + "/_doc/" + lidvid + "?_source=product_class"; - Request req = client.createRequest(Request.Method.GET, reqUrl); + Request.Get req = client.createGetRequest() + .includeField("product_class") + .setId(lidvid) + .setIndex(this.indexName); Response resp = null; try @@ -140,8 +134,9 @@ else if(type == 'S') query = URLEncoder.encode(query, "UTF-8"); // Request URL - String reqUrl = "/" + indexName + "-refs/_count?q=" + query; - Request req = client.createRequest(Request.Method.GET, reqUrl); + Request.Count req = client.createCountRequest() + .setIndex(this.indexName + "-refs") + .setQuery(query); Response resp = null; try @@ -210,8 +205,10 @@ public List getRefs(String collectionLidVid, char type, int page) throws if(collectionLidVid == null) return null; String docId = collectionLidVid + "::" + type + page; - String reqUrl = "/" + indexName + "-refs/_doc/" + docId + "?_source=product_lidvid"; - Request req = client.createRequest(Request.Method.GET, reqUrl); + Request.Get req = client.createGetRequest() + .includeField("product_lidvid") + .setId(docId) + .setIndex(this.indexName + "-refs"); Response resp = null; try @@ -276,12 +273,9 @@ public void updateArchiveStatus(Collection lidvids, String status) throw { if(lidvids == null || status == null) return; - String json = ProductRequestBuilder.buildUpdateStatusJson(lidvids, status); - log.debug("Request:\n" + json); - - String reqUrl = "/" + indexName + "/_bulk"; //?refresh=wait_for"; - Request req = client.createRequest(Request.Method.POST, reqUrl); - req.setJsonEntity(json); + Request.Bulk req = client.createBulkRequest() + .buildUpdateStatus(lidvids, status) + .setIndex(this.indexName); Response resp = client.performRequest(req); @@ -360,8 +354,11 @@ public LidvidSet getCollectionIds(String bundleLidvid) throws Exception { if(bundleLidvid == null) return null; - String reqUrl = "/" + indexName + "/_doc/" + bundleLidvid + "?_source=ref_lidvid_collection,ref_lid_collection"; - Request req = client.createRequest(Request.Method.GET, reqUrl); + Request.Get req = client.createGetRequest() + .includeField("ref_lidvid_collection") + .includeField("ref_lid_collection") + .setId(bundleLidvid) + .setIndex(this.indexName); Response resp = null; try @@ -470,15 +467,9 @@ public List getLatestLidVids(Collection lids) throws Exception { if(lids == null || lids.isEmpty()) return null; - String json = ProductRequestBuilder.buildGetLatestLidVidsJson(lids); - log.debug("getGetLatestLidVids() request: " + json); - - if(json == null) return null; - - String reqUrl = "/" + indexName + "/_search/"; - Request req = client.createRequest(Request.Method.GET, reqUrl); - req.setJsonEntity(json); - + Request.Search req = client.createSearchRequest() + .setIndex(this.indexName) + .buildLatestLidVids(lids); Response resp = null; try diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductRequestBuilder.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductRequestBuilder.java deleted file mode 100644 index 3522170..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductRequestBuilder.java +++ /dev/null @@ -1,134 +0,0 @@ -package gov.nasa.pds.registry.common.es.dao; - -import java.io.IOException; -import java.io.StringWriter; -import java.util.Collection; - -import com.google.gson.stream.JsonWriter; - -import gov.nasa.pds.registry.common.meta.Metadata; -import gov.nasa.pds.registry.common.util.CloseUtils; - -/** - * Builds Elasticsearch JSON queries - * @author karpenko - */ -public class ProductRequestBuilder -{ - /** - * Build update product archive status JSON request - * @param lidvids list of LIDVIDs to update - * @param status new status - * @return JSON - */ - public static String buildUpdateStatusJson(Collection lidvids, String status) - { - if(lidvids == null || lidvids.isEmpty()) return null; - if(status == null || status.isEmpty()) throw new IllegalArgumentException("Status could not be null or empty."); - - StringBuilder bld = new StringBuilder(); - String dataLine = "{ \"doc\" : {\"" + Metadata.FLD_ARCHIVE_STATUS + "\" : \"" + status + "\"} }\n"; - - // Build NJSON (new-line delimited JSON) - for(String lidvid: lidvids) - { - // Line 1: Elasticsearch document ID - bld.append("{ \"update\" : {\"_id\" : \"" + lidvid + "\" } }\n"); - // Line 2: Data - bld.append(dataLine); - } - - return bld.toString(); - } - - - /** - * Build aggregation query to select latest versions of lids - * @param lids list of LIDs - * @return JSON - */ - public static String buildGetLatestLidVidsJson(Collection lids) throws IOException - { - if(lids == null || lids.isEmpty()) return null; - - JsonWriter jw = null; - - try - { - StringWriter strWriter = new StringWriter(); - jw = new JsonWriter(strWriter); - - jw.beginObject(); - - jw.name("_source").value(false); - jw.name("size").value(0); - - // Query - jw.name("query"); - jw.beginObject(); - - jw.name("terms"); - jw.beginObject(); - - jw.name("lid"); - jw.beginArray(); - for(String lid: lids) - { - jw.value(lid); - } - jw.endArray(); - - jw.endObject(); // terms - jw.endObject(); // query - - // Aggs - jw.name("aggs"); - jw.beginObject(); - - jw.name("lids"); - jw.beginObject(); - - jw.name("terms"); - jw.beginObject(); - jw.name("field").value("lid"); - jw.name("size").value(5000); - jw.endObject(); - - jw.name("aggs"); - jw.beginObject(); - jw.name("latest"); - jw.beginObject(); - jw.name("top_hits"); - jw.beginObject(); - - jw.name("sort"); - jw.beginArray(); - jw.beginObject(); - jw.name("vid"); - jw.beginObject(); - jw.name("order").value("desc"); - jw.endObject(); - jw.endObject(); - jw.endArray(); - - jw.name("_source").value(false); - jw.name("size").value(1); - - jw.endObject(); // top_hits - jw.endObject(); // latest - jw.endObject(); // aggs - - jw.endObject(); // lids - jw.endObject(); // aggs - - jw.endObject(); - - return strWriter.toString(); - } - finally - { - CloseUtils.close(jw); - } - } - -} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DDRequestBuilder.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DDRequestBuilder.java deleted file mode 100644 index 4484e95..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DDRequestBuilder.java +++ /dev/null @@ -1,178 +0,0 @@ -package gov.nasa.pds.registry.common.es.dao.dd; - -import java.io.IOException; -import java.io.StringWriter; - -import com.google.gson.stream.JsonWriter; - -import gov.nasa.pds.registry.common.es.dao.BaseRequestBuilder; - - -/** - * Methods to build JSON requests for Elasticsearch APIs. - * @author karpenko - */ -public class DDRequestBuilder extends BaseRequestBuilder -{ - /** - * Constructor - * @param pretty Format JSON for humans to read. - */ - public DDRequestBuilder(boolean pretty) - { - super(pretty); - } - - /** - * Constructor - */ - public DDRequestBuilder() - { - this(false); - } - - - /** - * Create get data dictionary (LDD) info request. - * @param namespace LDD namespace ID, such as 'pds', 'cart', etc. - * @return Elasticsearch query in JSON format - * @throws IOException an exception - */ - public String createListLddsRequest(String namespace) throws IOException - { - StringWriter wr = new StringWriter(); - JsonWriter jw = createJsonWriter(wr); - - jw.beginObject(); - // Size (number of records to return) - jw.name("size").value(1000); - - // Start query - jw.name("query"); - jw.beginObject(); - jw.name("bool"); - jw.beginObject(); - - jw.name("must"); - jw.beginArray(); - appendMatch(jw, "class_ns", "registry"); - appendMatch(jw, "class_name", "LDD_Info"); - if(namespace != null) - { - appendMatch(jw, "attr_ns", namespace); - } - jw.endArray(); - - jw.endObject(); - jw.endObject(); - // End query - - // Start source - jw.name("_source"); - jw.beginArray(); - jw.value("date").value("attr_name").value("attr_ns").value("im_version"); - jw.endArray(); - // End source - - jw.endObject(); - jw.close(); - - return wr.toString(); - } - - - public String createListFieldsRequest(String dataType) throws IOException - { - StringWriter wr = new StringWriter(); - JsonWriter jw = createJsonWriter(wr); - - jw.beginObject(); - // Size (number of records to return) - jw.name("size").value(1000); - - // Start query - jw.name("query"); - jw.beginObject(); - jw.name("bool"); - jw.beginObject(); - - jw.name("must"); - jw.beginArray(); - appendMatch(jw, "es_data_type", dataType); - jw.endArray(); - - jw.endObject(); - jw.endObject(); - // End query - - // Start source - jw.name("_source"); - jw.beginArray(); - jw.value("es_field_name"); - jw.endArray(); - // End source - - jw.endObject(); - jw.close(); - - return wr.toString(); - } - - - /** - * Create get data dictionary (LDD) info request. - * @param namespace LDD namespace ID, such as 'pds', 'cart', etc. - * @return Elasticsearch query in JSON format - * @throws IOException an exception - */ - public String createGetLddInfoRequest(String namespace) throws IOException - { - StringWriter wr = new StringWriter(); - JsonWriter jw = createJsonWriter(wr); - - jw.beginObject(); - // Size (number of records to return) - jw.name("size").value(1000); - - // Start query - jw.name("query"); - jw.beginObject(); - jw.name("bool"); - jw.beginObject(); - - jw.name("must"); - jw.beginArray(); - appendMatch(jw, "class_ns", "registry"); - appendMatch(jw, "class_name", "LDD_Info"); - appendMatch(jw, "attr_ns", namespace); - jw.endArray(); - - jw.endObject(); - jw.endObject(); - // End query - - // Start source - jw.name("_source"); - jw.beginArray(); - jw.value("date").value("attr_name"); - jw.endArray(); - // End source - - jw.endObject(); - jw.close(); - - return wr.toString(); - } - - - private static void appendMatch(JsonWriter jw, String field, String value) throws IOException - { - jw.beginObject(); - jw.name("match"); - jw.beginObject(); - jw.name(field).value(value); - jw.endObject(); - jw.endObject(); - } - -} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java index f563728..1364571 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java @@ -13,7 +13,6 @@ import gov.nasa.pds.registry.common.Request; import gov.nasa.pds.registry.common.Response; import gov.nasa.pds.registry.common.RestClient; -import gov.nasa.pds.registry.common.es.dao.schema.SchemaRequestBuilder; import gov.nasa.pds.registry.common.util.SearchResponseParser; import gov.nasa.pds.registry.common.util.Tuple; @@ -84,11 +83,9 @@ public void onRecord(String id, Object rec) throws Exception */ public LddVersions getLddInfo(String namespace) throws Exception { - DDRequestBuilder bld = new DDRequestBuilder(); - String json = bld.createListLddsRequest(namespace); - - Request req = client.createRequest(Request.Method.GET, "/" + indexName + "-dd/_search"); - req.setJsonEntity(json); + Request.Search req = client.createSearchRequest() + .buildListLdds(namespace) + .setIndex(indexName + "-dd"); Response resp = client.performRequest(req); GetLddInfoRespParser parser = new GetLddInfoRespParser(); @@ -150,11 +147,9 @@ public void onRecord(String id, Object rec) throws Exception */ public List listLdds(String namespace) throws Exception { - DDRequestBuilder bld = new DDRequestBuilder(); - String json = bld.createListLddsRequest(namespace); - - Request req = client.createRequest(Request.Method.GET, "/" + indexName + "-dd/_search"); - req.setJsonEntity(json); + Request.Search req = client.createSearchRequest() + .buildListLdds(namespace) + .setIndex(this.indexName + "-dd"); Response resp = client.performRequest(req); ListLddsParser parser = new ListLddsParser(); @@ -199,11 +194,9 @@ public void onRecord(String id, Object rec) throws Exception */ public Set getFieldNamesByEsType(String esType) throws Exception { - DDRequestBuilder bld = new DDRequestBuilder(); - String json = bld.createListFieldsRequest(esType); - - Request req = client.createRequest(Request.Method.GET, "/" + indexName + "-dd/_search"); - req.setJsonEntity(json); + Request.Search req = client.createSearchRequest() + .buildListFields(esType) + .setIndex(this.indexName + "-dd"); Response resp = client.performRequest(req); ListFieldsParser parser = new ListFieldsParser(); @@ -229,12 +222,10 @@ public List getDataTypes(Collection ids, boolean stringForMissing List dtInfo = new ArrayList(); // Create request - Request req = client.createRequest(Request.Method.GET, "/" + indexName + "-dd/_mget?_source=es_data_type"); - - // Create request body - SchemaRequestBuilder bld = new SchemaRequestBuilder(); - String json = bld.createMgetRequest(ids); - req.setJsonEntity(json); + Request.Get req = client.createMGetRequest() + .setIds(ids) + .includeField("es_data_type") + .setIndex(this.indexName + "-dd"); // Call ES Response resp = client.performRequest(req); diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java index be280a5..e3ab18d 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java @@ -39,7 +39,7 @@ public SchemaDao(RestClient client, String indexName) */ public Set getFieldNames() throws Exception { - Request req = client.createRequest(Request.Method.GET, "/" + indexName + "/_mappings"); + Request.Mapping req = client.createMappingRequest().setIndex(indexName); Response resp = client.performRequest(req); MappingsParser parser = new MappingsParser(indexName); @@ -56,11 +56,9 @@ public void updateSchema(List fields) throws Exception { if(fields == null || fields.isEmpty()) return; - SchemaRequestBuilder bld = new SchemaRequestBuilder(); - String json = bld.createUpdateSchemaRequest(fields); - - Request req = client.createRequest(Request.Method.PUT, "/" + indexName + "/_mapping"); - req.setJsonEntity(json); + Request.Mapping req = client.createMappingRequest() + .buildUpdateFieldSchema(fields) + .setIndex(this.indexName); client.performRequest(req); } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaRequestBuilder.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaRequestBuilder.java deleted file mode 100644 index d66aeb4..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaRequestBuilder.java +++ /dev/null @@ -1,67 +0,0 @@ -package gov.nasa.pds.registry.common.es.dao.schema; - -import java.io.IOException; -import java.io.StringWriter; -import java.util.List; - -import com.google.gson.stream.JsonWriter; - -import gov.nasa.pds.registry.common.es.dao.BaseRequestBuilder; -import gov.nasa.pds.registry.common.util.Tuple; - - -/** - * Methods to build JSON requests for Elasticsearch APIs. - * @author karpenko - */ -public class SchemaRequestBuilder extends BaseRequestBuilder -{ - /** - * Constructor - * @param pretty Format JSON for humans to read. - */ - public SchemaRequestBuilder(boolean pretty) - { - super(pretty); - } - - /** - * Constructor - */ - public SchemaRequestBuilder() - { - this(false); - } - - - /** - * Create update Elasticsearch schema request - * @param fields A list of fields to add. Each field tuple has a name and a data type. - * @return Elasticsearch query in JSON format - * @throws IOException an exception - */ - public String createUpdateSchemaRequest(List fields) throws IOException - { - StringWriter wr = new StringWriter(); - JsonWriter jw = createJsonWriter(wr); - - jw.beginObject(); - - jw.name("properties"); - jw.beginObject(); - for(Tuple field: fields) - { - jw.name(field.item1); - jw.beginObject(); - jw.name("type").value(field.item2); - jw.endObject(); - } - jw.endObject(); - - jw.endObject(); - jw.close(); - - return wr.toString(); - } - -} diff --git a/src/main/java/gov/nasa/pds/registry/common/util/Tuple.java b/src/main/java/gov/nasa/pds/registry/common/util/Tuple.java index e7ea681..027a0ad 100644 --- a/src/main/java/gov/nasa/pds/registry/common/util/Tuple.java +++ b/src/main/java/gov/nasa/pds/registry/common/util/Tuple.java @@ -9,8 +9,8 @@ public class Tuple { public String item1; public String item2; - - + public Tuple() { + } /** * Constructor * @param item1 value 1 From 36d533b022dadb129051a7cfdba53bde8791f7c8 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Sun, 31 Mar 2024 16:45:27 -0700 Subject: [PATCH 20/41] made it part way through serverless but still WIP --- .../gov/nasa/pds/registry/common/Request.java | 4 +- .../common/connection/aws/BulkImpl.java | 98 +++++++++++++++++++ .../common/connection/aws/CountImpl.java | 18 ++++ .../common/connection/aws/GetImpl.java | 39 ++++++++ .../common/connection/aws/MGetImpl.java | 47 +++++++++ .../common/connection/aws/MappingImpl.java | 21 ++++ .../common/connection/aws/RequestWrapper.java | 13 --- .../connection/aws/RestClientWrapper.java | 79 +++++++++++++-- .../common/connection/aws/SearchImpl.java | 39 ++++++++ .../common/connection/aws/SettingImpl.java | 10 ++ .../common/connection/es/BulkImpl.java | 14 ++- .../registry/common/es/dao/DataLoader.java | 2 +- 12 files changed, 359 insertions(+), 25 deletions(-) create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/CountImpl.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/GetImpl.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetImpl.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java delete mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/RequestWrapper.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java diff --git a/src/main/java/gov/nasa/pds/registry/common/Request.java b/src/main/java/gov/nasa/pds/registry/common/Request.java index 5a55652..26a10d3 100644 --- a/src/main/java/gov/nasa/pds/registry/common/Request.java +++ b/src/main/java/gov/nasa/pds/registry/common/Request.java @@ -6,11 +6,11 @@ public interface Request { public interface Bulk { // _bulk - //public void add (String statement); // delete + enum Refresh { False, True, WaitFor }; public void add (String statement, String document); // create, index, update public Bulk buildUpdateStatus(Collection lidvids, String status); public Bulk setIndex(String name); - public Bulk setRefresh(String type); + public Bulk setRefresh(Refresh type); } public interface Count { // _count public Count setIndex (String name); diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java new file mode 100644 index 0000000..803f9c9 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java @@ -0,0 +1,98 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.client.opensearch.core.bulk.CreateOperation; +import org.opensearch.client.opensearch.core.bulk.IndexOperation; +import org.opensearch.client.opensearch.core.bulk.UpdateOperation; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import gov.nasa.pds.registry.common.Request.Bulk; +import gov.nasa.pds.registry.common.meta.Metadata; + +class BulkImpl implements Bulk { + final BulkRequest.Builder craftsman = new BulkRequest.Builder(); + @Override + public void add(String statement, String document) { + BulkOperation.Builder journeyman = new BulkOperation.Builder(); + Gson gson = new Gson(); + Map> cmd; + Map params; + Object doc = gson.fromJson(document, Object.class); + Type contentType = new TypeToken>>(){}.getType(); + cmd = gson.fromJson(statement, contentType); + + if (cmd.containsKey("create")) { + CreateOperation.Builder apprentice = new CreateOperation.Builder().document(doc); + params = cmd.get("create"); + if (params.containsKey("_id")) { + apprentice.id(params.get("_id")); + } + if (params.containsKey("_index")) { + apprentice.index(params.get("_index")); + } + journeyman.create(apprentice.build()); + } else if (cmd.containsKey("index")) { + IndexOperation.Builder apprentice = new IndexOperation.Builder().document(doc); + params = cmd.get("index"); + if (params.containsKey("_id")) { + apprentice.id(params.get("_id")); + } + if (params.containsKey("_index")) { + apprentice.index(params.get("_index")); + } + journeyman.index(apprentice.build()); + } else if (cmd.containsKey("update")) { + doc = ((Map)doc).get("doc"); + UpdateOperation.Builder apprentice = new UpdateOperation.Builder().document(doc); + params = cmd.get("update"); + if (params.containsKey("_id")) { + apprentice.id(params.get("_id")); + } + if (params.containsKey("_index")) { + apprentice.index(params.get("_index")); + } + journeyman.update(apprentice.build()); + } else { + throw new RuntimeException("Received a statement that did not contain one of the expected keys: create, index, or update."); + } + this.craftsman.operations(journeyman.build()); + } + @Override + public Bulk buildUpdateStatus(Collection lidvids, String status) { + ArrayList updates = new ArrayList(); + Map doc; + for (String lidvid : lidvids) { + doc = new HashMap(); + doc.put(Metadata.FLD_ARCHIVE_STATUS, status); + updates.add(new BulkOperation.Builder().update(new UpdateOperation.Builder>().document(doc).id(lidvid).build()).build()); + } + this.craftsman.operations(updates); + return this; + } + @Override + public Bulk setIndex(String name) { + this.craftsman.index(name); + return this; + } + @Override + public Bulk setRefresh(Refresh type) { + switch (type) { + case False: + this.craftsman.refresh(org.opensearch.client.opensearch._types.Refresh.False); + break; + case True: + this.craftsman.refresh(org.opensearch.client.opensearch._types.Refresh.True); + break; + case WaitFor: + this.craftsman.refresh(org.opensearch.client.opensearch._types.Refresh.WaitFor); + break; + } + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/CountImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/CountImpl.java new file mode 100644 index 0000000..9e3f413 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/CountImpl.java @@ -0,0 +1,18 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.opensearch.client.opensearch.core.CountRequest; +import gov.nasa.pds.registry.common.Request.Count; + +class CountImpl implements Count { + final CountRequest.Builder craftsman = new CountRequest.Builder(); + @Override + public Count setIndex(String name) { + this.craftsman.index(name); + return this; + } + @Override + public Count setQuery(String q) { + this.craftsman.q(q); + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetImpl.java new file mode 100644 index 0000000..ebf58c0 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetImpl.java @@ -0,0 +1,39 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.List; +import org.opensearch.client.opensearch.core.GetRequest; +import gov.nasa.pds.registry.common.Request.Get; + +class GetImpl implements Get { + final GetRequest.Builder craftsman = new GetRequest.Builder(); + @Override + public Get excludeField(String field) { + this.craftsman.sourceExcludes(field); + return this; + } + @Override + public Get excludeFields(List fields) { + this.craftsman.sourceExcludes(fields); + return this; + } + @Override + public Get includeField(String field) { + this.craftsman.sourceIncludes(field); + return this; + } + @Override + public Get includeFields(List fields) { + this.craftsman.sourceIncludes(fields); + return this; + } + @Override + public Get setId(String id) { + this.craftsman.id(id); + return this; + } + @Override + public Get setIndex(String index) { + this.craftsman.index(index); + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetImpl.java new file mode 100644 index 0000000..95329b1 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetImpl.java @@ -0,0 +1,47 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import org.opensearch.client.opensearch.core.MgetRequest; +import gov.nasa.pds.registry.common.Request.Get; +import gov.nasa.pds.registry.common.Request.MGet; + +class MGetImpl implements MGet { + final MgetRequest.Builder craftsman = new MgetRequest.Builder(); + @Override + public Get excludeField(String field) { + this.craftsman.sourceExcludes(field); + return this; + } + @Override + public Get excludeFields(List fields) { + this.craftsman.sourceExcludes(fields); + return this; + } + @Override + public Get includeField(String field) { + this.craftsman.sourceIncludes(field); + return this; + } + @Override + public Get includeFields(List fields) { + this.craftsman.sourceIncludes(fields); + return this; + } + @Override + public Get setId(String id) { + this.craftsman.ids(id); + return this; + } + @Override + public Get setIndex(String index) { + this.craftsman.index(index); + return this; + } + @Override + public MGet setIds(Collection ids) { + this.craftsman.ids(new ArrayList(ids)); + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java new file mode 100644 index 0000000..fe96368 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java @@ -0,0 +1,21 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.Collection; +import gov.nasa.pds.registry.common.Request.Mapping; +import gov.nasa.pds.registry.common.util.Tuple; + +public class MappingImpl implements Mapping { + + @Override + public Mapping buildUpdateFieldSchema(Collection pairs) { + // TODO Auto-generated method stub + return null; + } + + @Override + public Mapping setIndex(String name) { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RequestWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RequestWrapper.java deleted file mode 100644 index 5c5e4d7..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RequestWrapper.java +++ /dev/null @@ -1,13 +0,0 @@ -package gov.nasa.pds.registry.common.connection.aws; - -import gov.nasa.pds.registry.common.Request; - -class RequestWrapper implements Request { - - @Override - public void setJsonEntity(String entity) { - // TODO Auto-generated method stub - - } - -} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java index 3af91e4..07f2a0b 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java @@ -5,13 +5,18 @@ import org.opensearch.client.transport.aws.AwsSdk2Transport; import org.opensearch.client.transport.aws.AwsSdk2TransportOptions; import gov.nasa.pds.registry.common.ConnectionFactory; -import gov.nasa.pds.registry.common.Request; import gov.nasa.pds.registry.common.Request.Bulk; -import gov.nasa.pds.registry.common.Request.Method; +import gov.nasa.pds.registry.common.Request.Count; +import gov.nasa.pds.registry.common.Request.Get; +import gov.nasa.pds.registry.common.Request.MGet; +import gov.nasa.pds.registry.common.Request.Mapping; +import gov.nasa.pds.registry.common.Request.Search; +import gov.nasa.pds.registry.common.Request.Setting; import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.regions.Region; import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.ResponseException; import gov.nasa.pds.registry.common.RestClient; public class RestClientWrapper implements RestClient { @@ -35,24 +40,84 @@ public void close() throws IOException { this.httpClient.close(); } @Override - public Bulk createBulkRequest() throws IOException { + public Bulk createBulkRequest() { // TODO Auto-generated method stub return null; } @Override - public Request createRequest(Method method, String endpoint) { + public Count createCountRequest() { // TODO Auto-generated method stub return null; } @Override - public Response performRequest(Request request) throws IOException { + public Get createGetRequest() { // TODO Auto-generated method stub return null; } @Override - public Response performRequest(Bulk request) throws IOException { + public Mapping createMappingRequest() { + // TODO Auto-generated method stub + return null; + } + @Override + public MGet createMGetRequest() { + // TODO Auto-generated method stub + return null; + } + @Override + public Search createSearchRequest() { + // TODO Auto-generated method stub + return null; + } + @Override + public Setting createSettingRequest() { + // TODO Auto-generated method stub + return null; + } + @Override + public Response create(String indexName, String configAsJson) + throws IOException, ResponseException { + // TODO Auto-generated method stub + return null; + } + @Override + public Response delete(String indexName) throws IOException, ResponseException { + // TODO Auto-generated method stub + return null; + } + @Override + public Response exists(String indexName) throws IOException, ResponseException { + // TODO Auto-generated method stub + return null; + } + @Override + public Response performRequest(Bulk request) throws IOException, ResponseException { + // TODO Auto-generated method stub + return null; + } + @Override + public Response performRequest(Count request) throws IOException, ResponseException { + // TODO Auto-generated method stub + return null; + } + @Override + public Response performRequest(Get request) throws IOException, ResponseException { + // TODO Auto-generated method stub + return null; + } + @Override + public Response performRequest(Mapping request) throws IOException, ResponseException { + // TODO Auto-generated method stub + return null; + } + @Override + public Response performRequest(Search request) throws IOException, ResponseException { + // TODO Auto-generated method stub + return null; + } + @Override + public Response performRequest(Setting request) throws IOException, ResponseException { // TODO Auto-generated method stub return null; } - } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java new file mode 100644 index 0000000..a08398c --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java @@ -0,0 +1,39 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.Collection; +import org.opensearch.client.opensearch.core.SearchRequest; +import gov.nasa.pds.registry.common.Request.Search; + +public class SearchImpl implements Search { + final SearchRequest.Builder craftsman = new SearchRequest.Builder(); + @Override + public Search buildAlternativeIds(Collection lids) { + return this; + } + @Override + public Search buildLatestLidVids(Collection lids) { + return this; + } + @Override + public Search buildListFields(String dataType) { + return this; + } + @Override + public Search buildListLdds(String namespace) { + return this; + } + @Override + public Search buildTheseIds(Collection lids) { + return this; + } + @Override + public Search setIndex(String name) { + this.craftsman.index(name); + return this; + } + @Override + public Search setPretty(boolean pretty) { + // ignored because Java v2 returns a document not JSON + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java new file mode 100644 index 0000000..1c42dba --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java @@ -0,0 +1,10 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import gov.nasa.pds.registry.common.Request.Setting; + +public class SettingImpl implements Setting { + @Override + public Setting setIndex(String name) { + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkImpl.java index b1f81f5..c45798a 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkImpl.java @@ -29,8 +29,18 @@ public Bulk setIndex(String name) { return this; } @Override - public Bulk setRefresh(String type) { - this.refresh = type; + public Bulk setRefresh(Refresh type) { + switch (type) { + case False: + this.refresh = "false"; + break; + case True: + this.refresh = "true"; + break; + case WaitFor: + this.refresh = "wait_for"; + break; + } return this; } @Override diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java index 83093fc..f5be89e 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java @@ -231,7 +231,7 @@ public int loadBatch(List data, Set errorLidvids, int retries) t try { - bulk = this.conFactory.createRestClient().createBulkRequest().setRefresh("wait_for"); + bulk = this.conFactory.createRestClient().createBulkRequest().setRefresh(Request.Bulk.Refresh.WaitFor); for (int index = 0 ; index < data.size() ; index++) { bulk.add(data.get(index), data.get(++index)); } From 9f9a35dd6a7191320d782e68d63088c9ad03fbb4 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 3 Apr 2024 11:49:36 -0700 Subject: [PATCH 21/41] add handler Not sure how it was tested, but added the handler to the system property so that it can be found by harvest. Not sure where all this should go or will go, but definitely needs to be here. --- .../pds/registry/common/EstablishConnectionFactory.java | 7 ++++++- .../java/gov/nasa/pds/registry/common/app/Handler.java | 2 -- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java index f1e13fe..acc141a 100644 --- a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java @@ -13,7 +13,12 @@ public static ConnectionFactory from (String urlToRegistryConnection) throws Exc public static ConnectionFactory from (String urlToRegistryConnection, String authFile) throws Exception { return EstablishConnectionFactory.from (urlToRegistryConnection, AuthContent.from(authFile)); } - private static ConnectionFactory from (String urlToRegistryConnection, AuthContent auth) throws Exception { + private static synchronized ConnectionFactory from (String urlToRegistryConnection, AuthContent auth) throws Exception { + final String key = "java.protocol.handler.pkgs", me = "gov.nasa.pds.registry.common"; + String providers = System.getProperty(key, ""); + if (providers.isEmpty()) providers = me; + if (!providers.contains(me)) providers += "|" + me; + System.setProperty(key, providers); RegistryConnectionContent conn = RegistryConnectionContent.from (new URL(urlToRegistryConnection)); if (conn.isDirectConnection()) return Direct.build(conn.getServerUrl(), auth); diff --git a/src/main/java/gov/nasa/pds/registry/common/app/Handler.java b/src/main/java/gov/nasa/pds/registry/common/app/Handler.java index ecc3554..d38415a 100644 --- a/src/main/java/gov/nasa/pds/registry/common/app/Handler.java +++ b/src/main/java/gov/nasa/pds/registry/common/app/Handler.java @@ -6,11 +6,9 @@ import java.net.URLStreamHandler; public class Handler extends URLStreamHandler { - @Override protected URLConnection openConnection(URL u) throws IOException { System.out.println("here"); return null; } - } From e4bbfe1423e5a2d4c0978da0d3087e95fac2a5b7 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 3 Apr 2024 12:48:16 -0700 Subject: [PATCH 22/41] correct parsing of app:// --- .../nasa/pds/registry/common/app/Handler.java | 18 ++++++++++++++++++ .../connection/RegistryConnectionContent.java | 3 ++- .../connections/direct/localhost2.xml | 4 ++++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 src/main/resources/connections/direct/localhost2.xml diff --git a/src/main/java/gov/nasa/pds/registry/common/app/Handler.java b/src/main/java/gov/nasa/pds/registry/common/app/Handler.java index d38415a..2a2a55a 100644 --- a/src/main/java/gov/nasa/pds/registry/common/app/Handler.java +++ b/src/main/java/gov/nasa/pds/registry/common/app/Handler.java @@ -1,11 +1,29 @@ package gov.nasa.pds.registry.common.app; +import java.io.File; import java.io.IOException; import java.net.URL; import java.net.URLConnection; import java.net.URLStreamHandler; public class Handler extends URLStreamHandler { + @Override + protected void parseURL(URL u, String spec, int start, int limit) { + /* + * Ugly backwards compatibility. Flip any file separator + * characters to be forward slashes. This is a nop on Unix + * and "fixes" win32 file paths. According to RFC 2396, + * only forward slashes may be used to represent hierarchy + * separation in a URL but previous releases unfortunately + * performed this "fixup" behavior in the file URL parsing code + * rather than forcing this to be fixed in the caller of the URL + * class where it belongs. Since backslash is an "unwise" + * character that would normally be encoded if literally intended + * as a non-seperator character the damage of veering away from the + * specification is presumably limited. + */ + super.parseURL(u, spec.replace(File.separatorChar, '/'), start, limit); + } @Override protected URLConnection openConnection(URL u) throws IOException { System.out.println("here"); diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java b/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java index 8fd4177..18c2e32 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java @@ -20,7 +20,8 @@ public static RegistryConnectionContent from (URL registry_connection) throws IO throw new IllegalArgumentException("URL protocol must be one of " + acceptable + " not the one specified: " + registry_connection); } if (registry_connection.getProtocol().equalsIgnoreCase("app")) { - registry_connection = RegistryConnectionContent.class.getResource(registry_connection.getPath()); + String resource = "/" + registry_connection.getAuthority() + registry_connection.getPath(); + registry_connection = RegistryConnectionContent.class.getResource(resource); } xml = (RegistryConnection)jaxbContext.createUnmarshaller().unmarshal(registry_connection); return new RegistryConnectionContent(xml); diff --git a/src/main/resources/connections/direct/localhost2.xml b/src/main/resources/connections/direct/localhost2.xml new file mode 100644 index 0000000..8d78b7e --- /dev/null +++ b/src/main/resources/connections/direct/localhost2.xml @@ -0,0 +1,4 @@ + + + https://localhost:9200 + From d37e781fb260cd6baf7faea902d38cbf0df5daeb Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 3 Apr 2024 13:01:07 -0700 Subject: [PATCH 23/41] add controlling the java SDK to the options --- .../common/connection/config/DirectType.java | 37 +++++++++++++++++++ src/main/resources/registry-connection.xsd | 12 ++++++ 2 files changed, 49 insertions(+) diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/config/DirectType.java b/src/main/java/gov/nasa/pds/registry/common/connection/config/DirectType.java index a343574..2228301 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/config/DirectType.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/config/DirectType.java @@ -7,6 +7,7 @@ package gov.nasa.pds.registry.common.connection.config; +import java.math.BigInteger; import jakarta.xml.bind.annotation.XmlAccessType; import jakarta.xml.bind.annotation.XmlAccessorType; import jakarta.xml.bind.annotation.XmlAttribute; @@ -27,6 +28,11 @@ * it is often necessary to use self signed * certificates. Setting this option to true * will all for self signed certificates. + * + * An optional 3rd piece of information @sdk is the Java SDK version to use + * for the connection. The traditional SDK is 1. The newer SDK 2 was + * introduced with the serverless opensearch development and beyond can + * also be used. The default is 2 as we are deprecating the use of 1. * * *

Java class for direct_type complex type. @@ -37,6 +43,7 @@ * * * + * * * * @@ -55,6 +62,8 @@ public class DirectType { @XmlJavaTypeAdapter(NormalizedStringAdapter.class) @XmlSchemaType(name = "normalizedString") protected String value; + @XmlAttribute(name = "sdk") + protected BigInteger sdk; @XmlAttribute(name = "trust_self_signed") protected Boolean trustSelfSigned; @@ -82,6 +91,34 @@ public void setValue(String value) { this.value = value; } + /** + * Gets the value of the sdk property. + * + * @return + * possible object is + * {@link BigInteger } + * + */ + public BigInteger getSdk() { + if (sdk == null) { + return new BigInteger("2"); + } else { + return sdk; + } + } + + /** + * Sets the value of the sdk property. + * + * @param value + * allowed object is + * {@link BigInteger } + * + */ + public void setSdk(BigInteger value) { + this.sdk = value; + } + /** * Gets the value of the trustSelfSigned property. * diff --git a/src/main/resources/registry-connection.xsd b/src/main/resources/registry-connection.xsd index 5184956..b4c02a1 100644 --- a/src/main/resources/registry-connection.xsd +++ b/src/main/resources/registry-connection.xsd @@ -60,16 +60,28 @@ it is often necessary to use self signed certificates. Setting this option to true will all for self signed certificates. + + An optional 3rd piece of information @sdk is the Java SDK version to use + for the connection. The traditional SDK is 1. The newer SDK 2 was + introduced with the serverless opensearch development and beyond can + also be used. The default is 2 as we are deprecating the use of 1. + + + + + + + From 733864632d4b521ac6faa8b2827b256b4b76357b Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 3 Apr 2024 13:04:04 -0700 Subject: [PATCH 24/41] make it more clear for localhost --- src/main/resources/connections/direct/localhost1.xml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/main/resources/connections/direct/localhost1.xml diff --git a/src/main/resources/connections/direct/localhost1.xml b/src/main/resources/connections/direct/localhost1.xml new file mode 100644 index 0000000..7b5d93e --- /dev/null +++ b/src/main/resources/connections/direct/localhost1.xml @@ -0,0 +1,4 @@ + + + https://localhost:9200 + From c13338ef30a8b2c8f37754aeec339190c4e86b7c Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 3 Apr 2024 13:34:43 -0700 Subject: [PATCH 25/41] refactored a bit For testing, going to use a local opensearch with java v2 to simplify things. However this means adding the ability to do it and change the differentiator between the two connection types (now three): direct (managed) and multitenancy (serverless) and SDK 1 or SDK 2. The refactoring results in establishing dire connections using SDK 1 or SDK 2 or multitenancy with sdk 2. --- .../common/EstablishConnectionFactory.java | 12 ++-- .../connection/RegistryConnectionContent.java | 4 +- .../{Direct.java => UseOpensearchSDK1.java} | 10 +-- ...ltiTenancy.java => UseOpensearchSDK2.java} | 19 +++--- .../common/connection/aws/MappingImpl.java | 53 ++++++++++++++-- .../connection/aws/RestClientWrapper.java | 61 ++++++++++--------- .../common/connection/aws/SearchImpl.java | 44 +++++++++++++ .../common/connection/aws/SettingImpl.java | 3 + 8 files changed, 153 insertions(+), 53 deletions(-) rename src/main/java/gov/nasa/pds/registry/common/connection/{Direct.java => UseOpensearchSDK1.java} (82%) rename src/main/java/gov/nasa/pds/registry/common/connection/{MultiTenancy.java => UseOpensearchSDK2.java} (87%) diff --git a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java index acc141a..ca2f126 100644 --- a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java @@ -2,8 +2,8 @@ import java.net.URL; import gov.nasa.pds.registry.common.connection.AuthContent; -import gov.nasa.pds.registry.common.connection.Direct; -import gov.nasa.pds.registry.common.connection.MultiTenancy; +import gov.nasa.pds.registry.common.connection.UseOpensearchSDK1; +import gov.nasa.pds.registry.common.connection.UseOpensearchSDK2; import gov.nasa.pds.registry.common.connection.RegistryConnectionContent; public class EstablishConnectionFactory { @@ -21,8 +21,12 @@ private static synchronized ConnectionFactory from (String urlToRegistryConnecti System.setProperty(key, providers); RegistryConnectionContent conn = RegistryConnectionContent.from (new URL(urlToRegistryConnection)); - if (conn.isDirectConnection()) return Direct.build(conn.getServerUrl(), auth); - if (conn.isCognitoConnection()) return MultiTenancy.build(conn.getCognitoClientId(), auth); + if (conn.isDirectConnection()) { + if (conn.getServerUrl().getSdk().intValue() == 1) UseOpensearchSDK1.build(conn.getServerUrl(), auth).setIndexName(conn.getIndex()); + if (conn.getServerUrl().getSdk().intValue() == 2) UseOpensearchSDK2.build(conn.getServerUrl(), auth).setIndexName(conn.getIndex()); + throw new RuntimeException("The SDK version '" + String.valueOf(conn.getServerUrl().getSdk()) + "is not supported"); + } + if (conn.isCognitoConnection()) return UseOpensearchSDK2.build(conn.getCognitoClientId(), auth).setIndexName(conn.getIndex()); throw new RuntimeException("New XML/Java choices in src/main/resources/registry_connection.xsd that are not handled."); } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java b/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java index 18c2e32..442164b 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java @@ -20,8 +20,8 @@ public static RegistryConnectionContent from (URL registry_connection) throws IO throw new IllegalArgumentException("URL protocol must be one of " + acceptable + " not the one specified: " + registry_connection); } if (registry_connection.getProtocol().equalsIgnoreCase("app")) { - String resource = "/" + registry_connection.getAuthority() + registry_connection.getPath(); - registry_connection = RegistryConnectionContent.class.getResource(resource); + registry_connection = RegistryConnectionContent.class.getResource + ("/" + registry_connection.getAuthority() + registry_connection.getPath()); } xml = (RegistryConnection)jaxbContext.createUnmarshaller().unmarshal(registry_connection); return new RegistryConnectionContent(xml); diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java similarity index 82% rename from src/main/java/gov/nasa/pds/registry/common/connection/Direct.java rename to src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java index ef5d078..535e10c 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/Direct.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java @@ -10,14 +10,14 @@ import gov.nasa.pds.registry.common.connection.config.DirectType; import gov.nasa.pds.registry.common.connection.es.RestClientWrapper; -public class Direct implements Cloneable, ConnectionFactory { +public class UseOpensearchSDK1 implements Cloneable, ConnectionFactory { final private boolean veryTrusting; final private AuthContent auth; final private HttpHost host; final private URL service; private String index = null; - public static Direct build (DirectType url, AuthContent auth) throws Exception { + public static UseOpensearchSDK1 build (DirectType url, AuthContent auth) throws Exception { URL service = new URL(url.getValue()); // Trust self-signed certificates if(url.isTrustSelfSigned()) @@ -25,10 +25,10 @@ public static Direct build (DirectType url, AuthContent auth) throws Exception { SSLContext sslCtx = SSLUtils.createTrustAllContext(); HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory()); } - return new Direct (service, auth, url.isTrustSelfSigned()); + return new UseOpensearchSDK1 (service, auth, url.isTrustSelfSigned()); } - private Direct (URL service, AuthContent auth, boolean trustSelfSigned) { + private UseOpensearchSDK1 (URL service, AuthContent auth, boolean trustSelfSigned) { this.auth = auth; this.host = new HttpHost(service.getHost(), service.getPort(), service.getProtocol()); this.service = service; @@ -36,7 +36,7 @@ private Direct (URL service, AuthContent auth, boolean trustSelfSigned) { } @Override public ConnectionFactory clone() { - return new Direct(this.service, this.auth, this.veryTrusting).setIndexName(this.index); + return new UseOpensearchSDK1(this.service, this.auth, this.veryTrusting).setIndexName(this.index); } @Override public RestClient createRestClient() throws Exception { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK2.java similarity index 87% rename from src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java rename to src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK2.java index 54f1a91..64e5ece 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/MultiTenancy.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK2.java @@ -18,13 +18,15 @@ import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.common.connection.aws.RestClientWrapper; import gov.nasa.pds.registry.common.connection.config.CognitoType; +import gov.nasa.pds.registry.common.connection.config.DirectType; -public final class MultiTenancy implements ConnectionFactory { +public final class UseOpensearchSDK2 implements ConnectionFactory { + final private boolean isServerless; final private AuthContent auth; final private HttpHost host; final private URL endpoint; private String index = null; - public static MultiTenancy build (CognitoType cog, AuthContent auth) throws IOException, InterruptedException { + public static UseOpensearchSDK2 build (CognitoType cog, AuthContent auth) throws IOException, InterruptedException { boolean expectedContent = true; Gson gson = new Gson(); HttpClient client = HttpClient.newHttpClient(); @@ -83,21 +85,24 @@ public static MultiTenancy build (CognitoType cog, AuthContent auth) throws IOEx awsCreds.setProperty("aws.sessionToken", content.get("Credentials").get("SessionToken")); System.setProperties(awsCreds); URL signed = new URL("https://p5qmxrldysl1gy759hqf.us-west-2.aoss.amazonaws.com"); // move to config or lambda?? - return new MultiTenancy(auth, signed); + return new UseOpensearchSDK2(auth, signed, true); } - - private MultiTenancy (AuthContent auth, URL opensearchEndpoint) { + public static UseOpensearchSDK2 build (DirectType url, AuthContent auth) throws Exception { + return null; + } + private UseOpensearchSDK2 (AuthContent auth, URL opensearchEndpoint, boolean isServerless) { this.auth = auth; this.endpoint = opensearchEndpoint; this.host = new HttpHost(this.endpoint.getHost(), this.endpoint.getPort(), this.endpoint.getProtocol()); + this.isServerless = isServerless; } @Override public ConnectionFactory clone() { - return new MultiTenancy(this.auth, this.endpoint).setIndexName(this.index); + return new UseOpensearchSDK2(this.auth, this.endpoint, this.isServerless).setIndexName(this.index); } @Override public RestClient createRestClient() throws Exception { - return new RestClientWrapper(this); + return new RestClientWrapper(this, this.isServerless); } @Override public CredentialsProvider getCredentials() { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java index fe96368..b932703 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java @@ -1,21 +1,64 @@ package gov.nasa.pds.registry.common.connection.aws; import java.util.Collection; +import java.util.HashMap; +import org.opensearch.client.opensearch._types.mapping.BooleanProperty; +import org.opensearch.client.opensearch._types.mapping.DoubleNumberProperty; +import org.opensearch.client.opensearch._types.mapping.FloatNumberProperty; +import org.opensearch.client.opensearch._types.mapping.IntegerNumberProperty; +import org.opensearch.client.opensearch._types.mapping.KeywordProperty; +import org.opensearch.client.opensearch._types.mapping.Property; +import org.opensearch.client.opensearch._types.mapping.TextProperty; +import org.opensearch.client.opensearch.indices.GetMappingRequest; +import org.opensearch.client.opensearch.indices.PutMappingRequest; import gov.nasa.pds.registry.common.Request.Mapping; import gov.nasa.pds.registry.common.util.Tuple; public class MappingImpl implements Mapping { - + boolean isGet = true; + final GetMappingRequest.Builder craftsman_get = new GetMappingRequest.Builder(); + final PutMappingRequest.Builder craftsman_set = new PutMappingRequest.Builder(); @Override public Mapping buildUpdateFieldSchema(Collection pairs) { - // TODO Auto-generated method stub - return null; + HashMap mapping = new HashMap(); + for (Tuple t : pairs) { + Property.Builder journeyman = new Property.Builder(); + String fieldName = t.item1; + String fieldType = t.item2; + switch (fieldType) { + case "boolean": + journeyman.boolean_(new BooleanProperty.Builder().build()); + break; + case "double": + journeyman.double_(new DoubleNumberProperty.Builder().build()); + break; + case "float": + journeyman.float_(new FloatNumberProperty.Builder().build()); + break; + case "integer": + journeyman.integer(new IntegerNumberProperty.Builder().build()); + break; + case "keyword": + journeyman.keyword(new KeywordProperty.Builder().build()); + break; + case "text": + journeyman.text(new TextProperty.Builder().build()); + break; + default: + throw new RuntimeException("Cannot map type '" + fieldType + "' yet. Please review code and fix."); + } + mapping.put(fieldName, journeyman.build()); + } + this.craftsman_set.properties(mapping); + this.isGet = false; + return this; } @Override public Mapping setIndex(String name) { - // TODO Auto-generated method stub - return null; + this.craftsman_get.index(name); + this.craftsman_set.index(name); + return this; } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java index 07f2a0b..617d30c 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java @@ -2,6 +2,10 @@ import java.io.IOException; import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.indices.CreateIndexRequest; +import org.opensearch.client.opensearch.indices.DeleteIndexRequest; +import org.opensearch.client.opensearch.indices.ExistsRequest; import org.opensearch.client.transport.aws.AwsSdk2Transport; import org.opensearch.client.transport.aws.AwsSdk2TransportOptions; import gov.nasa.pds.registry.common.ConnectionFactory; @@ -22,17 +26,21 @@ public class RestClientWrapper implements RestClient { final private OpenSearchClient client; final private SdkHttpClient httpClient; - public RestClientWrapper(ConnectionFactory conFact) { + public RestClientWrapper(ConnectionFactory conFact, boolean isServerless) { this.httpClient = ApacheHttpClient.builder().build(); - this.client = new OpenSearchClient( - new AwsSdk2Transport( - httpClient, - conFact.getHostName(), - "aoss", - Region.US_WEST_2, // signing service region that we should probably get from host name?? - AwsSdk2TransportOptions.builder().build() - ) - ); + if (isServerless) { + this.client = new OpenSearchClient( + new AwsSdk2Transport( + httpClient, + conFact.getHostName(), + "aoss", + Region.US_WEST_2, // signing service region that we should probably get from host name?? + AwsSdk2TransportOptions.builder().build() + ) + ); + } else { + this.client = null; + } } @Override public void close() throws IOException { @@ -41,58 +49,51 @@ public void close() throws IOException { } @Override public Bulk createBulkRequest() { - // TODO Auto-generated method stub - return null; + return new BulkImpl(); } @Override public Count createCountRequest() { - // TODO Auto-generated method stub - return null; + return new CountImpl(); } @Override public Get createGetRequest() { - // TODO Auto-generated method stub - return null; + return new GetImpl(); } @Override public Mapping createMappingRequest() { - // TODO Auto-generated method stub - return null; + return new MappingImpl(); } @Override public MGet createMGetRequest() { - // TODO Auto-generated method stub - return null; + return new MGetImpl(); } @Override public Search createSearchRequest() { - // TODO Auto-generated method stub - return null; + return new SearchImpl(); } @Override public Setting createSettingRequest() { - // TODO Auto-generated method stub - return null; + return new SettingImpl(); } @Override - public Response create(String indexName, String configAsJson) - throws IOException, ResponseException { - // TODO Auto-generated method stub + public Response create(String indexName, String configAsJson) throws IOException, ResponseException { + // FIXME: need to change JSON into V2 mapping + this.client.indices().create(new CreateIndexRequest.Builder().index(indexName).build()); return null; } @Override public Response delete(String indexName) throws IOException, ResponseException { - // TODO Auto-generated method stub + this.client.indices().delete(new DeleteIndexRequest.Builder().index(indexName).build()); return null; } @Override public Response exists(String indexName) throws IOException, ResponseException { - // TODO Auto-generated method stub + this.client.indices().exists(new ExistsRequest.Builder().index(indexName).build()); return null; } @Override public Response performRequest(Bulk request) throws IOException, ResponseException { - // TODO Auto-generated method stub + BulkResponse resp = this.client.bulk(((BulkImpl)request).craftsman.build()); return null; } @Override diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java index a08398c..fe0945a 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java @@ -1,29 +1,73 @@ package gov.nasa.pds.registry.common.connection.aws; +import java.util.ArrayList; import java.util.Collection; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; +import org.opensearch.client.opensearch._types.query_dsl.IdsQuery; +import org.opensearch.client.opensearch._types.query_dsl.MatchQuery; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.Query.Builder; +import org.opensearch.client.opensearch._types.query_dsl.TermsQuery; +import org.opensearch.client.opensearch._types.query_dsl.TermsQueryField; import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.client.opensearch.core.search.SourceConfig; +import org.opensearch.client.opensearch.core.search.SourceFilter; import gov.nasa.pds.registry.common.Request.Search; public class SearchImpl implements Search { final SearchRequest.Builder craftsman = new SearchRequest.Builder(); + private void buildIds (Collection lids, boolean alt) { + SourceConfig.Builder journeyman = new SourceConfig.Builder(); + if (alt) { + journeyman.filter(new SourceFilter.Builder().includes("alternate_ids").build()); + } + this.craftsman.query(new Query.Builder().ids(new IdsQuery.Builder().values(new ArrayList(lids)).build()).build()); + this.craftsman.size(lids.size()); + this.craftsman.source(journeyman.fetch(alt).build()); + } + private Query.Builder matchQuery (String fieldname, String fieldvalue) { + return (Builder)new Query.Builder().match(new MatchQuery.Builder().field(fieldname).query(new FieldValue.Builder().stringValue(fieldvalue).build()).build()); + } @Override public Search buildAlternativeIds(Collection lids) { + this.buildIds(lids, true); return this; } @Override public Search buildLatestLidVids(Collection lids) { + ArrayList terms = new ArrayList(lids.size()); + for (String lid : lids) { + terms.add(new FieldValue.Builder().stringValue(lid).build()); + } + // FIXME: need to work out aggregates + // this.craftsman.aggregations("latest", ); + this.craftsman.query(new Query.Builder().terms(new TermsQuery.Builder().field("lid").terms(new TermsQueryField.Builder().value(terms).build()).build()).build()); + this.craftsman.size(0); + this.craftsman.source(new SourceConfig.Builder().fetch(false).build()); return this; } @Override public Search buildListFields(String dataType) { + this.craftsman.query(new Query.Builder().bool(new BoolQuery.Builder().must(this.matchQuery("es_data_type", dataType).build()).build()).build()); + this.craftsman.size(1000); // have no idea why hardcoded but it is (.es.JsonHelper:217 + this.craftsman.source(new SourceConfig.Builder().filter(new SourceFilter.Builder().includes("es_field_name").build()).build()); return this; } @Override public Search buildListLdds(String namespace) { + BoolQuery.Builder journeyman = new BoolQuery.Builder() + .must(this.matchQuery("class_ns", "registry").build(), + this.matchQuery("class_name", "LDD_Info").build(), + this.matchQuery("attr_ns", namespace).build()); + this.craftsman.query(new Query.Builder().bool(journeyman.build()).build()); + this.craftsman.size(1000); // have no idea why hardcoded but it is (.es.JsonHelper:265 + this.craftsman.source(new SourceConfig.Builder().filter(new SourceFilter.Builder().includes("date", "attr_name").build()).build()); return this; } @Override public Search buildTheseIds(Collection lids) { + this.buildIds(lids, false); return this; } @Override diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java index 1c42dba..e1034f2 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java @@ -1,10 +1,13 @@ package gov.nasa.pds.registry.common.connection.aws; +import org.opensearch.client.opensearch.indices.GetIndicesSettingsRequest; import gov.nasa.pds.registry.common.Request.Setting; public class SettingImpl implements Setting { + final GetIndicesSettingsRequest.Builder craftsman = new GetIndicesSettingsRequest.Builder(); @Override public Setting setIndex(String name) { + this.craftsman.index(name); return this; } } From 4183c150016a89dff09bd1b3134c339c1c3d50d2 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 4 Apr 2024 08:59:48 -0700 Subject: [PATCH 26/41] add configuration value for endpoint in AWS --- .../common/connection/config/CognitoType.java | 38 ++++++++++++++++++- src/main/resources/registry-connection.xsd | 6 ++- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/config/CognitoType.java b/src/main/java/gov/nasa/pds/registry/common/connection/config/CognitoType.java index a07bc4a..d7834c9 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/config/CognitoType.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/config/CognitoType.java @@ -22,9 +22,10 @@ * Currently, cognito to serverless opensearh requires three bits of * information: * 1. clientID: magic ID which must be given and is not a secret but - * sure looks like one + * sure looks like one + * 3. endpoint: the endpoint for serverless opensearch * 2. @gateway: an endpoint that exchanges the cognito authentication - * to a signed URL that can be used for communicating with + * to an IAM role used for access control with serverless * opensearch on AWS * 3. @IDP: the cognito authentication endpoint that exchanges the * username/password to a set of tokens used by the gateway. @@ -38,6 +39,7 @@ * * * + * * * * @@ -57,6 +59,10 @@ public class CognitoType { @XmlJavaTypeAdapter(NormalizedStringAdapter.class) @XmlSchemaType(name = "normalizedString") protected String value; + @XmlAttribute(name = "endpoint") + @XmlJavaTypeAdapter(NormalizedStringAdapter.class) + @XmlSchemaType(name = "normalizedString") + protected String endpoint; @XmlAttribute(name = "gateway") @XmlJavaTypeAdapter(NormalizedStringAdapter.class) @XmlSchemaType(name = "normalizedString") @@ -90,6 +96,34 @@ public void setValue(String value) { this.value = value; } + /** + * Gets the value of the endpoint property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getEndpoint() { + if (endpoint == null) { + return "https://p5qmxrldysl1gy759hqf.us-west-2.aoss.amazonaws.com"; + } else { + return endpoint; + } + } + + /** + * Sets the value of the endpoint property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setEndpoint(String value) { + this.endpoint = value; + } + /** * Gets the value of the gateway property. * diff --git a/src/main/resources/registry-connection.xsd b/src/main/resources/registry-connection.xsd index b4c02a1..0e723df 100644 --- a/src/main/resources/registry-connection.xsd +++ b/src/main/resources/registry-connection.xsd @@ -32,9 +32,10 @@ Currently, cognito to serverless opensearh requires three bits of information: 1. clientID: magic ID which must be given and is not a secret but - sure looks like one + sure looks like one + 3. endpoint: the endpoint for serverless opensearch 2. @gateway: an endpoint that exchanges the cognito authentication - to a signed URL that can be used for communicating with + to an IAM role used for access control with serverless opensearch on AWS 3. @IDP: the cognito authentication endpoint that exchanges the username/password to a set of tokens used by the gateway. @@ -42,6 +43,7 @@ + From 9c7d52ced317afae715e7055ee53b1556f2edb5f Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Thu, 11 Apr 2024 11:45:08 -0700 Subject: [PATCH 27/41] wip: chaning up the returns to be interfaces too --- .../registry/common/ConnectionFactory.java | 2 + .../nasa/pds/registry/common/Response.java | 28 +++++ .../nasa/pds/registry/common/RestClient.java | 14 +-- .../common/connection/AuthContent.java | 14 +++ .../common/connection/UseOpensearchSDK1.java | 10 ++ .../common/connection/UseOpensearchSDK2.java | 25 +++-- .../common/connection/aws/BulkRespWrap.java | 66 ++++++++++++ .../connection/aws/CreateIndexRespWrap.java | 23 ++++ .../common/connection/aws/MappingImpl.java | 2 +- .../connection/aws/MappingRespImpl.java | 21 ++++ .../connection/aws/ResponseWrapper.java | 23 ---- .../connection/aws/RestClientWrapper.java | 80 +++++++++----- .../common/connection/aws/SearchImpl.java | 2 +- .../common/connection/aws/SettingImpl.java | 2 +- .../connection/aws/SettingRespImpl.java | 29 +++++ .../common/connection/es/BulkRespImpl.java | 90 ++++++++++++++++ .../common/connection/es/JsonHelper.java | 25 +++++ .../common/connection/es/MappingImpl.java | 2 +- .../common/connection/es/MappingRespImpl.java | 17 +++ .../es}/MappingsParser.java | 42 +++----- .../connection/es/ResponseNotImplYet.java | 18 ++++ .../connection/es/RestClientWrapper.java | 45 +++++--- .../connection/es/SettingsRespImpl.java | 102 ++++++++++++++++++ .../common/es/dao/BulkResponseParser.java | 22 +++- .../registry/common/es/dao/DataLoader.java | 89 ++++----------- .../registry/common/es/dao/ProductDao.java | 53 +-------- .../common/es/dao/schema/SchemaDao.java | 6 +- 27 files changed, 619 insertions(+), 233 deletions(-) create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkRespWrap.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexRespWrap.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingRespImpl.java delete mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/ResponseWrapper.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingRespImpl.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/MappingRespImpl.java rename src/main/java/gov/nasa/pds/registry/common/{es/dao/schema => connection/es}/MappingsParser.java (80%) create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseNotImplYet.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/SettingsRespImpl.java diff --git a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java index 5d3a540..a88ff85 100644 --- a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java @@ -6,7 +6,9 @@ public interface ConnectionFactory { public RestClient createRestClient() throws Exception; public CredentialsProvider getCredentials(); + public org.apache.hc.client5.http.auth.CredentialsProvider getCredentials5(); public HttpHost getHost(); + public org.apache.hc.core5.http.HttpHost getHost5(); public String getHostName(); public String getIndexName(); public boolean isTrustingSelfSigned(); diff --git a/src/main/java/gov/nasa/pds/registry/common/Response.java b/src/main/java/gov/nasa/pds/registry/common/Response.java index ea79b80..d9917a1 100644 --- a/src/main/java/gov/nasa/pds/registry/common/Response.java +++ b/src/main/java/gov/nasa/pds/registry/common/Response.java @@ -1,9 +1,37 @@ package gov.nasa.pds.registry.common; +import java.util.List; +import java.util.Set; import org.apache.http.HttpEntity; import org.apache.http.StatusLine; public interface Response { + public interface Bulk { + public interface Item { + public boolean error(); + public String id(); + public String index(); + public String operation(); + public String reason(); + public String result(); + public int status(); + } + public boolean errors(); + public List items(); + public long took(); + } + public interface CreatedIndex { + public boolean acknowledge(); + public boolean acknowledgeShards(); + public String getIndex(); + } + public interface Mapping { + public Set fieldNames(); + } + public interface Settings { + public int replicas(); + public int shards(); + } public HttpEntity getEntity(); public StatusLine getStatusLine(); public void printWarnings(); diff --git a/src/main/java/gov/nasa/pds/registry/common/RestClient.java b/src/main/java/gov/nasa/pds/registry/common/RestClient.java index a630a4e..77fde98 100644 --- a/src/main/java/gov/nasa/pds/registry/common/RestClient.java +++ b/src/main/java/gov/nasa/pds/registry/common/RestClient.java @@ -11,13 +11,13 @@ public interface RestClient extends Closeable { public Request.MGet createMGetRequest(); public Request.Search createSearchRequest(); public Request.Setting createSettingRequest(); - public Response create (String indexName, String configAsJson) throws IOException,ResponseException; - public Response delete (String indexName) throws IOException,ResponseException; - public Response exists (String indexName) throws IOException,ResponseException; - public Response performRequest(Request.Bulk request) throws IOException,ResponseException; - public Response performRequest(Request.Count request) throws IOException,ResponseException; + public Response.CreatedIndex create (String indexName, String configAsJson) throws IOException,ResponseException; + public void delete (String indexName) throws IOException,ResponseException; + public boolean exists (String indexName) throws IOException,ResponseException; + public Response.Bulk performRequest(Request.Bulk request) throws IOException,ResponseException; + public long performRequest(Request.Count request) throws IOException,ResponseException; public Response performRequest(Request.Get request) throws IOException,ResponseException; - public Response performRequest(Request.Mapping request) throws IOException,ResponseException; + public Response.Mapping performRequest(Request.Mapping request) throws IOException,ResponseException; public Response performRequest(Request.Search request) throws IOException,ResponseException; - public Response performRequest(Request.Setting request) throws IOException,ResponseException; + public Response.Settings performRequest(Request.Setting request) throws IOException,ResponseException; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java index b8a90ec..2b0203d 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java @@ -2,6 +2,7 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; +import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; @@ -57,6 +58,19 @@ public CredentialsProvider getCredentials() { creds.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(this.getUser(), this.getPassword())); return creds; } + public CredentialsProvider getCredentials(HttpHost scope) { + CredentialsProvider creds; + creds = new BasicCredentialsProvider(); + creds.setCredentials(new AuthScope(scope), new UsernamePasswordCredentials(this.getUser(), this.getPassword())); + return creds; + } + public org.apache.hc.client5.http.auth.CredentialsProvider getCredentials5(org.apache.hc.core5.http.HttpHost scope) { + org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider creds = + new org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider(); + creds.setCredentials(new org.apache.hc.client5.http.auth.AuthScope(scope), + new org.apache.hc.client5.http.auth.UsernamePasswordCredentials(this.getUser(), this.getPassword().toCharArray())); + return creds; + } public String getPassword() { return password; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java index 535e10c..ae8d57b 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java @@ -14,6 +14,7 @@ public class UseOpensearchSDK1 implements Cloneable, ConnectionFactory { final private boolean veryTrusting; final private AuthContent auth; final private HttpHost host; + final private org.apache.hc.core5.http.HttpHost host5; final private URL service; private String index = null; @@ -31,6 +32,7 @@ public static UseOpensearchSDK1 build (DirectType url, AuthContent auth) throws private UseOpensearchSDK1 (URL service, AuthContent auth, boolean trustSelfSigned) { this.auth = auth; this.host = new HttpHost(service.getHost(), service.getPort(), service.getProtocol()); + this.host5 = new org.apache.hc.core5.http.HttpHost(service.getProtocol(), service.getHost(), service.getPort()); this.service = service; this.veryTrusting = trustSelfSigned; } @@ -43,6 +45,10 @@ public RestClient createRestClient() throws Exception { return new RestClientWrapper(this); } @Override + public org.apache.hc.client5.http.auth.CredentialsProvider getCredentials5() { + return this.auth.getCredentials5(this.getHost5()); + } + @Override public HttpHost getHost() { return this.host; } @@ -51,6 +57,10 @@ public String getHostName() { return this.host.getHostName(); } @Override + public org.apache.hc.core5.http.HttpHost getHost5() { + return this.host5; + } + @Override public String getIndexName() { return this.index; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK2.java b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK2.java index 64e5ece..d61f67c 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK2.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK2.java @@ -22,8 +22,10 @@ public final class UseOpensearchSDK2 implements ConnectionFactory { final private boolean isServerless; + final private boolean veryTrusting; final private AuthContent auth; final private HttpHost host; + final private org.apache.hc.core5.http.HttpHost host5; final private URL endpoint; private String index = null; public static UseOpensearchSDK2 build (CognitoType cog, AuthContent auth) throws IOException, InterruptedException { @@ -84,21 +86,22 @@ public static UseOpensearchSDK2 build (CognitoType cog, AuthContent auth) throws awsCreds.setProperty("aws.secretAccessKey", content.get("Credentials").get("SecretKey")); awsCreds.setProperty("aws.sessionToken", content.get("Credentials").get("SessionToken")); System.setProperties(awsCreds); - URL signed = new URL("https://p5qmxrldysl1gy759hqf.us-west-2.aoss.amazonaws.com"); // move to config or lambda?? - return new UseOpensearchSDK2(auth, signed, true); + return new UseOpensearchSDK2(auth, new URL(cog.getEndpoint()), true, false); } public static UseOpensearchSDK2 build (DirectType url, AuthContent auth) throws Exception { - return null; + return new UseOpensearchSDK2(auth, new URL(url.getValue()), false, url.isTrustSelfSigned()); } - private UseOpensearchSDK2 (AuthContent auth, URL opensearchEndpoint, boolean isServerless) { + private UseOpensearchSDK2 (AuthContent auth, URL opensearchEndpoint, boolean isServerless, boolean veryTrusting) { this.auth = auth; this.endpoint = opensearchEndpoint; this.host = new HttpHost(this.endpoint.getHost(), this.endpoint.getPort(), this.endpoint.getProtocol()); + this.host5 = new org.apache.hc.core5.http.HttpHost(this.endpoint.getProtocol(), this.endpoint.getHost(), this.endpoint.getPort()); this.isServerless = isServerless; + this.veryTrusting = veryTrusting; } @Override public ConnectionFactory clone() { - return new UseOpensearchSDK2(this.auth, this.endpoint, this.isServerless).setIndexName(this.index); + return new UseOpensearchSDK2(this.auth, this.endpoint, this.isServerless, this.veryTrusting).setIndexName(this.index); } @Override public RestClient createRestClient() throws Exception { @@ -106,13 +109,21 @@ public RestClient createRestClient() throws Exception { } @Override public CredentialsProvider getCredentials() { - return this.auth.getCredentials(); + return this.auth.getCredentials(this.getHost()); + } + @Override + public org.apache.hc.client5.http.auth.CredentialsProvider getCredentials5() { + return this.auth.getCredentials5(this.getHost5()); } @Override public HttpHost getHost() { return this.host; } @Override + public org.apache.hc.core5.http.HttpHost getHost5() { + return this.host5; + } + @Override public String getHostName() { return this.host.getHostName(); } @@ -122,7 +133,7 @@ public String getIndexName() { } @Override public boolean isTrustingSelfSigned() { - return false; + return this.veryTrusting; } @Override public ConnectionFactory setIndexName(String idxName) { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkRespWrap.java new file mode 100644 index 0000000..010e387 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkRespWrap.java @@ -0,0 +1,66 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.ArrayList; +import java.util.List; +import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; +import gov.nasa.pds.registry.common.Response; + +class BulkRespWrap implements Response.Bulk { + private class ItemWrap implements Response.Bulk.Item { + final BulkResponseItem parent; + ItemWrap (BulkResponseItem parent) { + this.parent = parent; + } + @Override + public String id() { + return this.parent.id(); + } + @Override + public String index() { + return this.parent.index(); + } + @Override + public String result() { + return this.parent.result(); + } + @Override + public int status() { + return this.parent.status(); + } + @Override + public boolean error() { + return this.parent.error() != null; + } + @Override + public String operation() { + return this.parent.operationType().jsonValue(); + } + @Override + public String reason() { + return this.error() ? this.parent.error().reason() : ""; + } + }; + final private ArrayList items = new ArrayList(); + final private BulkResponse parent; + BulkRespWrap(BulkResponse parent) { + this.parent = parent; + } + @Override + public boolean errors() { + return this.parent.errors(); + } + @Override + public synchronized List items() { + if (this.parent.items().size() != this.items.size()) { + for (BulkResponseItem item : this.parent.items()) { + this.items.add(new ItemWrap(item)); + } + } + return this.items(); + } + @Override + public long took() { + return this.parent.took(); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexRespWrap.java new file mode 100644 index 0000000..07f77bb --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexRespWrap.java @@ -0,0 +1,23 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.opensearch.client.opensearch.indices.CreateIndexResponse; +import gov.nasa.pds.registry.common.Response; + +class CreateIndexRespWrap implements Response.CreatedIndex { + final private CreateIndexResponse parent; + CreateIndexRespWrap(CreateIndexResponse parent) { + this.parent = parent; + } + @Override + public boolean acknowledge() { + return this.parent.acknowledged(); + } + @Override + public boolean acknowledgeShards() { + return this.parent.shardsAcknowledged(); + } + @Override + public String getIndex() { + return this.parent.index(); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java index b932703..1a4c9d6 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java @@ -14,7 +14,7 @@ import gov.nasa.pds.registry.common.Request.Mapping; import gov.nasa.pds.registry.common.util.Tuple; -public class MappingImpl implements Mapping { +class MappingImpl implements Mapping { boolean isGet = true; final GetMappingRequest.Builder craftsman_get = new GetMappingRequest.Builder(); final PutMappingRequest.Builder craftsman_set = new PutMappingRequest.Builder(); diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingRespImpl.java new file mode 100644 index 0000000..d4f2a86 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingRespImpl.java @@ -0,0 +1,21 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.HashSet; +import java.util.Set; +import org.opensearch.client.opensearch.indices.GetMappingResponse; +import org.opensearch.client.opensearch.indices.PutMappingResponse; +import gov.nasa.pds.registry.common.Response; + +class MappingRespImpl implements Response.Mapping { + final Set fieldNames; + MappingRespImpl (GetMappingResponse response) { + this.fieldNames = response.result().keySet(); + } + MappingRespImpl (PutMappingResponse response) { + this.fieldNames = new HashSet(); + } + @Override + public Set fieldNames() { + return new HashSet(this.fieldNames); // make a copy so data is never corrupted + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/ResponseWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/ResponseWrapper.java deleted file mode 100644 index 405a224..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/ResponseWrapper.java +++ /dev/null @@ -1,23 +0,0 @@ -package gov.nasa.pds.registry.common.connection.aws; - -import org.apache.http.HttpEntity; -import org.apache.http.StatusLine; -import gov.nasa.pds.registry.common.Response; - -public class ResponseWrapper implements Response { - @Override - public HttpEntity getEntity() { - // TODO Auto-generated method stub - return null; - } - @Override - public StatusLine getStatusLine() { - // TODO Auto-generated method stub - return null; - } - @Override - public void printWarnings() { - // TODO Auto-generated method stub - - } -} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java index 617d30c..4601296 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java @@ -1,13 +1,24 @@ package gov.nasa.pds.registry.common.connection.aws; import java.io.IOException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import javax.net.ssl.SSLContext; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.ssl.SSLContextBuilder; import org.opensearch.client.opensearch.OpenSearchClient; -import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.core.GetResponse; import org.opensearch.client.opensearch.indices.CreateIndexRequest; import org.opensearch.client.opensearch.indices.DeleteIndexRequest; import org.opensearch.client.opensearch.indices.ExistsRequest; +import org.opensearch.client.transport.OpenSearchTransport; import org.opensearch.client.transport.aws.AwsSdk2Transport; import org.opensearch.client.transport.aws.AwsSdk2TransportOptions; +import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.Request.Bulk; import gov.nasa.pds.registry.common.Request.Count; @@ -39,7 +50,34 @@ public RestClientWrapper(ConnectionFactory conFact, boolean isServerless) { ) ); } else { - this.client = null; + OpenSearchClient localClient = null; + try { + SSLContext sslcontext = SSLContextBuilder + .create() + .loadTrustMaterial((chains, authType) -> true) + .build(); + final ApacheHttpClient5TransportBuilder builder = ApacheHttpClient5TransportBuilder.builder(conFact.getHost5()); + builder.setHttpClientConfigCallback(httpClientBuilder -> { + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() + .setSslContext(sslcontext) + .build(); + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder + .create() + .setTlsStrategy(tlsStrategy) + .build(); + return httpClientBuilder + .setDefaultCredentialsProvider(conFact.getCredentials5()) + .setConnectionManager(connectionManager); + }); + final OpenSearchTransport transport = ApacheHttpClient5TransportBuilder.builder(conFact.getHost5()).build(); + localClient = new OpenSearchClient(transport); + } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + finally { + this.client = localClient; + } } } @Override @@ -76,40 +114,35 @@ public Setting createSettingRequest() { return new SettingImpl(); } @Override - public Response create(String indexName, String configAsJson) throws IOException, ResponseException { - // FIXME: need to change JSON into V2 mapping - this.client.indices().create(new CreateIndexRequest.Builder().index(indexName).build()); - return null; + public Response.CreatedIndex create(String indexName, String configAsJson) throws IOException, ResponseException { + return new CreateIndexRespWrap(this.client.indices().create(new CreateIndexRequest.Builder().index(indexName).build())); } @Override - public Response delete(String indexName) throws IOException, ResponseException { + public void delete(String indexName) throws IOException, ResponseException { this.client.indices().delete(new DeleteIndexRequest.Builder().index(indexName).build()); - return null; } @Override - public Response exists(String indexName) throws IOException, ResponseException { - this.client.indices().exists(new ExistsRequest.Builder().index(indexName).build()); - return null; + public boolean exists(String indexName) throws IOException, ResponseException { + return this.client.indices().exists(new ExistsRequest.Builder().index(indexName).build()).value(); } @Override - public Response performRequest(Bulk request) throws IOException, ResponseException { - BulkResponse resp = this.client.bulk(((BulkImpl)request).craftsman.build()); - return null; + public Response.Bulk performRequest(Bulk request) throws IOException, ResponseException { + return new BulkRespWrap(this.client.bulk(((BulkImpl)request).craftsman.build())); } @Override - public Response performRequest(Count request) throws IOException, ResponseException { - // TODO Auto-generated method stub - return null; + public long performRequest(Count request) throws IOException, ResponseException { + return this.client.count(((CountImpl)request).craftsman.build()).count(); } @Override public Response performRequest(Get request) throws IOException, ResponseException { - // TODO Auto-generated method stub + GetResponse x = this.client.get(((GetImpl)request).craftsman.build(), Object.class); return null; } @Override - public Response performRequest(Mapping request) throws IOException, ResponseException { - // TODO Auto-generated method stub - return null; + public Response.Mapping performRequest(Mapping request) throws IOException, ResponseException { + MappingImpl req = (MappingImpl)request; + return req.isGet ? new MappingRespImpl(this.client.indices().getMapping(req.craftsman_get.build())) : + new MappingRespImpl(this.client.indices().putMapping(req.craftsman_set.build())); } @Override public Response performRequest(Search request) throws IOException, ResponseException { @@ -117,8 +150,7 @@ public Response performRequest(Search request) throws IOException, ResponseExcep return null; } @Override - public Response performRequest(Setting request) throws IOException, ResponseException { - // TODO Auto-generated method stub - return null; + public Response.Settings performRequest(Setting request) throws IOException, ResponseException { + return new SettingRespImpl(this.client.indices().getSettings(((SettingImpl)request).craftsman.build())); } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java index fe0945a..dd5630c 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java @@ -15,7 +15,7 @@ import org.opensearch.client.opensearch.core.search.SourceFilter; import gov.nasa.pds.registry.common.Request.Search; -public class SearchImpl implements Search { +class SearchImpl implements Search { final SearchRequest.Builder craftsman = new SearchRequest.Builder(); private void buildIds (Collection lids, boolean alt) { SourceConfig.Builder journeyman = new SourceConfig.Builder(); diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java index e1034f2..37b7146 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java @@ -3,7 +3,7 @@ import org.opensearch.client.opensearch.indices.GetIndicesSettingsRequest; import gov.nasa.pds.registry.common.Request.Setting; -public class SettingImpl implements Setting { +class SettingImpl implements Setting { final GetIndicesSettingsRequest.Builder craftsman = new GetIndicesSettingsRequest.Builder(); @Override public Setting setIndex(String name) { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingRespImpl.java new file mode 100644 index 0000000..cbf9911 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingRespImpl.java @@ -0,0 +1,29 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.opensearch.client.opensearch.indices.GetIndicesSettingsResponse; +import org.opensearch.client.opensearch.indices.IndexState; +import gov.nasa.pds.registry.common.Response; + +class SettingRespImpl implements Response.Settings { + final private int replicas; + final private int shards; + SettingRespImpl (GetIndicesSettingsResponse response) { + int r = -1, s = -1; + for (IndexState state : response.result().values()) { + // should only ever be one + r = Integer.valueOf(state.settings().numberOfReplicas()); + s = Integer.valueOf(state.settings().numberOfShards()); + } + this.replicas = r; + this.shards = s; + } + @Override + public int replicas() { + return this.replicas; + } + + @Override + public int shards() { + return this.shards; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java new file mode 100644 index 0000000..56126b2 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java @@ -0,0 +1,90 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.util.List; +import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import com.google.gson.Gson; +import gov.nasa.pds.registry.common.Response; + +class BulkRespImpl implements Response.Bulk { + final private Logger log; + BulkRespImpl (org.elasticsearch.client.Response response) { + this.log = LogManager.getLogger(this.getClass()); + this.parse(response.toString()); + } + private int parse (String resp) { + int numErrors = 0; + try + { + // TODO: Use streaming parser. Stop parsing if there are no errors. + // Parse JSON response + Gson gson = new Gson(); + Map json = (Map)gson.fromJson(resp, Object.class); + Boolean hasErrors = (Boolean)json.get("errors"); + if(hasErrors) + { + List list = (List)json.get("items"); + + // List size = batch size (one item per document) + for(Object item: list) + { + Map action = (Map)((Map)item).get("index"); + if(action == null) + { + action = (Map)((Map)item).get("create"); + if(action != null) + { + String status = String.valueOf(action.get("status")); + // For "create" requests status=409 means that the record already exists. + // It is not an error. We use "create" action to insert records which don't exist + // and keep existing records as is. We do this when loading an old LDD and more + // recent version of the LDD is already loaded. + // NOTE: Gson JSON parser stores numbers as floats. + // The string value is usually "409.0". Can it be something else? + if(status.startsWith("409")) + { + // Increment to properly report number of processed records. + numErrors++; + continue; + } + } + } + if(action == null) continue; + + String id = (String)action.get("_id"); + Map error = (Map)action.get("error"); + if(error != null) + { + String message = (String)error.get("reason"); + String sanitizedLidvid = id.replace('\r', ' ').replace('\n', ' '); // protect vs log spoofing see code-scanning alert #37 + String sanitizedMessage = message.replace('\r', ' ').replace('\n', ' '); // protect vs log spoofing + log.error("LIDVID = " + sanitizedLidvid + ", Message = " + sanitizedMessage); + numErrors++; + } + } + } + + return numErrors; + } + catch(Exception ex) + { + return 0; + } + } + @Override + public boolean errors() { + // TODO Auto-generated method stub + return false; + } + @Override + public List items() { + // TODO Auto-generated method stub + return null; + } + @Override + public long took() { + // TODO Auto-generated method stub + return 0; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/JsonHelper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/JsonHelper.java index 57ed6b4..bad1610 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/JsonHelper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/JsonHelper.java @@ -1,8 +1,13 @@ package gov.nasa.pds.registry.common.connection.es; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.StringWriter; import java.util.Collection; +import org.apache.http.HttpEntity; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; import com.google.gson.stream.JsonWriter; import gov.nasa.pds.registry.common.meta.Metadata; import gov.nasa.pds.registry.common.util.Tuple; @@ -360,4 +365,24 @@ static String buildUpdateSchemaRequest(Collection fields) { return strWriter.toString(); } + static long findCount (HttpEntity entity) { + long count = 0; + try (InputStream is = entity.getContent()) { + JsonReader rd = new JsonReader(new InputStreamReader(is)); + rd.beginObject(); + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if("count".equals(name)) { + count = rd.nextInt(); + break; + } else { + rd.skipValue(); + } + } + rd.endObject(); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("some weird JSON serialization problem that should not be happening"); + } + return count; + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingImpl.java index 6d9aae4..a98d7a7 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingImpl.java @@ -5,7 +5,7 @@ import gov.nasa.pds.registry.common.util.Tuple; class MappingImpl implements Mapping { - private String index; + String index; String json = null; @Override public Mapping buildUpdateFieldSchema(Collection pairs) { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingRespImpl.java new file mode 100644 index 0000000..2ac8c4f --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingRespImpl.java @@ -0,0 +1,17 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.util.HashSet; +import java.util.Set; +import gov.nasa.pds.registry.common.Response; + +class MappingRespImpl implements Response.Mapping { + final private Set fieldNames; + MappingRespImpl (org.elasticsearch.client.Response response, String index) { + this.fieldNames = new MappingsParser(index).parse(response.getEntity()); + } + @Override + public Set fieldNames() { + return new HashSet(fieldNames); + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/MappingsParser.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingsParser.java similarity index 80% rename from src/main/java/gov/nasa/pds/registry/common/es/dao/schema/MappingsParser.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/MappingsParser.java index d5c2d7a..50e3161 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/MappingsParser.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingsParser.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.dao.schema; +package gov.nasa.pds.registry.common.connection.es; import java.io.IOException; import java.io.InputStream; @@ -18,7 +18,7 @@ * * @author karpenko */ -public class MappingsParser +class MappingsParser { private String indexName; private JsonReader rd; @@ -41,35 +41,27 @@ public MappingsParser(String indexName) * @return a collection of field names from a given Elasticsearch index. * @throws IOException an exception */ - public Set parse(HttpEntity entity) throws IOException + public Set parse(HttpEntity entity) { - InputStream is = entity.getContent(); + fields = new TreeSet<>(); + try (InputStream is = entity.getContent()) { rd = new JsonReader(new InputStreamReader(is)); - - fields = new TreeSet<>(); - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - // Usually there is only one root element = index name. - String name = rd.nextName(); - if(indexName.equals(name)) - { - parseMappings(); - } - // Usually we should not go here. - else - { - rd.skipValue(); - } + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + // Usually there is only one root element = index name. + String name = rd.nextName(); + if(indexName.equals(name)) { + parseMappings(); + } else { + rd.skipValue(); + } } - rd.endObject(); - rd.close(); - - return fields; + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("some strange exception because we should never get here"); + } + return fields; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseNotImplYet.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseNotImplYet.java new file mode 100644 index 0000000..cd1a12b --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseNotImplYet.java @@ -0,0 +1,18 @@ +package gov.nasa.pds.registry.common.connection.es; + +import gov.nasa.pds.registry.common.Response; + +class ResponseNotImplYet implements Response.CreatedIndex { + @Override + public boolean acknowledge() { + throw new RuntimeException("This method needs to be implemented for old style SDK"); + } + @Override + public boolean acknowledgeShards() { + throw new RuntimeException("This method needs to be implemented for old style SDK"); + } + @Override + public String getIndex() { + throw new RuntimeException("This method needs to be implemented for old style SDK"); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java index 5636100..2c5f940 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java @@ -1,6 +1,7 @@ package gov.nasa.pds.registry.common.connection.es; import java.io.IOException; +import java.util.List; import org.elasticsearch.client.Request; import org.elasticsearch.client.RestClientBuilder; import gov.nasa.pds.registry.common.ConnectionFactory; @@ -39,15 +40,25 @@ public RestClientWrapper(ConnectionFactory conFact) throws Exception bld.setRequestConfigCallback(reqCB); this.real_client = bld.build(); } - private Response performRequest(String endpoint, String json, String method) throws IOException,ResponseException { + private org.elasticsearch.client.Response performRequest(String endpoint, String json, String method) throws IOException,ResponseException { try { Request request = new Request(method, endpoint); if (json != null) request.setJsonEntity(json); - return new ResponseWrapper(this.real_client.performRequest(request)); + return this.real_client.performRequest(request); } catch (org.elasticsearch.client.ResponseException e) { throw new ResponseExceptionWrapper(e); } } + private void printWarnings(org.elasticsearch.client.Response resp) { + List warnings = resp.getWarnings(); + if(warnings != null) + { + for(String warn: warnings) + { + System.out.println("[WARN] " + warn); + } + } + } @Override public void close() throws IOException { this.real_client.close(); @@ -81,40 +92,42 @@ public Setting createSettingRequest() { return new SettingImpl(); } @Override - public Response create (String indexName, String configAsJSON) throws IOException,ResponseException { - return this.performRequest("/" + indexName, configAsJSON, "PUT"); + public Response.CreatedIndex create (String indexName, String configAsJSON) throws IOException,ResponseException { + this.printWarnings(this.performRequest("/" + indexName, configAsJSON, "PUT")); + return new ResponseNotImplYet(); } @Override - public Response delete (String indexName) throws IOException,ResponseException { - return this.performRequest(indexName, null, "DELETE"); + public void delete (String indexName) throws IOException,ResponseException { + this.printWarnings(this.performRequest(indexName, null, "DELETE")); } @Override - public Response exists (String indexName) throws IOException,ResponseException { - return this.performRequest ("/" + indexName, null, "HEAD"); + public boolean exists (String indexName) throws IOException,ResponseException { + return this.performRequest ("/" + indexName, null, "HEAD").getStatusLine().getStatusCode() == 200; } @Override - public Response performRequest(Bulk request) throws IOException,ResponseException { - return this.performRequest(request.toString(), ((BulkImpl)request).json, "POST"); + public Response.Bulk performRequest(Bulk request) throws IOException,ResponseException { + return new BulkRespImpl(this.performRequest(request.toString(), ((BulkImpl)request).json, "POST")); } @Override - public Response performRequest(Count request) throws IOException,ResponseException { - return this.performRequest(request.toString(), null, "GET"); + public long performRequest(Count request) throws IOException,ResponseException { + return JsonHelper.findCount(this.performRequest(request.toString(), null, "GET").getEntity()); } @Override public Response performRequest(Get request) throws IOException,ResponseException { return this.performRequest(request.toString(), ((GetImpl)request).json, "GET"); } @Override - public Response performRequest(Mapping request) throws IOException,ResponseException { + public Response.Mapping performRequest(Mapping request) throws IOException,ResponseException { + String index = ((MappingImpl)request).index; String json = ((MappingImpl)request).json; - return this.performRequest(request.toString(), json, json == null ? "GET" : "PUT"); + return new MappingRespImpl(this.performRequest(request.toString(), json, json == null ? "GET" : "PUT"), index); } @Override public Response performRequest(Search request) throws IOException,ResponseException { return this.performRequest(request.toString(), ((SearchImpl)request).json, "GET"); } @Override - public Response performRequest(Setting request) throws IOException,ResponseException { - return this.performRequest(request.toString(), null, "GET"); + public Response.Settings performRequest(Setting request) throws IOException,ResponseException { + return new SettingsRespImpl(this.performRequest(request.toString(), null, "GET")); } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SettingsRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SettingsRespImpl.java new file mode 100644 index 0000000..50fcb7b --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SettingsRespImpl.java @@ -0,0 +1,102 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import org.apache.http.HttpEntity; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import gov.nasa.pds.registry.common.Response; + +/** + * Parse Elasticsearch "/index_name/_settings" response. + * @author karpenko + */ +class SettingsRespImpl implements Response.Settings +{ + private int replicas; + private int shards; + /** + * Constructor + */ + public SettingsRespImpl(org.elasticsearch.client.Response response){ + this.parse(response.getEntity()); + } + + + /** + * Parse Elasticsearch response + * @param resp Elasticsearch response + * @return index settings + * @throws Exception an exception + */ + public void parse(HttpEntity entity) + { + try (InputStream is = entity.getContent()) { + JsonReader rd = new JsonReader(new InputStreamReader(is)); + rd.beginObject(); // root + rd.nextName(); // index name + rd.beginObject(); + rd.nextName(); // "settings" + rd.beginObject(); + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) + { + String name = rd.nextName(); + if("index".equals(name)) + { + parseIndex(rd); + } + else + { + rd.skipValue(); + } + } + + rd.endObject(); + rd.endObject(); + rd.endObject(); + + rd.close(); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("some weird json error because should never get here"); + } + } + private void parseIndex(JsonReader rd) throws IOException + { + if(rd == null) throw new IllegalArgumentException("JsonReader is null"); + + rd.beginObject(); + + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) + { + String name = rd.nextName(); + if("number_of_shards".equals(name)) + { + this.shards = rd.nextInt(); + } + else if("number_of_replicas".equals(name)) + { + this.replicas = rd.nextInt(); + } + else + { + rd.skipValue(); + } + } + + rd.endObject(); + } + + + @Override + public int replicas() { + return this.replicas; + } + + + @Override + public int shards() { + return this.shards; + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java index 24ca799..25a31fb 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java @@ -1,6 +1,7 @@ package gov.nasa.pds.registry.common.es.dao; import java.io.IOException; +import java.io.InputStreamReader; import java.io.Reader; import org.apache.logging.log4j.LogManager; @@ -8,7 +9,7 @@ import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; - +import gov.nasa.pds.registry.common.Response; import gov.nasa.pds.registry.common.util.CloseUtils; /** @@ -27,13 +28,30 @@ public BulkResponseParser() log = LogManager.getLogger(this.getClass()); } + public void parse(Response.Bulk response) { + /* FIXME + * This should log erros as parse(Reader reader) does. + * Just put this empty implementation in place to move + * forward with SDK v2 but leave this compile without + * doing any actual work. + * + * old invocations was like this: + * + is = resp.getEntity().getContent(); + rd = new InputStreamReader(is); + + BulkResponseParser parser = new BulkResponseParser(); + parser.parse(rd); + + */ + } /** * Parse JSON string * @param reader bulk API response stream * @throws IOException an exception */ - public void parse(Reader reader) throws IOException + protected void parse(Reader reader) throws IOException { if(reader == null) return; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java index f5be89e..25f3b70 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java @@ -8,7 +8,6 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -16,7 +15,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.google.gson.Gson; import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.Request; import gov.nasa.pds.registry.common.Response; @@ -236,13 +234,9 @@ public int loadBatch(List data, Set errorLidvids, int retries) t bulk.add(data.get(index), data.get(++index)); } // FIXME: response has changed so need to debug this - Response response = this.conFactory.createRestClient().performRequest(bulk); - // Read Elasticsearch response. - String respJson = response.toString(); - log.debug(respJson); - + Response.Bulk response = this.conFactory.createRestClient().performRequest(bulk); // Check for Elasticsearch errors. - int failedCount = processErrors(respJson, errorLidvids); + int failedCount = processErrors(response, errorLidvids); // Calculate number of successfully saved records // NOTE: data list has two lines per record (primary key + data) int loadedCount = data.size() / 2 - failedCount; @@ -290,69 +284,24 @@ public int loadBatch(List data) throws Exception return loadBatch(data, null); } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private int processErrors(String resp, Set errorLidvids) - { - int numErrors = 0; - - try - { - // TODO: Use streaming parser. Stop parsing if there are no errors. - // Parse JSON response - Gson gson = new Gson(); - Map json = (Map)gson.fromJson(resp, Object.class); - - Boolean hasErrors = (Boolean)json.get("errors"); - if(hasErrors) - { - List list = (List)json.get("items"); - - // List size = batch size (one item per document) - for(Object item: list) - { - Map action = (Map)((Map)item).get("index"); - if(action == null) - { - action = (Map)((Map)item).get("create"); - if(action != null) - { - String status = String.valueOf(action.get("status")); - // For "create" requests status=409 means that the record already exists. - // It is not an error. We use "create" action to insert records which don't exist - // and keep existing records as is. We do this when loading an old LDD and more - // recent version of the LDD is already loaded. - // NOTE: Gson JSON parser stores numbers as floats. - // The string value is usually "409.0". Can it be something else? - if(status.startsWith("409")) - { - // Increment to properly report number of processed records. - numErrors++; - continue; - } - } - } - if(action == null) continue; - - String id = (String)action.get("_id"); - Map error = (Map)action.get("error"); - if(error != null) - { - String message = (String)error.get("reason"); - String sanitizedLidvid = id.replace('\r', ' ').replace('\n', ' '); // protect vs log spoofing see code-scanning alert #37 - String sanitizedMessage = message.replace('\r', ' ').replace('\n', ' '); // protect vs log spoofing - log.error("LIDVID = " + sanitizedLidvid + ", Message = " + sanitizedMessage); - numErrors++; - if(errorLidvids != null) errorLidvids.add(id); - } - } + private int processErrors(Response.Bulk resp, Set errorLidvids) { + int numErrors = 0; + if (resp.errors()) { + for (Response.Bulk.Item item : resp.items()) { + if (item.error()) { + if (item.operation() == "create" && item.status() == 409) { + numErrors++; + } else { + String message = item.reason(); + String sanitizedLidvid = item.id().replace('\r', ' ').replace('\n', ' '); // protect vs log spoofing see code-scanning alert #37 + String sanitizedMessage = message.replace('\r', ' ').replace('\n', ' '); // protect vs log spoofing + log.error("LIDVID = " + sanitizedLidvid + ", Message = " + sanitizedMessage); + numErrors++; + if(errorLidvids != null) errorLidvids.add(item.id()); } - - return numErrors; - } - catch(Exception ex) - { - return 0; + } } + } + return numErrors; } } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java index acfbac9..a85266c 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java @@ -141,7 +141,7 @@ else if(type == 'S') try { - resp = client.performRequest(req); + return (int)client.performRequest(req); } catch(ResponseException ex) { @@ -157,37 +157,6 @@ else if(type == 'S') throw ex; } } - - InputStream is = null; - - try - { - is = resp.getEntity().getContent(); - JsonReader rd = new JsonReader(new InputStreamReader(is)); - - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("count".equals(name)) - { - return rd.nextInt(); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); - } - finally - { - CloseUtils.close(is); - } - - return 0; } @@ -277,24 +246,8 @@ public void updateArchiveStatus(Collection lidvids, String status) throw .buildUpdateStatus(lidvids, status) .setIndex(this.indexName); - Response resp = client.performRequest(req); - - // Check for Elasticsearch errors. - InputStream is = null; - InputStreamReader rd = null; - try - { - is = resp.getEntity().getContent(); - rd = new InputStreamReader(is); - - BulkResponseParser parser = new BulkResponseParser(); - parser.parse(rd); - } - finally - { - CloseUtils.close(rd); - CloseUtils.close(is); - } + Response .Bulk resp = client.performRequest(req); + new BulkResponseParser().parse(resp); } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java index e3ab18d..2ad13a0 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java @@ -3,7 +3,6 @@ import java.util.List; import java.util.Set; import gov.nasa.pds.registry.common.Request; -import gov.nasa.pds.registry.common.Response; import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.common.util.Tuple; @@ -40,10 +39,7 @@ public SchemaDao(RestClient client, String indexName) public Set getFieldNames() throws Exception { Request.Mapping req = client.createMappingRequest().setIndex(indexName); - Response resp = client.performRequest(req); - - MappingsParser parser = new MappingsParser(indexName); - return parser.parse(resp.getEntity()); + return client.performRequest(req).fieldNames(); } From 7fd42e5275e0e8494bc31d3317118b4c3f489cbd Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 16 Apr 2024 07:51:25 -0700 Subject: [PATCH 28/41] WIP All of the response interfaces have been defined and cleanly done for the older es. Need to update all of the aws to support the new responses. --- pom.xml | 6 +- .../registry/common/ConnectionFactory.java | 1 + .../common/EstablishConnectionFactory.java | 6 +- .../nasa/pds/registry/common/Response.java | 30 ++- .../registry/common/ResponseException.java | 3 +- .../nasa/pds/registry/common/RestClient.java | 4 +- .../common/connection/AuthContent.java | 3 +- .../common/connection/UseOpensearchSDK1.java | 2 +- .../common/connection/aws/BulkImpl.java | 3 +- .../common/connection/aws/MappingImpl.java | 29 +-- .../connection/aws/RestClientWrapper.java | 17 +- .../common/connection/es/BulkRespImpl.java | 2 + .../common/connection/es/EsUtils.java | 1 - .../es/ResponseExceptionWrapper.java | 7 +- .../common/connection/es/ResponseWrapper.java | 32 --- .../connection/es/RestClientWrapper.java | 8 +- .../es}/SearchResponseParser.java | 26 +- .../common/es/dao/BulkResponseParser.java | 3 +- .../pds/registry/common/es/dao/DaoUtils.java | 5 +- .../registry/common/es/dao/DataLoader.java | 4 +- .../pds/registry/common/es/dao/LidvidSet.java | 19 +- .../registry/common/es/dao/ProductDao.java | 246 +----------------- .../common/es/dao/dd/DataDictionaryDao.java | 180 +------------ 23 files changed, 115 insertions(+), 522 deletions(-) delete mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseWrapper.java rename src/main/java/gov/nasa/pds/registry/common/{util => connection/es}/SearchResponseParser.java (86%) diff --git a/pom.xml b/pom.xml index 06c06c0..2f3f09c 100644 --- a/pom.xml +++ b/pom.xml @@ -141,17 +141,17 @@ POSSIBILITY OF SUCH DAMAGE. org.opensearch.client opensearch-java - 2.9.1 + 2.10.0 software.amazon.awssdk opensearch - 2.25.21 + 2.25.31 software.amazon.awssdk apache-client - 2.25.21 + 2.25.31 diff --git a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java index a88ff85..6f380cb 100644 --- a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java @@ -4,6 +4,7 @@ import org.apache.http.client.CredentialsProvider; public interface ConnectionFactory { + public ConnectionFactory clone(); public RestClient createRestClient() throws Exception; public CredentialsProvider getCredentials(); public org.apache.hc.client5.http.auth.CredentialsProvider getCredentials5(); diff --git a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java index ca2f126..fce7b6f 100644 --- a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java @@ -22,9 +22,9 @@ private static synchronized ConnectionFactory from (String urlToRegistryConnecti RegistryConnectionContent conn = RegistryConnectionContent.from (new URL(urlToRegistryConnection)); if (conn.isDirectConnection()) { - if (conn.getServerUrl().getSdk().intValue() == 1) UseOpensearchSDK1.build(conn.getServerUrl(), auth).setIndexName(conn.getIndex()); - if (conn.getServerUrl().getSdk().intValue() == 2) UseOpensearchSDK2.build(conn.getServerUrl(), auth).setIndexName(conn.getIndex()); - throw new RuntimeException("The SDK version '" + String.valueOf(conn.getServerUrl().getSdk()) + "is not supported"); + if (conn.getServerUrl().getSdk().intValue() == 1) return UseOpensearchSDK1.build(conn.getServerUrl(), auth).setIndexName(conn.getIndex()); + if (conn.getServerUrl().getSdk().intValue() == 2) return UseOpensearchSDK2.build(conn.getServerUrl(), auth).setIndexName(conn.getIndex()); + throw new RuntimeException("The SDK version '" + String.valueOf(conn.getServerUrl().getSdk().intValue()) + "' is not supported"); } if (conn.isCognitoConnection()) return UseOpensearchSDK2.build(conn.getCognitoClientId(), auth).setIndexName(conn.getIndex()); throw new RuntimeException("New XML/Java choices in src/main/resources/registry_connection.xsd that are not handled."); diff --git a/src/main/java/gov/nasa/pds/registry/common/Response.java b/src/main/java/gov/nasa/pds/registry/common/Response.java index d9917a1..8cdb446 100644 --- a/src/main/java/gov/nasa/pds/registry/common/Response.java +++ b/src/main/java/gov/nasa/pds/registry/common/Response.java @@ -1,9 +1,14 @@ package gov.nasa.pds.registry.common; +import java.io.IOException; +import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Set; -import org.apache.http.HttpEntity; -import org.apache.http.StatusLine; +import gov.nasa.pds.registry.common.es.dao.dd.DataTypeNotFoundException; +import gov.nasa.pds.registry.common.es.dao.dd.LddInfo; +import gov.nasa.pds.registry.common.es.dao.dd.LddVersions; +import gov.nasa.pds.registry.common.util.Tuple; public interface Response { public interface Bulk { @@ -25,14 +30,29 @@ public interface CreatedIndex { public boolean acknowledgeShards(); public String getIndex(); } + public interface Get { + public interface IdSets { + public Set lids(); + public Set lidvids(); + } + public List dataTypes(boolean stringForMissing) throws IOException, DataTypeNotFoundException; + public IdSets ids(); // returns null if nothing is found in returned content + public String productClass(); // returns null if product class not in returned content + public List refs(); // returns null if nothing is found in returned content + } public interface Mapping { public Set fieldNames(); } + public interface Search { + public Map> altIds() throws UnsupportedOperationException, IOException; + public Set fields() throws UnsupportedOperationException, IOException; + public List latestLidvids(); // returns null if nothing is found in returned content + public LddVersions lddInfo() throws UnsupportedOperationException, IOException; + public List ldds() throws UnsupportedOperationException, IOException; + public Set nonExistingIds(Collection from_ids) throws UnsupportedOperationException, IOException; + } public interface Settings { public int replicas(); public int shards(); } - public HttpEntity getEntity(); - public StatusLine getStatusLine(); - public void printWarnings(); } diff --git a/src/main/java/gov/nasa/pds/registry/common/ResponseException.java b/src/main/java/gov/nasa/pds/registry/common/ResponseException.java index 905c709..d07b8fe 100644 --- a/src/main/java/gov/nasa/pds/registry/common/ResponseException.java +++ b/src/main/java/gov/nasa/pds/registry/common/ResponseException.java @@ -5,5 +5,6 @@ abstract public class ResponseException extends IOException { private static final long serialVersionUID = 8629769947735587642L; abstract public String extractErrorMessage(); - abstract public Response getResponse(); + abstract public int statusCode(); // -1 if not known + //abstract public Response getResponse(); } diff --git a/src/main/java/gov/nasa/pds/registry/common/RestClient.java b/src/main/java/gov/nasa/pds/registry/common/RestClient.java index 77fde98..73be161 100644 --- a/src/main/java/gov/nasa/pds/registry/common/RestClient.java +++ b/src/main/java/gov/nasa/pds/registry/common/RestClient.java @@ -16,8 +16,8 @@ public interface RestClient extends Closeable { public boolean exists (String indexName) throws IOException,ResponseException; public Response.Bulk performRequest(Request.Bulk request) throws IOException,ResponseException; public long performRequest(Request.Count request) throws IOException,ResponseException; - public Response performRequest(Request.Get request) throws IOException,ResponseException; + public Response.Get performRequest(Request.Get request) throws IOException,ResponseException; public Response.Mapping performRequest(Request.Mapping request) throws IOException,ResponseException; - public Response performRequest(Request.Search request) throws IOException,ResponseException; + public Response.Search performRequest(Request.Search request) throws IOException,ResponseException; public Response.Settings performRequest(Request.Setting request) throws IOException,ResponseException; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java index 2b0203d..39574ac 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java @@ -67,7 +67,8 @@ public CredentialsProvider getCredentials(HttpHost scope) { public org.apache.hc.client5.http.auth.CredentialsProvider getCredentials5(org.apache.hc.core5.http.HttpHost scope) { org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider creds = new org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider(); - creds.setCredentials(new org.apache.hc.client5.http.auth.AuthScope(scope), + //creds.setCredentials(new org.apache.hc.client5.http.auth.AuthScope(scope), + creds.setCredentials(new org.apache.hc.client5.http.auth.AuthScope(null,null,-1,null,null), // ANY new org.apache.hc.client5.http.auth.UsernamePasswordCredentials(this.getUser(), this.getPassword().toCharArray())); return creds; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java index ae8d57b..e6c6156 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java @@ -10,7 +10,7 @@ import gov.nasa.pds.registry.common.connection.config.DirectType; import gov.nasa.pds.registry.common.connection.es.RestClientWrapper; -public class UseOpensearchSDK1 implements Cloneable, ConnectionFactory { +public class UseOpensearchSDK1 implements ConnectionFactory { final private boolean veryTrusting; final private AuthContent auth; final private HttpHost host; diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java index 803f9c9..31ff4da 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java @@ -17,6 +17,7 @@ class BulkImpl implements Bulk { final BulkRequest.Builder craftsman = new BulkRequest.Builder(); + @SuppressWarnings("unchecked") @Override public void add(String statement, String document) { BulkOperation.Builder journeyman = new BulkOperation.Builder(); @@ -48,7 +49,7 @@ public void add(String statement, String document) { } journeyman.index(apprentice.build()); } else if (cmd.containsKey("update")) { - doc = ((Map)doc).get("doc"); + doc = ((Map)doc).get("doc"); UpdateOperation.Builder apprentice = new UpdateOperation.Builder().document(doc); params = cmd.get("update"); if (params.containsKey("_id")) { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java index 1a4c9d6..92d429e 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java @@ -2,13 +2,7 @@ import java.util.Collection; import java.util.HashMap; -import org.opensearch.client.opensearch._types.mapping.BooleanProperty; -import org.opensearch.client.opensearch._types.mapping.DoubleNumberProperty; -import org.opensearch.client.opensearch._types.mapping.FloatNumberProperty; -import org.opensearch.client.opensearch._types.mapping.IntegerNumberProperty; -import org.opensearch.client.opensearch._types.mapping.KeywordProperty; import org.opensearch.client.opensearch._types.mapping.Property; -import org.opensearch.client.opensearch._types.mapping.TextProperty; import org.opensearch.client.opensearch.indices.GetMappingRequest; import org.opensearch.client.opensearch.indices.PutMappingRequest; import gov.nasa.pds.registry.common.Request.Mapping; @@ -25,28 +19,7 @@ public Mapping buildUpdateFieldSchema(Collection pairs) { Property.Builder journeyman = new Property.Builder(); String fieldName = t.item1; String fieldType = t.item2; - switch (fieldType) { - case "boolean": - journeyman.boolean_(new BooleanProperty.Builder().build()); - break; - case "double": - journeyman.double_(new DoubleNumberProperty.Builder().build()); - break; - case "float": - journeyman.float_(new FloatNumberProperty.Builder().build()); - break; - case "integer": - journeyman.integer(new IntegerNumberProperty.Builder().build()); - break; - case "keyword": - journeyman.keyword(new KeywordProperty.Builder().build()); - break; - case "text": - journeyman.text(new TextProperty.Builder().build()); - break; - default: - throw new RuntimeException("Cannot map type '" + fieldType + "' yet. Please review code and fix."); - } + PropertyHelper.setType(journeyman, fieldType); mapping.put(fieldName, journeyman.build()); } this.craftsman_set.properties(mapping); diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java index 4601296..b88c25a 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java @@ -11,11 +11,9 @@ import org.apache.hc.core5.http.nio.ssl.TlsStrategy; import org.apache.hc.core5.ssl.SSLContextBuilder; import org.opensearch.client.opensearch.OpenSearchClient; -import org.opensearch.client.opensearch.core.GetResponse; import org.opensearch.client.opensearch.indices.CreateIndexRequest; import org.opensearch.client.opensearch.indices.DeleteIndexRequest; import org.opensearch.client.opensearch.indices.ExistsRequest; -import org.opensearch.client.transport.OpenSearchTransport; import org.opensearch.client.transport.aws.AwsSdk2Transport; import org.opensearch.client.transport.aws.AwsSdk2TransportOptions; import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; @@ -69,8 +67,7 @@ public RestClientWrapper(ConnectionFactory conFact, boolean isServerless) { .setDefaultCredentialsProvider(conFact.getCredentials5()) .setConnectionManager(connectionManager); }); - final OpenSearchTransport transport = ApacheHttpClient5TransportBuilder.builder(conFact.getHost5()).build(); - localClient = new OpenSearchClient(transport); + localClient = new OpenSearchClient(builder.build()); } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -115,7 +112,7 @@ public Setting createSettingRequest() { } @Override public Response.CreatedIndex create(String indexName, String configAsJson) throws IOException, ResponseException { - return new CreateIndexRespWrap(this.client.indices().create(new CreateIndexRequest.Builder().index(indexName).build())); + return new CreateIndexRespWrap(this.client.indices().create(CreateIndexConfigWrap.update(new CreateIndexRequest.Builder(), configAsJson).index(indexName).build())); } @Override public void delete(String indexName) throws IOException, ResponseException { @@ -134,9 +131,8 @@ public long performRequest(Count request) throws IOException, ResponseException return this.client.count(((CountImpl)request).craftsman.build()).count(); } @Override - public Response performRequest(Get request) throws IOException, ResponseException { - GetResponse x = this.client.get(((GetImpl)request).craftsman.build(), Object.class); - return null; + public Response.Get performRequest(Get request) throws IOException, ResponseException { + return new GetRespWrap(this.client.get(((GetImpl)request).craftsman.build(), Object.class)); } @Override public Response.Mapping performRequest(Mapping request) throws IOException, ResponseException { @@ -145,9 +141,8 @@ public Response.Mapping performRequest(Mapping request) throws IOException, Resp new MappingRespImpl(this.client.indices().putMapping(req.craftsman_set.build())); } @Override - public Response performRequest(Search request) throws IOException, ResponseException { - // TODO Auto-generated method stub - return null; + public Response.Search performRequest(Search request) throws IOException, ResponseException { + return new SearchRespWrap(this.client.search(((SearchImpl)request).craftsman.build(), Object.class)); } @Override public Response.Settings performRequest(Setting request) throws IOException, ResponseException { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java index 56126b2..d0d91a3 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java @@ -13,6 +13,7 @@ class BulkRespImpl implements Response.Bulk { this.log = LogManager.getLogger(this.getClass()); this.parse(response.toString()); } + @SuppressWarnings("rawtypes") // necessary evil to manipulate heterogenous structures private int parse (String resp) { int numErrors = 0; try @@ -24,6 +25,7 @@ private int parse (String resp) { Boolean hasErrors = (Boolean)json.get("errors"); if(hasErrors) { + @SuppressWarnings("unchecked") List list = (List)json.get("items"); // List size = batch size (one item per document) diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java index 0407f9d..7e4ebf4 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java @@ -6,7 +6,6 @@ import org.apache.http.HttpHost; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import gov.nasa.pds.registry.common.util.SearchResponseParser; /** * Elasticsearch utility methods. diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java index fcf1824..e08e4ba 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java @@ -2,9 +2,7 @@ import java.io.PrintStream; import java.io.PrintWriter; -import gov.nasa.pds.registry.common.Response; import gov.nasa.pds.registry.common.ResponseException; -import gov.nasa.pds.registry.common.util.SearchResponseParser; final class ResponseExceptionWrapper extends ResponseException { private static final long serialVersionUID = -5116172984798822803L; @@ -57,9 +55,14 @@ public void setStackTrace(StackTraceElement[] stackTrace) { if (this.real_exception == null) super.setStackTrace(stackTrace); else this.real_exception.setStackTrace(stackTrace); } @Override + public int statusCode() { + return this.real_exception.getResponse().getStatusLine().getStatusCode(); + } + /* public Response getResponse() { return new ResponseWrapper(this.real_exception.getResponse()); } + */ @Override public String extractErrorMessage() { String msg = this.real_exception.getMessage(); diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseWrapper.java deleted file mode 100644 index e54693d..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseWrapper.java +++ /dev/null @@ -1,32 +0,0 @@ -package gov.nasa.pds.registry.common.connection.es; - -import java.util.List; -import org.apache.http.HttpEntity; -import org.apache.http.StatusLine; -import gov.nasa.pds.registry.common.Response; - -class ResponseWrapper implements Response { - final org.elasticsearch.client.Response real_response; - ResponseWrapper (org.elasticsearch.client.Response real_response) { - this.real_response = real_response; - } - @Override - public HttpEntity getEntity() { - return this.real_response.getEntity(); - } - @Override - public StatusLine getStatusLine() { - return this.real_response.getStatusLine(); - } - @Override - public void printWarnings() { - List warnings = this.real_response.getWarnings(); - if(warnings != null) - { - for(String warn: warnings) - { - System.out.println("[WARN] " + warn); - } - } - } -} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java index 2c5f940..6b24ffc 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java @@ -113,8 +113,8 @@ public long performRequest(Count request) throws IOException,ResponseException { return JsonHelper.findCount(this.performRequest(request.toString(), null, "GET").getEntity()); } @Override - public Response performRequest(Get request) throws IOException,ResponseException { - return this.performRequest(request.toString(), ((GetImpl)request).json, "GET"); + public Response.Get performRequest(Get request) throws IOException,ResponseException { + return new GetRespImpl(this.performRequest(request.toString(), ((GetImpl)request).json, "GET")); } @Override public Response.Mapping performRequest(Mapping request) throws IOException,ResponseException { @@ -123,8 +123,8 @@ public Response.Mapping performRequest(Mapping request) throws IOException,Respo return new MappingRespImpl(this.performRequest(request.toString(), json, json == null ? "GET" : "PUT"), index); } @Override - public Response performRequest(Search request) throws IOException,ResponseException { - return this.performRequest(request.toString(), ((SearchImpl)request).json, "GET"); + public Response.Search performRequest(Search request) throws IOException,ResponseException { + return new SearchRespImpl(this.performRequest(request.toString(), ((SearchImpl)request).json, "GET")); } @Override public Response.Settings performRequest(Setting request) throws IOException,ResponseException { diff --git a/src/main/java/gov/nasa/pds/registry/common/util/SearchResponseParser.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchResponseParser.java similarity index 86% rename from src/main/java/gov/nasa/pds/registry/common/util/SearchResponseParser.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/SearchResponseParser.java index b1f32be..2c2dc76 100644 --- a/src/main/java/gov/nasa/pds/registry/common/util/SearchResponseParser.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchResponseParser.java @@ -1,14 +1,16 @@ -package gov.nasa.pds.registry.common.util; +package gov.nasa.pds.registry.common.connection.es; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.List; import java.util.Map; - +import org.elasticsearch.client.Response; import com.google.gson.Gson; +import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; -import gov.nasa.pds.registry.common.Response; /** @@ -16,7 +18,7 @@ * * @author karpenko */ -public class SearchResponseParser +class SearchResponseParser { /** * Inner callback interface @@ -30,7 +32,7 @@ public static interface Callback * @param rec Parsed content of "_source" field. Usually it is a field name-value map. * @throws Exception an exception */ - public void onRecord(String id, Object rec) throws Exception; + public void onRecord(String id, Object rec); } @@ -43,7 +45,7 @@ public static interface Callback /** * Constructor */ - public SearchResponseParser() + SearchResponseParser() { } @@ -72,9 +74,11 @@ public int getNumDocs() * Parse response. Callback.onRecord() will be called for each record. * @param resp Elasticsearch rest client's response object. * @param cb Callback interface. + * @throws IOException + * @throws UnsupportedOperationException * @throws Exception an exception */ - public void parseResponse(Response resp, Callback cb) throws Exception + public void parseResponse(Response resp, Callback cb) throws UnsupportedOperationException, IOException { if(cb == null) throw new IllegalArgumentException("Callback is null"); this.cb = cb; @@ -109,9 +113,10 @@ public void parseResponse(Response resp, Callback cb) throws Exception /** * Parse "hits" array in JSON response. * @param rd JSON reader + * @throws IOException * @throws Exception an exception */ - private void parseHits(JsonReader rd) throws Exception + private void parseHits(JsonReader rd) throws IOException { rd.beginObject(); @@ -140,9 +145,12 @@ private void parseHits(JsonReader rd) throws Exception /** * Parse a hit from "hits" array in JSON response. * @param rd JSON reader + * @throws IOException + * @throws JsonSyntaxException + * @throws JsonIOException * @throws Exception an exception */ - private void parseHit(JsonReader rd) throws Exception + private void parseHit(JsonReader rd) throws JsonIOException, JsonSyntaxException, IOException { Object src = null; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java index 25a31fb..fb52090 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java @@ -1,7 +1,6 @@ package gov.nasa.pds.registry.common.es.dao; import java.io.IOException; -import java.io.InputStreamReader; import java.io.Reader; import org.apache.logging.log4j.LogManager; @@ -51,7 +50,7 @@ public void parse(Response.Bulk response) { * @param reader bulk API response stream * @throws IOException an exception */ - protected void parse(Reader reader) throws IOException + public void parse(Reader reader) throws IOException // needs to be public for testing { if(reader == null) return; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DaoUtils.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DaoUtils.java index 681f06d..daf0b32 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DaoUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DaoUtils.java @@ -1,6 +1,7 @@ package gov.nasa.pds.registry.common.es.dao; import java.io.BufferedReader; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; @@ -54,7 +55,7 @@ public static String getLastLine(InputStream is) - public static List parseList(JsonReader rd) throws Exception + public static List parseList(JsonReader rd) throws IOException { List list = new ArrayList<>(); @@ -71,7 +72,7 @@ public static List parseList(JsonReader rd) throws Exception } - public static Set parseSet(JsonReader rd) throws Exception + public static Set parseSet(JsonReader rd) throws IOException { Set set = new TreeSet<>(); diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java index 25f3b70..bdefcf3 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java @@ -225,11 +225,9 @@ public int loadBatch(List data, Set errorLidvids, int retries) t if(data == null || data.isEmpty()) return 0; if(data.size() % 2 != 0) throw new Exception("Data list size should be an even number."); - Request.Bulk bulk = null; - try { - bulk = this.conFactory.createRestClient().createBulkRequest().setRefresh(Request.Bulk.Refresh.WaitFor); + Request.Bulk bulk = this.conFactory.createRestClient().createBulkRequest().setRefresh(Request.Bulk.Refresh.WaitFor).setIndex(this.conFactory.getIndexName()); for (int index = 0 ; index < data.size() ; index++) { bulk.add(data.get(index), data.get(++index)); } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/LidvidSet.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/LidvidSet.java index cb6c33f..a1de85c 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/LidvidSet.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/LidvidSet.java @@ -1,9 +1,26 @@ package gov.nasa.pds.registry.common.es.dao; import java.util.Set; +import gov.nasa.pds.registry.common.Response; -public class LidvidSet +public class LidvidSet implements Response.Get.IdSets { public Set lidvids; public Set lids; + public LidvidSet(Set lids, Set lidvids) { + this.lids = lids; + this.lidvids = lidvids; + } + public LidvidSet(Response.Get.IdSets ids) { + this.lids = ids.lids(); + this.lidvids = ids.lidvids(); + } + @Override + public Set lids() { + return this.lids; + } + @Override + public Set lidvids() { + return this.lidvids; + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java index a85266c..ca22ef8 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java @@ -1,18 +1,12 @@ package gov.nasa.pds.registry.common.es.dao; -import java.io.InputStream; -import java.io.InputStreamReader; import java.net.URLEncoder; import java.util.Collection; import java.util.List; -import com.google.gson.stream.JsonReader; -import com.google.gson.stream.JsonToken; import gov.nasa.pds.registry.common.Request; import gov.nasa.pds.registry.common.Response; import gov.nasa.pds.registry.common.ResponseException; import gov.nasa.pds.registry.common.RestClient; -import gov.nasa.pds.registry.common.util.CloseUtils; -import gov.nasa.pds.registry.common.util.LidVidUtils; /** * Product data access object. @@ -50,16 +44,16 @@ public String getProductClass(String lidvid) throws Exception .includeField("product_class") .setId(lidvid) .setIndex(this.indexName); - Response resp = null; + Response.Get resp = null; try { resp = client.performRequest(req); + return resp.productClass(); } catch(ResponseException ex) { - resp = ex.getResponse(); - int code = resp.getStatusLine().getStatusCode(); + int code = ex.statusCode(); // Invalid LIDVID if(code == 404 || code == 405) { @@ -70,37 +64,6 @@ public String getProductClass(String lidvid) throws Exception throw ex; } } - - InputStream is = null; - - try - { - is = resp.getEntity().getContent(); - JsonReader rd = new JsonReader(new InputStreamReader(is)); - - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("_source".equals(name)) - { - return parseProductClassSource(rd); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); - } - finally - { - CloseUtils.close(is); - } - - return null; } @@ -137,16 +100,13 @@ else if(type == 'S') Request.Count req = client.createCountRequest() .setIndex(this.indexName + "-refs") .setQuery(query); - Response resp = null; - try { return (int)client.performRequest(req); } catch(ResponseException ex) { - resp = ex.getResponse(); - int code = resp.getStatusLine().getStatusCode(); + int code = ex.statusCode(); // Invalid LIDVID if(code == 404 || code == 405) { @@ -177,17 +137,14 @@ public List getRefs(String collectionLidVid, char type, int page) throws Request.Get req = client.createGetRequest() .includeField("product_lidvid") .setId(docId) - .setIndex(this.indexName + "-refs"); - Response resp = null; - + .setIndex(this.indexName + "-refs"); try { - resp = client.performRequest(req); + return client.performRequest(req).refs(); } catch(ResponseException ex) { - resp = ex.getResponse(); - int code = resp.getStatusLine().getStatusCode(); + int code = ex.statusCode(); // Invalid LIDVID if(code == 404 || code == 405) { @@ -198,37 +155,6 @@ public List getRefs(String collectionLidVid, char type, int page) throws throw ex; } } - - InputStream is = null; - - try - { - is = resp.getEntity().getContent(); - JsonReader rd = new JsonReader(new InputStreamReader(is)); - - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("_source".equals(name)) - { - return parseRefs(rd); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); - } - finally - { - CloseUtils.close(is); - } - - return null; } @@ -250,53 +176,6 @@ public void updateArchiveStatus(Collection lidvids, String status) throw new BulkResponseParser().parse(resp); } - - private static String parseProductClassSource(JsonReader rd) throws Exception - { - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("product_class".equals(name)) - { - return rd.nextString(); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); - - return null; - } - - - private static List parseRefs(JsonReader rd) throws Exception - { - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("product_lidvid".equals(name)) - { - return DaoUtils.parseList(rd); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); - - return null; - } - - /** * Get collection references of a bundle. References can be either LIDs, LIDVIDs or both. * @param bundleLidvid bundle LIDVID @@ -311,17 +190,14 @@ public LidvidSet getCollectionIds(String bundleLidvid) throws Exception .includeField("ref_lidvid_collection") .includeField("ref_lid_collection") .setId(bundleLidvid) - .setIndex(this.indexName); - Response resp = null; - + .setIndex(this.indexName); try { - resp = client.performRequest(req); + return new LidvidSet(client.performRequest(req).ids()); } catch(ResponseException ex) { - resp = ex.getResponse(); - int code = resp.getStatusLine().getStatusCode(); + int code = ex.statusCode(); // Invalid LIDVID if(code == 404 || code == 405) { @@ -332,83 +208,7 @@ public LidvidSet getCollectionIds(String bundleLidvid) throws Exception throw ex; } } - - LidvidSet collectionIds = null; - InputStream is = null; - - try - { - is = resp.getEntity().getContent(); - JsonReader rd = new JsonReader(new InputStreamReader(is)); - - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("_source".equals(name)) - { - collectionIds = parseCollectionIdsSource(rd); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); - } - finally - { - CloseUtils.close(is); - } - - if(collectionIds == null || collectionIds.lidvids == null - || collectionIds.lids == null) return collectionIds; - - // Harvest converts LIDVIDs to LIDs, so let's delete those converted LIDs. - for(String lidvid: collectionIds.lidvids) - { - String lid = LidVidUtils.lidvidToLid(lidvid); - if(lid != null) - { - collectionIds.lids.remove(lid); - } - } - - - return collectionIds; - } - - - private static LidvidSet parseCollectionIdsSource(JsonReader rd) throws Exception - { - LidvidSet ids = new LidvidSet(); - - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("ref_lid_collection".equals(name)) - { - ids.lids = DaoUtils.parseSet(rd); - } - else if("ref_lidvid_collection".equals(name)) - { - ids.lidvids = DaoUtils.parseSet(rd); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); - - return ids; } - /** * Given a list of LIDs, find latest versions @@ -422,37 +222,15 @@ public List getLatestLidVids(Collection lids) throws Exception Request.Search req = client.createSearchRequest() .setIndex(this.indexName) - .buildLatestLidVids(lids); - Response resp = null; - + .buildLatestLidVids(lids); try { - resp = client.performRequest(req); + return client.performRequest(req).latestLidvids(); } catch(ResponseException ex) { throw ex; } - - //DebugUtils.dumpResponseBody(resp); - - InputStream is = null; - InputStreamReader rd = null; - - try - { - is = resp.getEntity().getContent(); - rd = new InputStreamReader(is); - - LatestLidsResponseParser parser = new LatestLidsResponseParser(); - parser.parse(rd); - return parser.getLidvids(); - } - finally - { - CloseUtils.close(rd); - CloseUtils.close(is); - } } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java index 1364571..48bf1ec 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java @@ -1,19 +1,11 @@ package gov.nasa.pds.registry.common.es.dao.dd; -import java.time.Instant; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import gov.nasa.pds.registry.common.Request; -import gov.nasa.pds.registry.common.Response; import gov.nasa.pds.registry.common.RestClient; -import gov.nasa.pds.registry.common.util.SearchResponseParser; import gov.nasa.pds.registry.common.util.Tuple; @@ -24,8 +16,6 @@ */ public class DataDictionaryDao { - private Logger log; - private RestClient client; private String indexName; @@ -37,42 +27,11 @@ public class DataDictionaryDao */ public DataDictionaryDao(RestClient client, String indexName) { - log = LogManager.getLogger(this.getClass()); - this.client = client; this.indexName = indexName; } - /** - * Inner private class to parse LDD information response from Elasticsearch. - * @author karpenko - */ - private static class GetLddInfoRespParser extends SearchResponseParser implements SearchResponseParser.Callback - { - public LddVersions info; - - public GetLddInfoRespParser() - { - info = new LddVersions(); - } - - @Override - public void onRecord(String id, Object rec) throws Exception - { - if(rec instanceof Map) - { - @SuppressWarnings("rawtypes") - Map map = (Map)rec; - - String strDate = (String)map.get("date"); - info.updateDate(strDate); - - String file = (String)map.get("attr_name"); - info.addSchemaFile(file); - } - } - } /** @@ -86,59 +45,10 @@ public LddVersions getLddInfo(String namespace) throws Exception Request.Search req = client.createSearchRequest() .buildListLdds(namespace) .setIndex(indexName + "-dd"); - Response resp = client.performRequest(req); - - GetLddInfoRespParser parser = new GetLddInfoRespParser(); - parser.parseResponse(resp, parser); - return parser.info; + return client.performRequest(req).lddInfo(); } - /** - * Inner private class to parse LDD information response from Elasticsearch. - * @author karpenko - */ - private static class ListLddsParser extends SearchResponseParser implements SearchResponseParser.Callback - { - public List list; - - public ListLddsParser() - { - list = new ArrayList<>(); - } - - @Override - public void onRecord(String id, Object rec) throws Exception - { - if(rec instanceof Map) - { - @SuppressWarnings("rawtypes") - Map map = (Map)rec; - - LddInfo info = new LddInfo(); - - // Namespace - info.namespace = (String)map.get("attr_ns"); - - // Date - String str = (String)map.get("date"); - if(str != null && !str.isEmpty()) - { - info.date = Instant.parse(str); - } - - // Versions - info.imVersion = (String)map.get("im_version"); - - // File name - info.file = (String)map.get("attr_name"); - - list.add(info); - } - } - } - - /** * List registered LDDs * @param namespace if this parameter is null list all LDDs @@ -150,43 +60,9 @@ public List listLdds(String namespace) throws Exception Request.Search req = client.createSearchRequest() .buildListLdds(namespace) .setIndex(this.indexName + "-dd"); - Response resp = client.performRequest(req); - - ListLddsParser parser = new ListLddsParser(); - parser.parseResponse(resp, parser); - return parser.list; + return client.performRequest(req).ldds(); } - - - /** - * Inner private class to parse LDD information response from Elasticsearch. - * @author karpenko - */ - private static class ListFieldsParser extends SearchResponseParser implements SearchResponseParser.Callback - { - public Set list; - - public ListFieldsParser() - { - list = new HashSet<>(200); - } - - @Override - public void onRecord(String id, Object rec) throws Exception - { - if(rec instanceof Map) - { - @SuppressWarnings("rawtypes") - Map map = (Map)rec; - - String fieldName = (String)map.get("es_field_name"); - list.add(fieldName); - } - } - } - - /** * Get field names by Elasticsearch type, such as "boolean" or "date". * @return a set of field names @@ -197,11 +73,7 @@ public Set getFieldNamesByEsType(String esType) throws Exception Request.Search req = client.createSearchRequest() .buildListFields(esType) .setIndex(this.indexName + "-dd"); - Response resp = client.performRequest(req); - - ListFieldsParser parser = new ListFieldsParser(); - parser.parseResponse(resp, parser); - return parser.list; + return client.performRequest(req).fields(); } @@ -218,56 +90,12 @@ public Set getFieldNamesByEsType(String esType) throws Exception public List getDataTypes(Collection ids, boolean stringForMissing) throws Exception { if(ids == null || ids.isEmpty()) return null; - - List dtInfo = new ArrayList(); - // Create request Request.Get req = client.createMGetRequest() .setIds(ids) .includeField("es_data_type") .setIndex(this.indexName + "-dd"); - - // Call ES - Response resp = client.performRequest(req); - GetDataTypesResponseParser parser = new GetDataTypesResponseParser(); - List records = parser.parse(resp.getEntity()); - - // Process response (list of fields) - boolean missing = false; - - for(GetDataTypesResponseParser.Record rec: records) - { - if(rec.found) - { - dtInfo.add(new Tuple(rec.id, rec.esDataType)); - } - // There is no data type for this field in ES registry-dd index - else - { - // Automatically assign data type for known fields - if(rec.id.startsWith("ref_lid_") || rec.id.startsWith("ref_lidvid_") - || rec.id.endsWith("_Area")) - { - dtInfo.add(new Tuple(rec.id, "keyword")); - continue; - } - - if(stringForMissing) - { - log.warn("Could not find datatype for field " + rec.id + ". Will use 'keyword'"); - dtInfo.add(new Tuple(rec.id, "keyword")); - } - else - { - log.error("Could not find datatype for field " + rec.id); - missing = true; - } - } - } - - if(stringForMissing == false && missing == true) throw new DataTypeNotFoundException(); - - return dtInfo; + return this.client.performRequest(req).dataTypes(stringForMissing); } } From 09b617d6db6c4733fa2e6b8c13f10b8cc06ce365 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 16 Apr 2024 07:52:02 -0700 Subject: [PATCH 29/41] new elements --- .../connection/aws/CreateIndexConfigWrap.java | 117 ++++++++++++ .../common/connection/aws/GetRespWrap.java | 11 ++ .../common/connection/aws/PropertyHelper.java | 49 +++++ .../common/connection/aws/SearchRespWrap.java | 11 ++ .../common/connection/es/GetAltIdsParser.java | 55 ++++++ .../common/connection/es/GetRespImpl.java | 169 ++++++++++++++++++ .../connection/es/NonExistingIdsResponse.java | 47 +++++ .../common/connection/es/SearchRespImpl.java | 142 +++++++++++++++ 8 files changed, 601 insertions(+) create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexConfigWrap.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/GetRespWrap.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/PropertyHelper.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/GetAltIdsParser.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/GetRespImpl.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/NonExistingIdsResponse.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexConfigWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexConfigWrap.java new file mode 100644 index 0000000..4113a53 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexConfigWrap.java @@ -0,0 +1,117 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.opensearch.client.opensearch._types.analysis.Analyzer; +import org.opensearch.client.opensearch._types.analysis.CustomAnalyzer; +import org.opensearch.client.opensearch._types.mapping.DynamicMapping; +import org.opensearch.client.opensearch._types.mapping.DynamicTemplate; +import org.opensearch.client.opensearch._types.mapping.Property; +import org.opensearch.client.opensearch._types.mapping.TextProperty; +import org.opensearch.client.opensearch._types.mapping.TypeMapping; +import org.opensearch.client.opensearch.indices.CreateIndexRequest; +import org.opensearch.client.opensearch.indices.IndexSettings; +import org.opensearch.client.opensearch.indices.IndexSettingsAnalysis; +import org.opensearch.client.opensearch.indices.IndexSettingsMapping; +import org.opensearch.client.opensearch.indices.IndexSettingsMappingLimit; +import com.google.gson.Gson; + +@SuppressWarnings("unchecked") // evil but necessary because of JSON heterogeneous structures +class CreateIndexConfigWrap { + static CreateIndexRequest.Builder update (CreateIndexRequest.Builder builder, String withJsonConfig) { + Map config = (Map)new Gson().fromJson(withJsonConfig, Object.class); + for (String pk : config.keySet()) { + if (pk.equalsIgnoreCase("mappings")) updateMappings (builder, (Map)config.get(pk)); + else if (pk.equalsIgnoreCase("settings")) updateSettings (builder, (Map)config.get(pk)); + else throw new RuntimeException("Unknown config key '" + pk + "' requiring fix to JSON or CreateIndexConfigWrap.update()"); + } + return builder; + } + private static void updateAnalysis (IndexSettings.Builder builder, Map analysis) { + IndexSettingsAnalysis.Builder craftsman = new IndexSettingsAnalysis.Builder(); + for (String pk : analysis.keySet()) { + if (pk.equalsIgnoreCase("normalizer")) { + Analyzer.Builder journeyman = new Analyzer.Builder(); + Map analyzers = (Map)analysis.get(pk); + for (String ak : analyzers.keySet()) { + Map analyzer = (Map)analyzers.get(ak); + String atype = (String)analyzer.get("type"); + if (atype.equalsIgnoreCase("custom")) + journeyman.custom(new CustomAnalyzer.Builder().filter((List)analyzer.get("filter")).tokenizer(ak.equalsIgnoreCase("keyword_lowercase") ? "keyword" : ak).build()); + else throw new RuntimeException("Unknown analyzer type '" + atype + "' requiring fix to JSON or CreateIndexConfigWrap.updateAnalysis()"); + } + craftsman.analyzer(pk, journeyman.build()); + } + else throw new RuntimeException("Unknown analysis key '" + pk + "' requiring fix to JSON or CreateIndexConfigWrap.updateAnalysis()"); + } + builder.analysis(craftsman.build()); + } + private static void updateMappings (CreateIndexRequest.Builder builder, Map mappings) { + TypeMapping.Builder craftsman = new TypeMapping.Builder(); + for (String pk : mappings.keySet()) { + if (pk.equalsIgnoreCase("dynamic")) craftsman.dynamic((Boolean)mappings.get(pk) ? DynamicMapping.True : DynamicMapping.False); + else if (pk.equalsIgnoreCase("dynamic_templates")) { + ArrayList> templates = new ArrayList>(); + List> templateList = (List>)mappings.get(pk); + for (Map namedTemplate : templateList) { + HashMap templateMap = new HashMap(); + for (String name : namedTemplate.keySet()) { + DynamicTemplate.Builder journeyman = new DynamicTemplate.Builder(); + Map template = (Map)namedTemplate.get(name); + for (String sk : template.keySet()) { + if (sk.equalsIgnoreCase("mapping")) + journeyman.mapping(PropertyHelper.setType(new Property.Builder(), + ((Map)template.get(sk)).get("type")).build()); + else if (sk.equalsIgnoreCase("match_mapping_type")) journeyman.matchMappingType((String)template.get(sk)); + else throw new RuntimeException("Unknown template key '" + pk + "' requiring fix to JSON or CreateIndexConfigWrap.updateMappings()"); + } + templateMap.put(name, journeyman.build()); + } + templates.add(templateMap); + } + craftsman.dynamicTemplates(templates); + } + else if (pk.equalsIgnoreCase("properties")) { + Map> propertyMap = (Map>)mappings.get(pk); + HashMap properties = new HashMap(); + for (String name : propertyMap.keySet()) { + Property.Builder journeyman = new Property.Builder(); + for (String sk : propertyMap.get(name).keySet()) { + if (sk.equalsIgnoreCase("analyzer")) { + if (propertyMap.get(name).containsKey("type") && propertyMap.get(name).get("type").equalsIgnoreCase("text")) { + journeyman.text(new TextProperty.Builder().analyzer(propertyMap.get(name).get(sk)).build()); + break; + } + } + else if (sk.equalsIgnoreCase("type")) PropertyHelper.setType(journeyman, propertyMap.get(name).get(sk)); + else throw new RuntimeException("Unknown property key '" + pk + "' requiring fix to JSON or CreateIndexConfigWrap.updateMappings()"); + } + properties.put(name, journeyman.build()); + } + craftsman.properties(properties); + } + else throw new RuntimeException("Unknown mapping key '" + pk + "' requiring fix to JSON or CreateIndexConfigWrap.updateMappings()"); + } + builder.mappings(craftsman.build()); + } + private static void updateSettings (CreateIndexRequest.Builder builder, Map settings) { + IndexSettings.Builder craftsman = new IndexSettings.Builder(); + IndexSettingsMapping.Builder journeyman = new IndexSettingsMapping.Builder(); + for (String pk : settings.keySet()) { + if (pk.equalsIgnoreCase("analysis")) updateAnalysis (craftsman, (Map)settings.get(pk)); + else if (pk.equalsIgnoreCase("index.mapping.total_fields.limit")) + journeyman.totalFields(new IndexSettingsMappingLimit.Builder().limit(Long.valueOf((String)settings.get(pk))).build()); + else if (pk.equalsIgnoreCase("index.max_result_window")) + craftsman.maxResultWindow(Integer.valueOf((String)settings.get(pk))); + else if (pk.equalsIgnoreCase("number_of_replicas")) + craftsman.numberOfReplicas(Integer.toString(((Double)settings.get(pk)).intValue())); + else if (pk.equalsIgnoreCase("number_of_shards")) + craftsman.numberOfShards(Integer.toString(((Double)settings.get(pk)).intValue())); + else throw new RuntimeException("Unknown setting key '" + pk + "' requiring fix to JSON or CreateIndexConfigWrap.updateSettings()"); + } + craftsman.mapping(journeyman.build()); + builder.settings(craftsman.build()); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetRespWrap.java new file mode 100644 index 0000000..097e578 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetRespWrap.java @@ -0,0 +1,11 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.opensearch.client.opensearch.core.GetResponse; +import gov.nasa.pds.registry.common.Response; + +class GetRespWrap implements Response.Get { + final private GetResponse parent; + GetRespWrap(GetResponse parent) { + this.parent = parent; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/PropertyHelper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/PropertyHelper.java new file mode 100644 index 0000000..426d281 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/PropertyHelper.java @@ -0,0 +1,49 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.opensearch.client.opensearch._types.mapping.BinaryProperty; +import org.opensearch.client.opensearch._types.mapping.BooleanProperty; +import org.opensearch.client.opensearch._types.mapping.DateProperty; +import org.opensearch.client.opensearch._types.mapping.DoubleNumberProperty; +import org.opensearch.client.opensearch._types.mapping.FloatNumberProperty; +import org.opensearch.client.opensearch._types.mapping.IntegerNumberProperty; +import org.opensearch.client.opensearch._types.mapping.KeywordProperty; +import org.opensearch.client.opensearch._types.mapping.LongNumberProperty; +import org.opensearch.client.opensearch._types.mapping.Property; +import org.opensearch.client.opensearch._types.mapping.TextProperty; + +final class PropertyHelper { + static Property.Builder setType (Property.Builder builder, String fieldType) { + switch (fieldType) { + case "binary": + builder.binary(new BinaryProperty.Builder().build()); + break; + case "boolean": + builder.boolean_(new BooleanProperty.Builder().build()); + break; + case "date": + builder.date(new DateProperty.Builder().build()); + break; + case "double": + builder.double_(new DoubleNumberProperty.Builder().build()); + break; + case "float": + builder.float_(new FloatNumberProperty.Builder().build()); + break; + case "integer": + builder.integer(new IntegerNumberProperty.Builder().build()); + break; + case "keyword": + builder.keyword(new KeywordProperty.Builder().build()); + break; + case "long": + builder.long_(new LongNumberProperty.Builder().build()); + break; + case "text": + builder.text(new TextProperty.Builder().build()); + break; + default: + throw new RuntimeException("Cannot map type '" + fieldType + "' yet. Please review PropertyHelper.setType() code and fix."); + } + return builder; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java new file mode 100644 index 0000000..4771d54 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java @@ -0,0 +1,11 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.opensearch.client.opensearch.core.SearchResponse; +import gov.nasa.pds.registry.common.Response; + +class SearchRespWrap implements Response.Search { + final private SearchResponse parent; + SearchRespWrap(SearchResponse parent) { + this.parent = parent; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/GetAltIdsParser.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/GetAltIdsParser.java new file mode 100644 index 0000000..82f6da9 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/GetAltIdsParser.java @@ -0,0 +1,55 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +class GetAltIdsParser implements SearchResponseParser.Callback +{ + private Map> map; + + + GetAltIdsParser() + { + map = new TreeMap<>(); + } + + + public Map> getIdMap() + { + return map; + } + + + @Override + public void onRecord(String id, Object rec) + { + if(rec instanceof Map) + { + Object obj = ((Map)rec).get("alternate_ids"); + if(obj == null) return; + + // Multiple values + if(obj instanceof List) + { + Set altIds = new TreeSet<>(); + for(Object item: (List)obj) + { + altIds.add(item.toString()); + } + + map.put(id, altIds); + } + // Single value + else if(obj instanceof String) + { + Set altIds = new TreeSet<>(); + altIds.add((String)obj); + map.put(id, altIds); + } + } + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/GetRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/GetRespImpl.java new file mode 100644 index 0000000..e410660 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/GetRespImpl.java @@ -0,0 +1,169 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.es.dao.DaoUtils; +import gov.nasa.pds.registry.common.es.dao.LidvidSet; +import gov.nasa.pds.registry.common.es.dao.dd.DataTypeNotFoundException; +import gov.nasa.pds.registry.common.es.dao.dd.GetDataTypesResponseParser; +import gov.nasa.pds.registry.common.util.LidVidUtils; +import gov.nasa.pds.registry.common.util.Tuple; + +class GetRespImpl implements Response.Get { + final private Logger log; + final private org.elasticsearch.client.Response response; + GetRespImpl(org.elasticsearch.client.Response response) { + this.log = LogManager.getLogger(this.getClass()); + this.response = response; + } + private LidvidSet parseCollectionIdsSource(JsonReader rd) throws IOException { + LidvidSet ids = new LidvidSet(null,null); + rd.beginObject(); + while (rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if ("ref_lid_collection".equals(name)) { + ids.lids = DaoUtils.parseSet(rd); + } else if ("ref_lidvid_collection".equals(name)) { + ids.lidvids = DaoUtils.parseSet(rd); + } else { + rd.skipValue(); + } + } + rd.endObject(); + return ids; + } + private String parseProductClassSource(JsonReader rd) throws IOException { + rd.beginObject(); + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if("product_class".equals(name)) { + return rd.nextString(); + } else { + rd.skipValue(); + } + } + rd.endObject(); + return null; + } + private List parseRefs(JsonReader rd) throws IOException { + rd.beginObject(); + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if("product_lidvid".equals(name)) { + return DaoUtils.parseList(rd); + } else { + rd.skipValue(); + } + } + rd.endObject(); + return null; + } + @Override + public String productClass() { + try (InputStream is = this.response.getEntity().getContent()) { + JsonReader rd = new JsonReader(new InputStreamReader(is)); + rd.beginObject(); + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if("_source".equals(name)) { + return parseProductClassSource(rd); + } else { + rd.skipValue(); + } + } + rd.endObject(); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("Weird JSON parsing error because should never reach this branch"); + } + return null; + } + @Override + public List refs() { + try (InputStream is = this.response.getEntity().getContent()) { + JsonReader rd = new JsonReader(new InputStreamReader(is)); + rd.beginObject(); + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if("_source".equals(name)) { + return parseRefs(rd); + } else { + rd.skipValue(); + } + } + rd.endObject(); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("Weird JSON parsing problem that should never occur"); + } + return null; + } + @Override + public IdSets ids() { + LidvidSet collectionIds = null; + try (InputStream is = this.response.getEntity().getContent()) { + JsonReader rd = new JsonReader(new InputStreamReader(is)); + rd.beginObject(); + while (rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if ("_source".equals(name)) { + collectionIds = parseCollectionIdsSource(rd); + } else { + rd.skipValue(); + } + } + rd.endObject(); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("Weird JSON parsing problem that should never get here"); + } + if (collectionIds == null || collectionIds.lidvids == null || collectionIds.lids == null) + return collectionIds; + // Harvest converts LIDVIDs to LIDs, so let's delete those converted LIDs. + for (String lidvid : collectionIds.lidvids) { + String lid = LidVidUtils.lidvidToLid(lidvid); + if (lid != null) { + collectionIds.lids.remove(lid); + } + } + return collectionIds; + } + @Override + public List dataTypes(boolean stringForMissing) throws IOException, DataTypeNotFoundException { + List dtInfo = new ArrayList(); + GetDataTypesResponseParser parser = new GetDataTypesResponseParser(); + List records = parser.parse(this.response.getEntity()); + // Process response (list of fields) + boolean missing = false; + for (GetDataTypesResponseParser.Record rec : records) { + if (rec.found) { + dtInfo.add(new Tuple(rec.id, rec.esDataType)); + } + // There is no data type for this field in ES registry-dd index + else { + // Automatically assign data type for known fields + if (rec.id.startsWith("ref_lid_") || rec.id.startsWith("ref_lidvid_") + || rec.id.endsWith("_Area")) { + dtInfo.add(new Tuple(rec.id, "keyword")); + continue; + } + if (stringForMissing) { + log.warn("Could not find datatype for field " + rec.id + ". Will use 'keyword'"); + dtInfo.add(new Tuple(rec.id, "keyword")); + } else { + log.error("Could not find datatype for field " + rec.id); + missing = true; + } + } + } + if (stringForMissing == false && missing == true) + throw new DataTypeNotFoundException(); + + return dtInfo; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/NonExistingIdsResponse.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/NonExistingIdsResponse.java new file mode 100644 index 0000000..c7ff219 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/NonExistingIdsResponse.java @@ -0,0 +1,47 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.util.Collection; +import java.util.Set; +import java.util.TreeSet; + +/** + * Helper class to process Elasticsearch response from "search IDs" query. + * + * @author karpenko + */ +class NonExistingIdsResponse implements SearchResponseParser.Callback +{ + private Set retIds; + + + /** + * Constructor + * @param ids Product IDs (lidvids) sent to Elasticsearch in "search IDs" query. + * IDs are copied to internal collection. + * After processing Elasticsearch response, all IDs existing in Elasticsearch + * "registry" index will be removed from this internal collection. + */ + NonExistingIdsResponse(Collection ids) + { + retIds = new TreeSet<>(ids); + } + + /** + * Return collection of product IDs (lidvids) non-existing in Elasticsearch. + * @return a collection of product IDs (lidvids) + */ + public Set getIds() + { + return retIds; + } + + + /** + * This method is called for each record in Elasticsearch response + */ + @Override + public void onRecord(String id, Object src) + { + retIds.remove(id); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java new file mode 100644 index 0000000..0996b95 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java @@ -0,0 +1,142 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.es.dao.LatestLidsResponseParser; +import gov.nasa.pds.registry.common.es.dao.dd.LddInfo; +import gov.nasa.pds.registry.common.es.dao.dd.LddVersions; + +class SearchRespImpl implements Response.Search { + /** + * Inner private class to parse LDD information response from Elasticsearch. + * + * @author karpenko + */ + private static class ListFieldsParser extends SearchResponseParser + implements SearchResponseParser.Callback { + public Set list; + public ListFieldsParser() { + list = new HashSet<>(200); + } + @Override + public void onRecord(String id, Object rec) { + if (rec instanceof Map) { + @SuppressWarnings("rawtypes") + Map map = (Map) rec; + String fieldName = (String) map.get("es_field_name"); + list.add(fieldName); + } + } + } + /** + * Inner private class to parse LDD information response from Elasticsearch. + * + * @author karpenko + */ + private static class GetLddInfoRespParser extends SearchResponseParser + implements SearchResponseParser.Callback { + public LddVersions info; + public GetLddInfoRespParser() { + info = new LddVersions(); + } + @Override + public void onRecord(String id, Object rec) { + if (rec instanceof Map) { + @SuppressWarnings("rawtypes") + Map map = (Map) rec; + String strDate = (String) map.get("date"); + info.updateDate(strDate); + String file = (String) map.get("attr_name"); + info.addSchemaFile(file); + } + } + } + /** + * Inner private class to parse LDD information response from Elasticsearch. + * + * @author karpenko + */ + private static class ListLddsParser extends SearchResponseParser + implements SearchResponseParser.Callback { + public List list; + public ListLddsParser() { + list = new ArrayList<>(); + } + @Override + public void onRecord(String id, Object rec) { + if (rec instanceof Map) { + @SuppressWarnings("rawtypes") + Map map = (Map) rec; + LddInfo info = new LddInfo(); + // Namespace + info.namespace = (String) map.get("attr_ns"); + // Date + String str = (String) map.get("date"); + if (str != null && !str.isEmpty()) { + info.date = Instant.parse(str); + } + // Versions + info.imVersion = (String) map.get("im_version"); + // File name + info.file = (String) map.get("attr_name"); + list.add(info); + } + } + } + + final private org.elasticsearch.client.Response response; + SearchRespImpl(org.elasticsearch.client.Response response) { + this.response = response; + } + @Override + public List latestLidvids() { + try (InputStream is = this.response.getEntity().getContent()) { + LatestLidsResponseParser parser = new LatestLidsResponseParser(); + parser.parse(new InputStreamReader(is)); + return parser.getLidvids(); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("Weird JSON parsing error and should never get here"); + } + } + @Override + public LddVersions lddInfo() throws UnsupportedOperationException, IOException { + GetLddInfoRespParser parser = new GetLddInfoRespParser(); + parser.parseResponse(this.response, parser); + return parser.info; + } + @Override + public List ldds() throws UnsupportedOperationException, IOException { + ListLddsParser parser = new ListLddsParser(); + parser.parseResponse(this.response, parser); + return parser.list; + } + @Override + public Set fields() throws UnsupportedOperationException, IOException { + ListFieldsParser parser = new ListFieldsParser(); + parser.parseResponse(this.response, parser); + return parser.list; + } + @Override + public Map> altIds() throws UnsupportedOperationException, IOException { + GetAltIdsParser cb = new GetAltIdsParser(); + SearchResponseParser parser = new SearchResponseParser(); + parser.parseResponse(this.response, cb); + return cb.getIdMap(); + } + @Override + public Set nonExistingIds(Collection from_ids) throws UnsupportedOperationException, IOException { + NonExistingIdsResponse idsResp = new NonExistingIdsResponse(from_ids); + SearchResponseParser parser = new SearchResponseParser(); + parser.parseResponse(this.response, idsResp); + return idsResp.getIds(); + } +} From 0014e52e94d422fabb4347f1d04a8836c9d0e25f Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 17 Apr 2024 08:41:52 -0700 Subject: [PATCH 30/41] harvest is working --- .../nasa/pds/registry/common/Response.java | 2 +- .../common/connection/aws/GetRespWrap.java | 34 ++++++++++ .../common/connection/aws/MGetRespWrap.java | 41 ++++++++++++ .../connection/aws/RestClientWrapper.java | 2 + .../common/connection/aws/SearchRespWrap.java | 64 +++++++++++++++++++ 5 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetRespWrap.java diff --git a/src/main/java/gov/nasa/pds/registry/common/Response.java b/src/main/java/gov/nasa/pds/registry/common/Response.java index 8cdb446..02708fb 100644 --- a/src/main/java/gov/nasa/pds/registry/common/Response.java +++ b/src/main/java/gov/nasa/pds/registry/common/Response.java @@ -35,7 +35,7 @@ public interface IdSets { public Set lids(); public Set lidvids(); } - public List dataTypes(boolean stringForMissing) throws IOException, DataTypeNotFoundException; + public List dataTypes(boolean stringForMissing) throws IOException, DataTypeNotFoundException; public IdSets ids(); // returns null if nothing is found in returned content public String productClass(); // returns null if product class not in returned content public List refs(); // returns null if nothing is found in returned content diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetRespWrap.java index 097e578..b3fd6a0 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetRespWrap.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetRespWrap.java @@ -1,11 +1,45 @@ package gov.nasa.pds.registry.common.connection.aws; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.lang3.NotImplementedException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.client.opensearch.core.GetResponse; import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.es.dao.LidvidSet; +import gov.nasa.pds.registry.common.es.dao.dd.DataTypeNotFoundException; +import gov.nasa.pds.registry.common.util.Tuple; class GetRespWrap implements Response.Get { final private GetResponse parent; + final protected Logger log; GetRespWrap(GetResponse parent) { + this.log = LogManager.getLogger(this.getClass()); this.parent = parent; } + @Override + public List dataTypes(boolean stringForMissing) + throws IOException, DataTypeNotFoundException { + throw new RuntimeException("This method is supported via MGet and should never be called here"); + } + @Override + public IdSets ids() { + LidvidSet result = new LidvidSet(null,null); + if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + return result; + } + @Override + public String productClass() { + String result = ""; + if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + return result; + } + @Override + public List refs() { + ArrayList results = new ArrayList(); + if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + return results; + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetRespWrap.java new file mode 100644 index 0000000..e2054e4 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetRespWrap.java @@ -0,0 +1,41 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.opensearch.client.opensearch.core.MgetResponse; +import org.opensearch.client.opensearch.core.mget.MultiGetResponseItem; +import gov.nasa.pds.registry.common.es.dao.dd.DataTypeNotFoundException; +import gov.nasa.pds.registry.common.util.Tuple; + +class MGetRespWrap extends GetRespWrap { + final private MgetResponse parent; + MGetRespWrap (MgetResponse parent) { + super(null); + this.parent = parent; + } + @Override + public List dataTypes(boolean stringForMissing) throws IOException, DataTypeNotFoundException { + ArrayList results = new ArrayList(); + for (MultiGetResponseItem doc : this.parent.docs()) { + Tuple t = null; + if (doc.isResult()) { + @SuppressWarnings("unchecked") + Map src = (Map)doc.result().source(); + if (src != null && src.containsKey("es_data_type")) t = new Tuple(doc.result().id(), src.get("es_data_type")); + } else if (stringForMissing) { + if (doc.result().id().startsWith("ref_lid_") || doc.result().id().startsWith("ref_lidvid_") + || doc.result().id().endsWith("_Area")) { + t = new Tuple(doc.result().id(), "keyword"); + } else { + log.error("Could not find datatype for field " + doc.result().id()); + } + } else { + throw new DataTypeNotFoundException(); + } + if (t != null) results.add(t); + } + return results; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java index b88c25a..913bf12 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java @@ -132,6 +132,8 @@ public long performRequest(Count request) throws IOException, ResponseException } @Override public Response.Get performRequest(Get request) throws IOException, ResponseException { + if (request instanceof MGet) + return new MGetRespWrap(this.client.mget(((MGetImpl)request).craftsman.build(), Object.class)); return new GetRespWrap(this.client.get(((GetImpl)request).craftsman.build(), Object.class)); } @Override diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java index 4771d54..281d9f3 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java @@ -1,11 +1,75 @@ package gov.nasa.pds.registry.common.connection.aws; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.NotImplementedException; import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.search.Hit; import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.es.dao.dd.LddInfo; +import gov.nasa.pds.registry.common.es.dao.dd.LddVersions; +@SuppressWarnings("unchecked") // JSON heterogenous structures requires raw casting class SearchRespWrap implements Response.Search { final private SearchResponse parent; SearchRespWrap(SearchResponse parent) { this.parent = parent; } + @Override + public Map> altIds() throws UnsupportedOperationException, IOException { + HashMap> results = new HashMap>(); + if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + return results; + } + @Override + public Set fields() throws UnsupportedOperationException, IOException { + Set results = new HashSet(); + for (Hit hit : this.parent.hits().hits()) { + for (String value : ((Map)hit.source()).values()) { + results.add(value); + } + } + return results; + } + @Override + public List latestLidvids() { + ArrayList lidvids = new ArrayList(); + if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + return lidvids; + } + @Override + public LddVersions lddInfo() throws UnsupportedOperationException, IOException { + LddVersions result = new LddVersions(); + for (Hit hit : this.parent.hits().hits()) { + Map source = (Map)hit.source(); + if (source.containsKey("attr_name") && source.containsKey("date")) { + result.addSchemaFile(source.get("attr_name")); + result.updateDate(source.get("date")); + } else { + throw new UnsupportedOperationException("Either date or attr_name or both are missing from hit."); + } + } + return result; + } + @Override + public List ldds() throws UnsupportedOperationException, IOException { + ArrayList results = new ArrayList(); + if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + return results; + } + @Override + public Set nonExistingIds(Collection from_ids) + throws UnsupportedOperationException, IOException { + HashSet results = new HashSet(from_ids); + for (Hit hit : this.parent.hits().hits()) { + if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + } + return results; + } } From 1694a68c8dbead4ea9bd05a0f34ab22c592d987a Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 17 Apr 2024 12:15:03 -0700 Subject: [PATCH 31/41] fix fixme --- .../common/EstablishConnectionFactory.java | 7 ++--- .../nasa/pds/registry/common/Response.java | 1 + .../connection/KnownRegistryConnections.java | 3 +- .../common/connection/aws/BulkImpl.java | 11 +++++++- .../common/connection/aws/BulkRespWrap.java | 16 ++++++++++- .../connection/aws/RestClientWrapper.java | 4 ++- .../common/connection/es/BulkRespImpl.java | 28 ++++++++++++++----- .../es}/BulkResponseParser.java | 21 +------------- .../registry/common/es/dao/DataLoader.java | 1 - .../registry/common/es/dao/ProductDao.java | 3 +- src/main/resources/registry-connection.xsd | 2 +- src/test/java/dao/TestBulkResponseParser.java | 3 +- 12 files changed, 57 insertions(+), 43 deletions(-) rename src/main/java/gov/nasa/pds/registry/common/{es/dao => connection/es}/BulkResponseParser.java (83%) diff --git a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java index fce7b6f..a0ae3a3 100644 --- a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java +++ b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java @@ -2,6 +2,7 @@ import java.net.URL; import gov.nasa.pds.registry.common.connection.AuthContent; +import gov.nasa.pds.registry.common.connection.KnownRegistryConnections; import gov.nasa.pds.registry.common.connection.UseOpensearchSDK1; import gov.nasa.pds.registry.common.connection.UseOpensearchSDK2; import gov.nasa.pds.registry.common.connection.RegistryConnectionContent; @@ -14,11 +15,7 @@ public static ConnectionFactory from (String urlToRegistryConnection, String aut return EstablishConnectionFactory.from (urlToRegistryConnection, AuthContent.from(authFile)); } private static synchronized ConnectionFactory from (String urlToRegistryConnection, AuthContent auth) throws Exception { - final String key = "java.protocol.handler.pkgs", me = "gov.nasa.pds.registry.common"; - String providers = System.getProperty(key, ""); - if (providers.isEmpty()) providers = me; - if (!providers.contains(me)) providers += "|" + me; - System.setProperty(key, providers); + KnownRegistryConnections.initialzeAppHandler(); RegistryConnectionContent conn = RegistryConnectionContent.from (new URL(urlToRegistryConnection)); if (conn.isDirectConnection()) { diff --git a/src/main/java/gov/nasa/pds/registry/common/Response.java b/src/main/java/gov/nasa/pds/registry/common/Response.java index 02708fb..ade9fe0 100644 --- a/src/main/java/gov/nasa/pds/registry/common/Response.java +++ b/src/main/java/gov/nasa/pds/registry/common/Response.java @@ -23,6 +23,7 @@ public interface Item { } public boolean errors(); public List items(); + public void logErrors(); public long took(); } public interface CreatedIndex { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/KnownRegistryConnections.java b/src/main/java/gov/nasa/pds/registry/common/connection/KnownRegistryConnections.java index 3dbc3b8..14f7c3a 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/KnownRegistryConnections.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/KnownRegistryConnections.java @@ -46,11 +46,10 @@ static public List list(@SuppressWarnings("rawtypes") Class cls) throws URI String candidate = i.next().toString(); candidate = candidate.substring(candidate.indexOf("/connections")); if (candidate.endsWith(".xml")) { - result.add(new URL("app:" + candidate)); + result.add(new URL("app:/" + candidate)); } } } - for (URL url : result) System.out.println(url.toString()); return result; } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java index 31ff4da..adca455 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java @@ -17,6 +17,10 @@ class BulkImpl implements Bulk { final BulkRequest.Builder craftsman = new BulkRequest.Builder(); + final private boolean isServerless; + BulkImpl (boolean isServerless) { + this.isServerless = isServerless; + } @SuppressWarnings("unchecked") @Override public void add(String statement, String document) { @@ -91,7 +95,12 @@ public Bulk setRefresh(Refresh type) { this.craftsman.refresh(org.opensearch.client.opensearch._types.Refresh.True); break; case WaitFor: - this.craftsman.refresh(org.opensearch.client.opensearch._types.Refresh.WaitFor); + if (this.isServerless) { + // No OP because not supported in serverless realm + } + else { + this.craftsman.refresh(org.opensearch.client.opensearch._types.Refresh.WaitFor); + } break; } return this; diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkRespWrap.java index 010e387..04c3f30 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkRespWrap.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkRespWrap.java @@ -2,6 +2,8 @@ import java.util.ArrayList; import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.opensearch.client.opensearch.core.BulkResponse; import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; import gov.nasa.pds.registry.common.Response; @@ -43,7 +45,9 @@ public String reason() { }; final private ArrayList items = new ArrayList(); final private BulkResponse parent; + final private Logger log; BulkRespWrap(BulkResponse parent) { + this.log = LogManager.getLogger(this.getClass()); this.parent = parent; } @Override @@ -57,10 +61,20 @@ public synchronized List items() { this.items.add(new ItemWrap(item)); } } - return this.items(); + return this.items; } @Override public long took() { return this.parent.took(); } + @Override + public void logErrors() { + if (this.parent.errors()) { + for (BulkResponseItem item : this.parent.items()) { + if (item.error() != null && item.error().reason() != null && !item.error().reason().isBlank()) { + log.error(item.error().reason()); + } + } + } + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java index 913bf12..5f383ad 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java @@ -33,10 +33,12 @@ import gov.nasa.pds.registry.common.RestClient; public class RestClientWrapper implements RestClient { + final private boolean isServerless; final private OpenSearchClient client; final private SdkHttpClient httpClient; public RestClientWrapper(ConnectionFactory conFact, boolean isServerless) { this.httpClient = ApacheHttpClient.builder().build(); + this.isServerless = isServerless; if (isServerless) { this.client = new OpenSearchClient( new AwsSdk2Transport( @@ -84,7 +86,7 @@ public void close() throws IOException { } @Override public Bulk createBulkRequest() { - return new BulkImpl(); + return new BulkImpl(this.isServerless); } @Override public Count createCountRequest() { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java index d0d91a3..3a383e0 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java @@ -1,17 +1,24 @@ package gov.nasa.pds.registry.common.connection.es; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.NotImplementedException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import com.google.gson.Gson; import gov.nasa.pds.registry.common.Response; class BulkRespImpl implements Response.Bulk { + final int errorCount; final private Logger log; + final private org.elasticsearch.client.Response response; BulkRespImpl (org.elasticsearch.client.Response response) { this.log = LogManager.getLogger(this.getClass()); - this.parse(response.toString()); + this.errorCount = this.parse(response.toString()); + this.response = response; } @SuppressWarnings("rawtypes") // necessary evil to manipulate heterogenous structures private int parse (String resp) { @@ -76,17 +83,24 @@ private int parse (String resp) { } @Override public boolean errors() { - // TODO Auto-generated method stub - return false; + return errorCount != 0; } @Override public List items() { - // TODO Auto-generated method stub - return null; + throw new NotImplementedException(); } @Override public long took() { - // TODO Auto-generated method stub - return 0; + throw new NotImplementedException(); + } + @Override + public void logErrors() { + BulkResponseParser parser = new BulkResponseParser(); + try (InputStream is = this.response.getEntity().getContent()) { + InputStreamReader reader = new InputStreamReader(is); + parser.parse(reader); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("Some weird JSON parsing exception and should never get here."); + } } } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponseParser.java similarity index 83% rename from src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponseParser.java index fb52090..ff1441b 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponseParser.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.dao; +package gov.nasa.pds.registry.common.connection.es; import java.io.IOException; import java.io.Reader; @@ -8,7 +8,6 @@ import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; -import gov.nasa.pds.registry.common.Response; import gov.nasa.pds.registry.common.util.CloseUtils; /** @@ -27,24 +26,6 @@ public BulkResponseParser() log = LogManager.getLogger(this.getClass()); } - public void parse(Response.Bulk response) { - /* FIXME - * This should log erros as parse(Reader reader) does. - * Just put this empty implementation in place to move - * forward with SDK v2 but leave this compile without - * doing any actual work. - * - * old invocations was like this: - * - is = resp.getEntity().getContent(); - rd = new InputStreamReader(is); - - BulkResponseParser parser = new BulkResponseParser(); - parser.parse(rd); - - */ - } - /** * Parse JSON string * @param reader bulk API response stream diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java index bdefcf3..134fb38 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java @@ -231,7 +231,6 @@ public int loadBatch(List data, Set errorLidvids, int retries) t for (int index = 0 ; index < data.size() ; index++) { bulk.add(data.get(index), data.get(++index)); } - // FIXME: response has changed so need to debug this Response.Bulk response = this.conFactory.createRestClient().performRequest(bulk); // Check for Elasticsearch errors. int failedCount = processErrors(response, errorLidvids); diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java index ca22ef8..52a82d8 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java @@ -172,8 +172,7 @@ public void updateArchiveStatus(Collection lidvids, String status) throw .buildUpdateStatus(lidvids, status) .setIndex(this.indexName); - Response .Bulk resp = client.performRequest(req); - new BulkResponseParser().parse(resp); + client.performRequest(req).logErrors();; } /** diff --git a/src/main/resources/registry-connection.xsd b/src/main/resources/registry-connection.xsd index 0e723df..263ff91 100644 --- a/src/main/resources/registry-connection.xsd +++ b/src/main/resources/registry-connection.xsd @@ -45,7 +45,7 @@ - diff --git a/src/test/java/dao/TestBulkResponseParser.java b/src/test/java/dao/TestBulkResponseParser.java index 2b91cce..d4c2444 100644 --- a/src/test/java/dao/TestBulkResponseParser.java +++ b/src/test/java/dao/TestBulkResponseParser.java @@ -1,8 +1,7 @@ package dao; import java.io.StringReader; - -import gov.nasa.pds.registry.common.es.dao.BulkResponseParser; +import gov.nasa.pds.registry.common.connection.es.BulkResponseParser; import tt.TestLogConfigurator; public class TestBulkResponseParser From 0df3fce28887d5f620c4cb0a0e018bfeae4b0773 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 17 Apr 2024 12:15:37 -0700 Subject: [PATCH 32/41] add cognito --- src/main/resources/connections/cognito/development.xml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/main/resources/connections/cognito/development.xml diff --git a/src/main/resources/connections/cognito/development.xml b/src/main/resources/connections/cognito/development.xml new file mode 100644 index 0000000..a1c777a --- /dev/null +++ b/src/main/resources/connections/cognito/development.xml @@ -0,0 +1,7 @@ + + + 47d9j6ks9un4errq6pnbu0bc1r + From 090aab21b9cdbcd19d8862139e8748c946e04a4a Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Fri, 19 Apr 2024 11:03:06 -0700 Subject: [PATCH 33/41] clean up path for harvest #158 --- .../nasa/pds/registry/common/meta/FileMetadataExtractor.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/gov/nasa/pds/registry/common/meta/FileMetadataExtractor.java b/src/main/java/gov/nasa/pds/registry/common/meta/FileMetadataExtractor.java index 6f79040..8d355ee 100644 --- a/src/main/java/gov/nasa/pds/registry/common/meta/FileMetadataExtractor.java +++ b/src/main/java/gov/nasa/pds/registry/common/meta/FileMetadataExtractor.java @@ -259,6 +259,10 @@ private String getFileRef(File file, List refRules) if(filePath.startsWith(rule.prefix)) { filePath = rule.replacement + filePath.substring(rule.prefix.length()); + String protocol = filePath.substring(0, filePath.indexOf(':')+3); // include :// as part of protocol + String body = filePath.substring(protocol.length()); + while (body.contains("//")) body = body.replace("//", "/"); + filePath = protocol + body; break; } } From 8d760fe2e6668e8e46272be4aa4fb54e7a1569a5 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Mon, 29 Apr 2024 12:24:45 -0700 Subject: [PATCH 34/41] moved over bad references from mgr --- .../gov/nasa/pds/registry/common/Request.java | 9 ++++ .../nasa/pds/registry/common/Response.java | 2 + .../nasa/pds/registry/common/RestClient.java | 51 +++++++++++++++++++ .../connection/es/RestClientWrapper.java | 10 ++++ .../common/connection/es/SearchImpl.java | 19 ++++++- .../common/connection/es/SearchRespImpl.java | 35 +++++++++++++ 6 files changed, 124 insertions(+), 2 deletions(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/Request.java b/src/main/java/gov/nasa/pds/registry/common/Request.java index 26a10d3..844eb5d 100644 --- a/src/main/java/gov/nasa/pds/registry/common/Request.java +++ b/src/main/java/gov/nasa/pds/registry/common/Request.java @@ -16,6 +16,12 @@ public interface Count { // _count public Count setIndex (String name); public Count setQuery (String q); } + public interface DeleteByQuery { // delete_by_query + public DeleteByQuery createFilterQuery(String key, String value); + public DeleteByQuery createMatchAllQuery(); + public DeleteByQuery setIndex (String name); + public DeleteByQuery setRefresh(boolean state); + } public interface Get { // _doc public Get excludeField (String field); public Get excludeFields (List fields); @@ -32,7 +38,10 @@ public interface MGet extends Get { // _mget public MGet setIds (Collection ids); } public interface Search { // _search + public Search all(String sortField, int size, String searchAfter); + public Search all(String filterField, String filterValue, String sortField, int size, String searchAfter); public Search buildAlternativeIds(Collection lids); + public Search buildGetField(String field_name, String lidvid); public Search buildLatestLidVids(Collection lids); public Search buildListFields(String dataType); public Search buildListLdds (String namespace); diff --git a/src/main/java/gov/nasa/pds/registry/common/Response.java b/src/main/java/gov/nasa/pds/registry/common/Response.java index ade9fe0..18eee60 100644 --- a/src/main/java/gov/nasa/pds/registry/common/Response.java +++ b/src/main/java/gov/nasa/pds/registry/common/Response.java @@ -46,6 +46,8 @@ public interface Mapping { } public interface Search { public Map> altIds() throws UnsupportedOperationException, IOException; + public List batch() throws UnsupportedOperationException, IOException; + public String field(String name) throws NoSuchFieldException; // null means blob not in document and NoSuchFieldException document not found public Set fields() throws UnsupportedOperationException, IOException; public List latestLidvids(); // returns null if nothing is found in returned content public LddVersions lddInfo() throws UnsupportedOperationException, IOException; diff --git a/src/main/java/gov/nasa/pds/registry/common/RestClient.java b/src/main/java/gov/nasa/pds/registry/common/RestClient.java index 73be161..e5b8434 100644 --- a/src/main/java/gov/nasa/pds/registry/common/RestClient.java +++ b/src/main/java/gov/nasa/pds/registry/common/RestClient.java @@ -1,11 +1,20 @@ package gov.nasa.pds.registry.common; import java.io.Closeable; +import java.io.File; +import java.io.FileReader; import java.io.IOException; +import java.io.StringWriter; +import java.util.Map; +import java.util.TreeMap; +import com.google.gson.Gson; +import com.google.gson.stream.JsonWriter; +import gov.nasa.pds.registry.common.util.CloseUtils; public interface RestClient extends Closeable { public Request.Bulk createBulkRequest(); public Request.Count createCountRequest(); + public Request.DeleteByQuery createDeleteByQuery(); public Request.Get createGetRequest(); public Request.Mapping createMappingRequest(); public Request.MGet createMGetRequest(); @@ -16,8 +25,50 @@ public interface RestClient extends Closeable { public boolean exists (String indexName) throws IOException,ResponseException; public Response.Bulk performRequest(Request.Bulk request) throws IOException,ResponseException; public long performRequest(Request.Count request) throws IOException,ResponseException; + public long performRequest(Request.DeleteByQuery request) throws IOException,ResponseException; public Response.Get performRequest(Request.Get request) throws IOException,ResponseException; public Response.Mapping performRequest(Request.Mapping request) throws IOException,ResponseException; public Response.Search performRequest(Request.Search request) throws IOException,ResponseException; public Response.Settings performRequest(Request.Setting request) throws IOException,ResponseException; + /** + * Build create index request + * + * @param schemaFile index schema file + * @param shards number of shards + * @param replicas number of replicas + * @return JSON + * @throws Exception Generic exception + */ + @SuppressWarnings({"rawtypes", "unchecked"}) + public static String createCreateIndexRequest(File schemaFile, int shards, int replicas) + throws Exception { + // Read schema template + FileReader rd = new FileReader(schemaFile); + Gson gson = new Gson(); + Object rootObj = gson.fromJson(rd, Object.class); + CloseUtils.close(rd); + Object settingsObj = ((Map) rootObj).get("settings"); + if (settingsObj == null) { + settingsObj = new TreeMap(); + } + Object mappingsObj = ((Map) rootObj).get("mappings"); + if (mappingsObj == null) { + throw new Exception("Missing mappings in schema file " + schemaFile.getAbsolutePath()); + } + StringWriter out = new StringWriter(); + JsonWriter writer = new JsonWriter(out); + writer.beginObject(); + Map settingsMap = (Map) settingsObj; + settingsMap.put("number_of_shards", shards); + settingsMap.put("number_of_replicas", replicas); + // Settings + writer.name("settings"); + gson.toJson(settingsObj, Object.class, writer); + // Mappings + writer.name("mappings"); + gson.toJson(mappingsObj, Object.class, writer); + writer.endObject(); + writer.close(); + return out.toString(); + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java index 6b24ffc..f9387ea 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java @@ -7,6 +7,7 @@ import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.Request.Bulk; import gov.nasa.pds.registry.common.Request.Count; +import gov.nasa.pds.registry.common.Request.DeleteByQuery; import gov.nasa.pds.registry.common.Request.Get; import gov.nasa.pds.registry.common.Request.Mapping; import gov.nasa.pds.registry.common.Request.MGet; @@ -130,4 +131,13 @@ public Response.Search performRequest(Search request) throws IOException,Respons public Response.Settings performRequest(Setting request) throws IOException,ResponseException { return new SettingsRespImpl(this.performRequest(request.toString(), null, "GET")); } + @Override + public DeleteByQuery createDeleteByQuery() { + return null; + } + @Override + public long performRequest(DeleteByQuery request) throws IOException,ResponseException { + return ((DeleteByQueryImpl)request).extractNumDeleted( + this.performRequest(request.toString(), ((DeleteByQueryImpl)request).query, "POST")); + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java index 83f982e..cf7e725 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java @@ -9,7 +9,7 @@ class SearchImpl implements Search { String json = null; @Override public Search buildAlternativeIds(Collection lids) { - JsonHelper.buildSearchIdsRequest(lids, lids.size(), true); + this.json = JsonHelper.buildSearchIdsRequest(lids, lids.size(), true); return this; } @Override @@ -29,7 +29,7 @@ public Search buildListLdds(String namespace) { } @Override public Search buildTheseIds(Collection lids) { - JsonHelper.buildSearchIdsRequest(lids, lids.size(), true); + this.json = JsonHelper.buildSearchIdsRequest(lids, lids.size(), true); return this; } @Override @@ -46,4 +46,19 @@ public Search setPretty(boolean pretty) { this.pretty = pretty; return this; } + @Override + public Search buildGetField(String field_name, String lidvid) { + this.json = new RegistryRequestBuilder().createGetBlobRequest(field_name, lidvid); + return this; + } + @Override + public Search all(String sortField, int size, String searchAfter) { + this.json = new RegistryRequestBuilder().createExportAllDataRequest(sortField, size, searchAfter); + return this; + } + @Override + public Search all(String filterField, String filterValue, String sortField, int size, String searchAfter) { + this.json = new RegistryRequestBuilder().createExportDataRequest(filterField, filterValue, sortField, size, searchAfter); + return this; + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java index 0996b95..1323083 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java @@ -16,6 +16,28 @@ import gov.nasa.pds.registry.common.es.dao.dd.LddVersions; class SearchRespImpl implements Response.Search { + private static class BatchObjects implements SearchResponseParser.Callback { + final ArrayList content = new ArrayList(); + public void onRecord(String id, Object rec) { + content.add(rec); + } + } + private static class FieldNameFinder implements SearchResponseParser.Callback { + final private String fieldName; + boolean found = false; + String blob = null; + @Override + @SuppressWarnings("rawtypes") + public void onRecord(String id, Object rec) { + found = true; + this.blob = ((Map) rec).get(this.fieldName).toString(); + } + public FieldNameFinder(String fieldName) { + super(); + this.fieldName = fieldName; + } + } + /** * Inner private class to parse LDD information response from Elasticsearch. * @@ -139,4 +161,17 @@ public Set nonExistingIds(Collection from_ids) throws Unsupporte parser.parseResponse(this.response, idsResp); return idsResp.getIds(); } + @Override + public String field(String name) throws NoSuchFieldException { + FieldNameFinder fnf = new FieldNameFinder(name); + if (!fnf.found) throw new NoSuchFieldException(); + return fnf.blob; + } + @Override + public List batch() throws UnsupportedOperationException, IOException { + BatchObjects content = new BatchObjects(); + SearchResponseParser parser = new SearchResponseParser(); + parser.parseResponse(this.response, content); + return content.content; + } } From 2c7d15f28e2406a66d2bb3d7dab09b24113e4a68 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Mon, 29 Apr 2024 12:28:02 -0700 Subject: [PATCH 35/41] moved over bad references from mgr --- .../connection/es/DeleteByQueryImpl.java | 52 +++++ .../common/connection/es/EsQueryUtils.java | 97 +++++++++ .../connection/es/RegistryRequestBuilder.java | 202 ++++++++++++++++++ 3 files changed, 351 insertions(+) create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/DeleteByQueryImpl.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/EsQueryUtils.java create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/es/RegistryRequestBuilder.java diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/DeleteByQueryImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/DeleteByQueryImpl.java new file mode 100644 index 0000000..0b151ad --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/DeleteByQueryImpl.java @@ -0,0 +1,52 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.Map; +import com.google.gson.Gson; +import gov.nasa.pds.registry.common.Request.DeleteByQuery; + +class DeleteByQueryImpl implements DeleteByQuery { + private boolean refresh = false; + private String index = "undefnned"; + String query = ""; + @Override + public DeleteByQuery setIndex(String name) { + this.index = name; + return this; + } + @Override + public DeleteByQuery createFilterQuery(String key, String value) { + this.query = new RegistryRequestBuilder().createFilterQuery(key, value); + return this; + } + @Override + public DeleteByQuery createMatchAllQuery() { + this.query = new RegistryRequestBuilder().createMatchAllQuery(); + return this; + } + @Override + public String toString() { + return "/" + this.index + "/_delete_by_query" + (this.refresh ? "?refresh=true" : ""); + } + @SuppressWarnings("rawtypes") + long extractNumDeleted(org.elasticsearch.client.Response resp) { + try { + InputStream is = resp.getEntity().getContent(); + Reader rd = new InputStreamReader(is); + Gson gson = new Gson(); + Object obj = gson.fromJson(rd, Object.class); + rd.close(); + obj = ((Map) obj).get("deleted"); + return ((Double)obj).longValue(); + } catch (Exception ex) { + return 0; + } + } + @Override + public DeleteByQuery setRefresh(boolean state) { + this.refresh = state; + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsQueryUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsQueryUtils.java new file mode 100644 index 0000000..67a36bf --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsQueryUtils.java @@ -0,0 +1,97 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; + +import com.google.gson.stream.JsonWriter; + +/** + * Helper methods for building Elasticsearch queries. + * + * @author karpenko + */ +class EsQueryUtils +{ + /** + * Append "match_all" object. + * @param writer JSON writer + * @throws IOException an exception + */ + public static void appendMatchAll(JsonWriter writer) throws IOException + { + writer.name("match_all"); + writer.beginObject(); + writer.endObject(); + } + + + /** + * Append match all query. + * @param writer JSON writer + * @throws IOException an exception + */ + public static void appendMatchAllQuery(JsonWriter writer) throws IOException + { + writer.name("query"); + writer.beginObject(); + appendMatchAll(writer); + writer.endObject(); + } + + + /** + * Append filter query + * @param writer JSON writer + * @param field field name + * @param value field value + * @throws IOException an exception + */ + public static void appendFilterQuery(JsonWriter writer, String field, String value) throws IOException + { + writer.name("query"); + writer.beginObject(); + + writer.name("bool"); + writer.beginObject(); + appendMustMatchAll(writer); + appendTermFilter(writer, field, value); + writer.endObject(); + + writer.endObject(); + } + + + /** + * Append must match all criterion + * @param writer JSON writer + * @throws IOException an exception + */ + private static void appendMustMatchAll(JsonWriter writer) throws IOException + { + writer.name("must"); + writer.beginObject(); + appendMatchAll(writer); + writer.endObject(); + } + + + /** + * Append term filter + * @param writer JSON writer + * @param field field name + * @param value field value + * @throws IOException an exception + */ + private static void appendTermFilter(JsonWriter writer, String field, String value) throws IOException + { + writer.name("filter"); + writer.beginObject(); + + writer.name("term"); + writer.beginObject(); + writer.name(field).value(value); + writer.endObject(); + + writer.endObject(); + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RegistryRequestBuilder.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RegistryRequestBuilder.java new file mode 100644 index 0000000..15157af --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RegistryRequestBuilder.java @@ -0,0 +1,202 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; + +import com.google.gson.stream.JsonWriter; + +/** + * A class to build Elasticsearch API JSON requests. + * + * @author karpenko + */ +class RegistryRequestBuilder +{ + private boolean pretty; + + + /** + * Constructor + * @param pretty Pretty-format JSON requests + */ + RegistryRequestBuilder(boolean pretty) + { + this.pretty = pretty; + } + + + /** + * Constructor + */ + public RegistryRequestBuilder() + { + this(false); + } + + + private JsonWriter createJsonWriter(Writer writer) + { + JsonWriter jw = new JsonWriter(writer); + if (pretty) + { + jw.setIndent(" "); + } + + return jw; + } + + /** + * Build export data request + * @param filterField Filter field name, such as "lidvid". + * @param filterValue Filter value. + * @param sortField Sort field is required to paginate data and use "search_after" field. + * @param size Batch / page size + * @param searchAfter "search_after" field to perform pagination + * @return JSON + * @throws IOException an exception + */ + public String createExportDataRequest(String filterField, String filterValue, String sortField, int size, String searchAfter) { + StringWriter out = new StringWriter(); + try (JsonWriter writer = createJsonWriter(out)) { + writer.beginObject(); + // Size (number of records to return) + writer.name("size").value(size); + // Filter query + EsQueryUtils.appendFilterQuery(writer, filterField, filterValue); + // "search_after" parameter is used for pagination + if (searchAfter != null) { + writer.name("search_after").value(searchAfter); + } + // Sort is required by pagination + writer.name("sort"); + writer.beginObject(); + writer.name(sortField).value("asc"); + writer.endObject(); + writer.endObject(); + writer.close(); + } catch (IOException e) { + throw new RuntimeException("Should never get here"); + } + return out.toString(); + } + + + /** + * Build export all data request + * @param sortField Sort field is required to paginate data and use "search_after" field. + * @param size Batch / page size + * @param searchAfter "search_after" field to perform pagination + * @return JSON + * @throws IOException an exception + */ + public String createExportAllDataRequest(String sortField, int size, String searchAfter) { + StringWriter out = new StringWriter(); + try (JsonWriter writer = createJsonWriter(out)) { + writer.beginObject(); + // Size (number of records to return) + writer.name("size").value(size); + // Match all query + EsQueryUtils.appendMatchAllQuery(writer); + // "search_after" parameter is used for pagination + if (searchAfter != null) { + writer.name("search_after"); + writer.beginArray(); + writer.value(searchAfter); + writer.endArray(); + } + // Sort is required by pagination + writer.name("sort"); + writer.beginObject(); + writer.name(sortField).value("asc"); + writer.endObject(); + writer.endObject(); + writer.close(); + } catch (IOException e) { + throw new RuntimeException("Should never get here."); + } + return out.toString(); + } + + + /** + * Build get BLOB request + * @param lidvid a LidVid + * @return JSON + * @throws IOException an exception + */ + public String createGetBlobRequest(String fieldName, String lidvid) + { + StringWriter out = new StringWriter(); + try (JsonWriter writer = createJsonWriter(out)){ + + writer.beginObject(); + + // Return only BLOB + writer.name("_source"); + writer.beginArray(); + writer.value(fieldName); + writer.endArray(); + + // Query + EsQueryUtils.appendFilterQuery(writer, "lidvid", lidvid); + writer.endObject(); + + writer.close(); + } catch (IOException e) { + throw new RuntimeException("Should never get here"); + } + return out.toString(); + } + + + /** + * Create Elasticsearch filter query + * @param field filter field name + * @param value filter value + * @return JSON + * @throws IOException an exception + */ + public String createFilterQuery(String field, String value) + { + StringWriter out = new StringWriter(); + try (JsonWriter writer = createJsonWriter(out)) { + + writer.beginObject(); + EsQueryUtils.appendFilterQuery(writer, field, value); + writer.endObject(); + + writer.close(); + } catch (IOException e) { + throw new RuntimeException("Should never get here"); + } + return out.toString(); + } + + + /** + * Build match all query + * @return JSON + * @throws IOException an exception + */ + public String createMatchAllQuery() + { + StringWriter out = new StringWriter(); + try (JsonWriter writer = createJsonWriter(out)) { + + writer.beginObject(); + + writer.name("query"); + writer.beginObject(); + EsQueryUtils.appendMatchAll(writer); + writer.endObject(); + + writer.endObject(); + + writer.close(); + } catch (IOException e) { + throw new RuntimeException("Should never get here"); + } + return out.toString(); + } +} From 6af2a311611dbf156e994463c438efe9e67790bb Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Mon, 29 Apr 2024 13:21:56 -0700 Subject: [PATCH 36/41] add new functionality to aws --- .../common/connection/aws/DBQImpl.java | 34 +++++++++++++++++++ .../connection/aws/RestClientWrapper.java | 9 +++++ .../common/connection/aws/SearchImpl.java | 29 ++++++++++++++++ .../common/connection/aws/SearchRespWrap.java | 8 +++++ 4 files changed, 80 insertions(+) create mode 100644 src/main/java/gov/nasa/pds/registry/common/connection/aws/DBQImpl.java diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/DBQImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/DBQImpl.java new file mode 100644 index 0000000..ec4324c --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/DBQImpl.java @@ -0,0 +1,34 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.TermQuery; +import org.opensearch.client.opensearch.core.DeleteByQueryRequest; +import gov.nasa.pds.registry.common.Request; + +class DBQImpl implements Request.DeleteByQuery { + final DeleteByQueryRequest.Builder craftsman = new DeleteByQueryRequest.Builder(); + @Override + public Request.DeleteByQuery createFilterQuery(String key, String value) { + this.craftsman.query(new Query.Builder().term(new TermQuery.Builder() + .field(key) + .value(new FieldValue.Builder().stringValue(value).build()) + .build()).build()); + return this; + } + @Override + public Request.DeleteByQuery createMatchAllQuery() { + this.craftsman.query(new Query.Builder().build()); + return this; + } + @Override + public Request.DeleteByQuery setIndex(String name) { + this.craftsman.index(name); + return this; + } + @Override + public Request.DeleteByQuery setRefresh(boolean state) { + this.craftsman.refresh(state); + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java index 5f383ad..1c63cb3 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java @@ -20,6 +20,7 @@ import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.Request.Bulk; import gov.nasa.pds.registry.common.Request.Count; +import gov.nasa.pds.registry.common.Request.DeleteByQuery; import gov.nasa.pds.registry.common.Request.Get; import gov.nasa.pds.registry.common.Request.MGet; import gov.nasa.pds.registry.common.Request.Mapping; @@ -152,4 +153,12 @@ public Response.Search performRequest(Search request) throws IOException, Respon public Response.Settings performRequest(Setting request) throws IOException, ResponseException { return new SettingRespImpl(this.client.indices().getSettings(((SettingImpl)request).craftsman.build())); } + @Override + public DeleteByQuery createDeleteByQuery() { + return new DBQImpl(); + } + @Override + public long performRequest(DeleteByQuery request) throws IOException, ResponseException { + return this.client.deleteByQuery(((DBQImpl)request).craftsman.build()).deleted(); + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java index dd5630c..9aba1de 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java @@ -2,12 +2,16 @@ import java.util.ArrayList; import java.util.Collection; +import org.opensearch.client.opensearch._types.FieldSort; import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.SortOptions; +import org.opensearch.client.opensearch._types.SortOrder; import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; import org.opensearch.client.opensearch._types.query_dsl.IdsQuery; import org.opensearch.client.opensearch._types.query_dsl.MatchQuery; import org.opensearch.client.opensearch._types.query_dsl.Query; import org.opensearch.client.opensearch._types.query_dsl.Query.Builder; +import org.opensearch.client.opensearch._types.query_dsl.TermQuery; import org.opensearch.client.opensearch._types.query_dsl.TermsQuery; import org.opensearch.client.opensearch._types.query_dsl.TermsQueryField; import org.opensearch.client.opensearch.core.SearchRequest; @@ -80,4 +84,29 @@ public Search setPretty(boolean pretty) { // ignored because Java v2 returns a document not JSON return this; } + @Override + public Search all(String sortField, int size, String searchAfter) { + this.craftsman.searchAfter(searchAfter); + this.craftsman.sort(new SortOptions.Builder() + .field(new FieldSort.Builder().field(sortField).order(SortOrder.Asc).build()) + .build()); + return this; + } + @Override + public Search all(String filterField, String filterValue, String sortField, int size, + String searchAfter) { + this.all(sortField, size, searchAfter); + this.buildGetField(filterField, filterValue); + return this; + } + @Override + public Search buildGetField(String field_name, String field_value) { + BoolQuery.Builder journeyman = new BoolQuery.Builder() + .filter(new Query.Builder().term(new TermQuery.Builder() + .field(field_name) + .value(new FieldValue.Builder().stringValue(field_value).build()) + .build()).build()); + this.craftsman.query(new Query.Builder().bool(journeyman.build()).build()); + return this; + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java index 281d9f3..372ad3e 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java @@ -72,4 +72,12 @@ public Set nonExistingIds(Collection from_ids) } return results; } + @Override + public List batch() throws UnsupportedOperationException, IOException { + return this.parent.documents(); + } + @Override + public String field(String name) throws NoSuchFieldException { + return ((Map)this.parent.documents().get(0)).get(name).toString(); + } } From 04a4ae4b070554366ed75fa557acfa23cad303c4 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 18 Jun 2024 13:34:04 -0700 Subject: [PATCH 37/41] updates for ref integrity --- .../java/gov/nasa/pds/registry/common/Request.java | 2 ++ .../java/gov/nasa/pds/registry/common/Response.java | 1 + .../registry/common/connection/aws/SearchImpl.java | 13 +++++++++++++ .../common/connection/aws/SearchRespWrap.java | 8 ++++++++ .../resources/connections/cognito/development.xml | 2 +- 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/Request.java b/src/main/java/gov/nasa/pds/registry/common/Request.java index 844eb5d..b1d9f59 100644 --- a/src/main/java/gov/nasa/pds/registry/common/Request.java +++ b/src/main/java/gov/nasa/pds/registry/common/Request.java @@ -45,9 +45,11 @@ public interface Search { // _search public Search buildLatestLidVids(Collection lids); public Search buildListFields(String dataType); public Search buildListLdds (String namespace); + public Search buildTermQuery (String fieldname, String value); public Search buildTheseIds(Collection lids); public Search setIndex (String name); public Search setPretty (boolean pretty); + public Search setSize (int hitsperpage); } public interface Setting { // _settings public Setting setIndex (String name); diff --git a/src/main/java/gov/nasa/pds/registry/common/Response.java b/src/main/java/gov/nasa/pds/registry/common/Response.java index 18eee60..6dfee55 100644 --- a/src/main/java/gov/nasa/pds/registry/common/Response.java +++ b/src/main/java/gov/nasa/pds/registry/common/Response.java @@ -47,6 +47,7 @@ public interface Mapping { public interface Search { public Map> altIds() throws UnsupportedOperationException, IOException; public List batch() throws UnsupportedOperationException, IOException; + public List> documents(); public String field(String name) throws NoSuchFieldException; // null means blob not in document and NoSuchFieldException document not found public Set fields() throws UnsupportedOperationException, IOException; public List latestLidvids(); // returns null if nothing is found in returned content diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java index 9aba1de..7c256d6 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java @@ -109,4 +109,17 @@ public Search buildGetField(String field_name, String field_value) { this.craftsman.query(new Query.Builder().bool(journeyman.build()).build()); return this; } + @Override + public Search buildTermQuery(String fieldname, String value) { + TermQuery.Builder journeyman = new TermQuery.Builder() + .field(fieldname) + .value(new FieldValue.Builder().stringValue(value).build()); + this.craftsman.query(new Query.Builder().term(journeyman.build()).build()); + return this; + } + @Override + public Search setSize(int hitsperpage) { + this.craftsman.size(hitsperpage); + return this; + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java index 372ad3e..9fe6707 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java @@ -80,4 +80,12 @@ public List batch() throws UnsupportedOperationException, IOException { public String field(String name) throws NoSuchFieldException { return ((Map)this.parent.documents().get(0)).get(name).toString(); } + @Override + public List> documents() { + ArrayList> docs = new ArrayList>(); + for (Object doc : this.parent.documents()) { + docs.add((Map)doc); + } + return docs; + } } diff --git a/src/main/resources/connections/cognito/development.xml b/src/main/resources/connections/cognito/development.xml index a1c777a..f9d8d07 100644 --- a/src/main/resources/connections/cognito/development.xml +++ b/src/main/resources/connections/cognito/development.xml @@ -1,5 +1,5 @@ - + Date: Tue, 18 Jun 2024 14:07:57 -0700 Subject: [PATCH 38/41] fix for -dd-dd-dd --- .../gov/nasa/pds/registry/common/es/service/JsonLddLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/es/service/JsonLddLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/service/JsonLddLoader.java index 3f93cd4..4fd6815 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/service/JsonLddLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/service/JsonLddLoader.java @@ -46,7 +46,7 @@ public JsonLddLoader(DataDictionaryDao dao, ConnectionFactory conFact) throws Ex log = LogManager.getLogger(this.getClass()); dtMap = new Pds2EsDataTypeMap(); - loader = new DataLoader(conFact.setIndexName(conFact.getIndexName() + "-dd")); + loader = new DataLoader(conFact.clone().setIndexName(conFact.getIndexName() + "-dd")); this.dao = dao; } From a7b344f20b6fe2cd427c16e73fdd4753572400ce Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Tue, 18 Jun 2024 14:55:41 -0700 Subject: [PATCH 39/41] sdk1 unimplemented functionality --- .../pds/registry/common/connection/es/SearchImpl.java | 9 +++++++++ .../registry/common/connection/es/SearchRespImpl.java | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java index cf7e725..6394780 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java @@ -1,6 +1,7 @@ package gov.nasa.pds.registry.common.connection.es; import java.util.Collection; +import org.apache.commons.lang3.NotImplementedException; import gov.nasa.pds.registry.common.Request.Search; class SearchImpl implements Search { @@ -61,4 +62,12 @@ public Search all(String filterField, String filterValue, String sortField, int this.json = new RegistryRequestBuilder().createExportDataRequest(filterField, filterValue, sortField, size, searchAfter); return this; } + @Override + public Search buildTermQuery(String fieldname, String value) { + throw new NotImplementedException(); + } + @Override + public Search setSize(int hitsperpage) { + throw new NotImplementedException(); + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java index 1323083..08629b8 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.NotImplementedException; import gov.nasa.pds.registry.common.Response; import gov.nasa.pds.registry.common.es.dao.LatestLidsResponseParser; import gov.nasa.pds.registry.common.es.dao.dd.LddInfo; @@ -174,4 +175,8 @@ public List batch() throws UnsupportedOperationException, IOException { parser.parseResponse(this.response, content); return content.content; } + @Override + public List> documents() { + throw new NotImplementedException(); + } } From a017519e3017a2a383ad357616731f7244193cd7 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 19 Jun 2024 11:11:20 -0700 Subject: [PATCH 40/41] fix not filling -refs index --- .../registry/common/es/service/CollectionInventoryWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/es/service/CollectionInventoryWriter.java b/src/main/java/gov/nasa/pds/registry/common/es/service/CollectionInventoryWriter.java index cee75eb..51c30f0 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/service/CollectionInventoryWriter.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/service/CollectionInventoryWriter.java @@ -46,7 +46,7 @@ public class CollectionInventoryWriter public CollectionInventoryWriter(ConnectionFactory conFact) throws Exception { log = LogManager.getLogger(this.getClass()); - loader = new DataLoader(conFact); + loader = new DataLoader(conFact.clone().setIndexName(conFact.getIndexName()+"-refs")); } From ba0a495828ce46641acb0f0317ba08d02d200d43 Mon Sep 17 00:00:00 2001 From: Al Niessner Date: Wed, 19 Jun 2024 11:29:33 -0700 Subject: [PATCH 41/41] fix the unimplemented portion for harvest --- .../nasa/pds/registry/common/connection/aws/SearchRespWrap.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java index 9fe6707..89056f5 100644 --- a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java @@ -68,7 +68,7 @@ public Set nonExistingIds(Collection from_ids) throws UnsupportedOperationException, IOException { HashSet results = new HashSet(from_ids); for (Hit hit : this.parent.hits().hits()) { - if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + if (from_ids.contains(hit.id())) from_ids.remove(hit.id()); } return results; }