diff --git a/pom.xml b/pom.xml index 7bb2099..2f3f09c 100644 --- a/pom.xml +++ b/pom.xml @@ -116,6 +116,43 @@ POSSIBILITY OF SUCH DAMAGE. tika-core 1.28.3 + + + jakarta.xml.bind + jakarta.xml.bind-api + 4.0.1 + + + com.sun.xml.bind + jaxb-impl + 4.0.4 + + + jakarta.activation + jakarta.activation-api + 2.1.2 + + + + org.apache.httpcomponents.client5 + httpclient5 + 5.3.1 + + + org.opensearch.client + opensearch-java + 2.10.0 + + + software.amazon.awssdk + opensearch + 2.25.31 + + + software.amazon.awssdk + apache-client + 2.25.31 + diff --git a/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java new file mode 100644 index 0000000..6f380cb --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/ConnectionFactory.java @@ -0,0 +1,17 @@ +package gov.nasa.pds.registry.common; + +import org.apache.http.HttpHost; +import org.apache.http.client.CredentialsProvider; + +public interface ConnectionFactory { + public ConnectionFactory clone(); + public RestClient createRestClient() throws Exception; + public CredentialsProvider getCredentials(); + public org.apache.hc.client5.http.auth.CredentialsProvider getCredentials5(); + public HttpHost getHost(); + public org.apache.hc.core5.http.HttpHost getHost5(); + public String getHostName(); + public String getIndexName(); + public boolean isTrustingSelfSigned(); + public ConnectionFactory setIndexName (String idxName); +} diff --git a/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java new file mode 100644 index 0000000..a0ae3a3 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/EstablishConnectionFactory.java @@ -0,0 +1,29 @@ +package gov.nasa.pds.registry.common; + +import java.net.URL; +import gov.nasa.pds.registry.common.connection.AuthContent; +import gov.nasa.pds.registry.common.connection.KnownRegistryConnections; +import gov.nasa.pds.registry.common.connection.UseOpensearchSDK1; +import gov.nasa.pds.registry.common.connection.UseOpensearchSDK2; +import gov.nasa.pds.registry.common.connection.RegistryConnectionContent; + +public class EstablishConnectionFactory { + public static ConnectionFactory from (String urlToRegistryConnection) throws Exception { + return EstablishConnectionFactory.from (urlToRegistryConnection, AuthContent.DEFAULT); + } + public static ConnectionFactory from (String urlToRegistryConnection, String authFile) throws Exception { + return EstablishConnectionFactory.from (urlToRegistryConnection, AuthContent.from(authFile)); + } + private static synchronized ConnectionFactory from (String urlToRegistryConnection, AuthContent auth) throws Exception { + KnownRegistryConnections.initialzeAppHandler(); + RegistryConnectionContent conn = RegistryConnectionContent.from (new URL(urlToRegistryConnection)); + + if (conn.isDirectConnection()) { + if (conn.getServerUrl().getSdk().intValue() == 1) return UseOpensearchSDK1.build(conn.getServerUrl(), auth).setIndexName(conn.getIndex()); + if (conn.getServerUrl().getSdk().intValue() == 2) return UseOpensearchSDK2.build(conn.getServerUrl(), auth).setIndexName(conn.getIndex()); + throw new RuntimeException("The SDK version '" + String.valueOf(conn.getServerUrl().getSdk().intValue()) + "' is not supported"); + } + if (conn.isCognitoConnection()) return UseOpensearchSDK2.build(conn.getCognitoClientId(), auth).setIndexName(conn.getIndex()); + throw new RuntimeException("New XML/Java choices in src/main/resources/registry_connection.xsd that are not handled."); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/Request.java b/src/main/java/gov/nasa/pds/registry/common/Request.java new file mode 100644 index 0000000..b1d9f59 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/Request.java @@ -0,0 +1,57 @@ +package gov.nasa.pds.registry.common; + +import java.util.Collection; +import java.util.List; +import gov.nasa.pds.registry.common.util.Tuple; + +public interface Request { + public interface Bulk { // _bulk + enum Refresh { False, True, WaitFor }; + public void add (String statement, String document); // create, index, update + public Bulk buildUpdateStatus(Collection lidvids, String status); + public Bulk setIndex(String name); + public Bulk setRefresh(Refresh type); + } + public interface Count { // _count + public Count setIndex (String name); + public Count setQuery (String q); + } + public interface DeleteByQuery { // delete_by_query + public DeleteByQuery createFilterQuery(String key, String value); + public DeleteByQuery createMatchAllQuery(); + public DeleteByQuery setIndex (String name); + public DeleteByQuery setRefresh(boolean state); + } + public interface Get { // _doc + public Get excludeField (String field); + public Get excludeFields (List fields); + public Get includeField (String field); + public Get includeFields (List fields); + public Get setId (String id); + public Get setIndex (String index); + } + public interface Mapping { // _mapping + public Mapping buildUpdateFieldSchema (Collection pairs); + public Mapping setIndex(String name); + } + public interface MGet extends Get { // _mget + public MGet setIds (Collection ids); + } + public interface Search { // _search + public Search all(String sortField, int size, String searchAfter); + public Search all(String filterField, String filterValue, String sortField, int size, String searchAfter); + public Search buildAlternativeIds(Collection lids); + public Search buildGetField(String field_name, String lidvid); + public Search buildLatestLidVids(Collection lids); + public Search buildListFields(String dataType); + public Search buildListLdds (String namespace); + public Search buildTermQuery (String fieldname, String value); + public Search buildTheseIds(Collection lids); + public Search setIndex (String name); + public Search setPretty (boolean pretty); + public Search setSize (int hitsperpage); + } + public interface Setting { // _settings + public Setting setIndex (String name); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/Response.java b/src/main/java/gov/nasa/pds/registry/common/Response.java new file mode 100644 index 0000000..6dfee55 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/Response.java @@ -0,0 +1,62 @@ +package gov.nasa.pds.registry.common; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import gov.nasa.pds.registry.common.es.dao.dd.DataTypeNotFoundException; +import gov.nasa.pds.registry.common.es.dao.dd.LddInfo; +import gov.nasa.pds.registry.common.es.dao.dd.LddVersions; +import gov.nasa.pds.registry.common.util.Tuple; + +public interface Response { + public interface Bulk { + public interface Item { + public boolean error(); + public String id(); + public String index(); + public String operation(); + public String reason(); + public String result(); + public int status(); + } + public boolean errors(); + public List items(); + public void logErrors(); + public long took(); + } + public interface CreatedIndex { + public boolean acknowledge(); + public boolean acknowledgeShards(); + public String getIndex(); + } + public interface Get { + public interface IdSets { + public Set lids(); + public Set lidvids(); + } + public List dataTypes(boolean stringForMissing) throws IOException, DataTypeNotFoundException; + public IdSets ids(); // returns null if nothing is found in returned content + public String productClass(); // returns null if product class not in returned content + public List refs(); // returns null if nothing is found in returned content + } + public interface Mapping { + public Set fieldNames(); + } + public interface Search { + public Map> altIds() throws UnsupportedOperationException, IOException; + public List batch() throws UnsupportedOperationException, IOException; + public List> documents(); + public String field(String name) throws NoSuchFieldException; // null means blob not in document and NoSuchFieldException document not found + public Set fields() throws UnsupportedOperationException, IOException; + public List latestLidvids(); // returns null if nothing is found in returned content + public LddVersions lddInfo() throws UnsupportedOperationException, IOException; + public List ldds() throws UnsupportedOperationException, IOException; + public Set nonExistingIds(Collection from_ids) throws UnsupportedOperationException, IOException; + } + public interface Settings { + public int replicas(); + public int shards(); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/ResponseException.java b/src/main/java/gov/nasa/pds/registry/common/ResponseException.java new file mode 100644 index 0000000..d07b8fe --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/ResponseException.java @@ -0,0 +1,10 @@ +package gov.nasa.pds.registry.common; + +import java.io.IOException; + +abstract public class ResponseException extends IOException { + private static final long serialVersionUID = 8629769947735587642L; + abstract public String extractErrorMessage(); + abstract public int statusCode(); // -1 if not known + //abstract public Response getResponse(); +} diff --git a/src/main/java/gov/nasa/pds/registry/common/RestClient.java b/src/main/java/gov/nasa/pds/registry/common/RestClient.java new file mode 100644 index 0000000..e5b8434 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/RestClient.java @@ -0,0 +1,74 @@ +package gov.nasa.pds.registry.common; + +import java.io.Closeable; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.StringWriter; +import java.util.Map; +import java.util.TreeMap; +import com.google.gson.Gson; +import com.google.gson.stream.JsonWriter; +import gov.nasa.pds.registry.common.util.CloseUtils; + +public interface RestClient extends Closeable { + public Request.Bulk createBulkRequest(); + public Request.Count createCountRequest(); + public Request.DeleteByQuery createDeleteByQuery(); + public Request.Get createGetRequest(); + public Request.Mapping createMappingRequest(); + public Request.MGet createMGetRequest(); + public Request.Search createSearchRequest(); + public Request.Setting createSettingRequest(); + public Response.CreatedIndex create (String indexName, String configAsJson) throws IOException,ResponseException; + public void delete (String indexName) throws IOException,ResponseException; + public boolean exists (String indexName) throws IOException,ResponseException; + public Response.Bulk performRequest(Request.Bulk request) throws IOException,ResponseException; + public long performRequest(Request.Count request) throws IOException,ResponseException; + public long performRequest(Request.DeleteByQuery request) throws IOException,ResponseException; + public Response.Get performRequest(Request.Get request) throws IOException,ResponseException; + public Response.Mapping performRequest(Request.Mapping request) throws IOException,ResponseException; + public Response.Search performRequest(Request.Search request) throws IOException,ResponseException; + public Response.Settings performRequest(Request.Setting request) throws IOException,ResponseException; + /** + * Build create index request + * + * @param schemaFile index schema file + * @param shards number of shards + * @param replicas number of replicas + * @return JSON + * @throws Exception Generic exception + */ + @SuppressWarnings({"rawtypes", "unchecked"}) + public static String createCreateIndexRequest(File schemaFile, int shards, int replicas) + throws Exception { + // Read schema template + FileReader rd = new FileReader(schemaFile); + Gson gson = new Gson(); + Object rootObj = gson.fromJson(rd, Object.class); + CloseUtils.close(rd); + Object settingsObj = ((Map) rootObj).get("settings"); + if (settingsObj == null) { + settingsObj = new TreeMap(); + } + Object mappingsObj = ((Map) rootObj).get("mappings"); + if (mappingsObj == null) { + throw new Exception("Missing mappings in schema file " + schemaFile.getAbsolutePath()); + } + StringWriter out = new StringWriter(); + JsonWriter writer = new JsonWriter(out); + writer.beginObject(); + Map settingsMap = (Map) settingsObj; + settingsMap.put("number_of_shards", shards); + settingsMap.put("number_of_replicas", replicas); + // Settings + writer.name("settings"); + gson.toJson(settingsObj, Object.class, writer); + // Mappings + writer.name("mappings"); + gson.toJson(mappingsObj, Object.class, writer); + writer.endObject(); + writer.close(); + return out.toString(); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/app/Handler.java b/src/main/java/gov/nasa/pds/registry/common/app/Handler.java new file mode 100644 index 0000000..2a2a55a --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/app/Handler.java @@ -0,0 +1,32 @@ +package gov.nasa.pds.registry.common.app; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; + +public class Handler extends URLStreamHandler { + @Override + protected void parseURL(URL u, String spec, int start, int limit) { + /* + * Ugly backwards compatibility. Flip any file separator + * characters to be forward slashes. This is a nop on Unix + * and "fixes" win32 file paths. According to RFC 2396, + * only forward slashes may be used to represent hierarchy + * separation in a URL but previous releases unfortunately + * performed this "fixup" behavior in the file URL parsing code + * rather than forcing this to be fixed in the caller of the URL + * class where it belongs. Since backslash is an "unwise" + * character that would normally be encoded if literally intended + * as a non-seperator character the damage of veering away from the + * specification is presumably limited. + */ + super.parseURL(u, spec.replace(File.separatorChar, '/'), start, limit); + } + @Override + protected URLConnection openConnection(URL u) throws IOException { + System.out.println("here"); + return null; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/cfg/RegistryCfg.java b/src/main/java/gov/nasa/pds/registry/common/cfg/RegistryCfg.java deleted file mode 100644 index 6fc610c..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/cfg/RegistryCfg.java +++ /dev/null @@ -1,12 +0,0 @@ -package gov.nasa.pds.registry.common.cfg; - -/** - * Registry (Elasticsearch) configuration - * @author karpenko - */ -public class RegistryCfg -{ - public String url; - public String indexName; - public String authFile; -} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java new file mode 100644 index 0000000..39574ac --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/AuthContent.java @@ -0,0 +1,81 @@ +package gov.nasa.pds.registry.common.connection; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import gov.nasa.pds.registry.common.util.JavaProps; + +public class AuthContent { + final public static AuthContent DEFAULT = new AuthContent(); + final private static Logger LOG = LogManager.getLogger(AuthContent.class); + final private String password; + final private String user; + private String header = null; + + public static AuthContent from (String authfile) throws Exception { + if(authfile == null) { + throw new IllegalArgumentException("null is not an allowable value for authfile. Use AuthContent.DEFAULT in that case."); + } + + JavaProps props = new JavaProps(authfile); + String user = props.getProperty("user"); + String password = props.getProperty("password"); + + if (props.getProperty(ClientConstants.AUTH_TRUST_SELF_SIGNED) != null) { + LOG.warn("The keyword " + ClientConstants.AUTH_TRUST_SELF_SIGNED + + " is no longer used in the authorizaiton file and is being ignored. Please remove " + + ClientConstants.AUTH_TRUST_SELF_SIGNED + + " warning will turn into an error in the future"); + } + + if (user == null || password == null) { + throw new IllegalArgumentException("Must have both 'user' and 'password' in the authorization file: " + authfile); + } + return new AuthContent(password, user); + } + private AuthContent() { + this("admin", "admin"); + } + private AuthContent(String password, String user) { + this.password = password; + this.user = user; + } + public synchronized String getHeader() { + if (this.header == null) { + this.header = "Basic " + Base64.getEncoder().encodeToString((this.getUser() + ":" + this.getPassword()).getBytes(StandardCharsets.UTF_8)); + } + return this.header; + } + public CredentialsProvider getCredentials() { + CredentialsProvider creds; + creds = new BasicCredentialsProvider(); + creds.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(this.getUser(), this.getPassword())); + return creds; + } + public CredentialsProvider getCredentials(HttpHost scope) { + CredentialsProvider creds; + creds = new BasicCredentialsProvider(); + creds.setCredentials(new AuthScope(scope), new UsernamePasswordCredentials(this.getUser(), this.getPassword())); + return creds; + } + public org.apache.hc.client5.http.auth.CredentialsProvider getCredentials5(org.apache.hc.core5.http.HttpHost scope) { + org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider creds = + new org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider(); + //creds.setCredentials(new org.apache.hc.client5.http.auth.AuthScope(scope), + creds.setCredentials(new org.apache.hc.client5.http.auth.AuthScope(null,null,-1,null,null), // ANY + new org.apache.hc.client5.http.auth.UsernamePasswordCredentials(this.getUser(), this.getPassword().toCharArray())); + return creds; + } + public String getPassword() { + return password; + } + public String getUser() { + return user; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/ClientConstants.java b/src/main/java/gov/nasa/pds/registry/common/connection/ClientConstants.java similarity index 66% rename from src/main/java/gov/nasa/pds/registry/common/es/client/ClientConstants.java rename to src/main/java/gov/nasa/pds/registry/common/connection/ClientConstants.java index 2e41f94..c8a5db5 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/ClientConstants.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/ClientConstants.java @@ -1,11 +1,11 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection; /** * Constants used by different classes. * * @author karpenko */ -public class ClientConstants +class ClientConstants { public static final String AUTH_TRUST_SELF_SIGNED = "trust.self-signed"; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/KnownRegistryConnections.java b/src/main/java/gov/nasa/pds/registry/common/connection/KnownRegistryConnections.java new file mode 100644 index 0000000..14f7c3a --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/KnownRegistryConnections.java @@ -0,0 +1,55 @@ +package gov.nasa.pds.registry.common.connection; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.stream.Stream; + +public final class KnownRegistryConnections { + static { initialzeAppHandler(); } + static public void initialzeAppHandler() { + String handlers = System.getProperty("java.protocol.handler.pkgs"); + if (handlers == null) { + handlers = "gov.nasa.pds.registry.common"; + } else if (!handlers.contains("gov.nasa.pds.registry.common")) { + handlers += "|gov.nasa.pds.registry.common"; + } + System.setProperty("java.protocol.handler.pkgs", handlers); + } + static public List list() throws URISyntaxException, IOException { + return KnownRegistryConnections.list(KnownRegistryConnections.class); + } + static public List list(@SuppressWarnings("rawtypes") Class cls) throws URISyntaxException, IOException { + ArrayList result = new ArrayList(); + Path root; + URL resource = cls.getResource("/connections"); + if (resource == null) { + throw new RuntimeException("Resource files are not packaged in the jar file containing class " + cls.getName()); + } + if (resource.getProtocol().equalsIgnoreCase("jar")) { + FileSystem jarfiles = FileSystems.newFileSystem(resource.toURI(), Collections.emptyMap()); + root = jarfiles.getPath("/connections"); + } else { + root = Paths.get(resource.toURI()); + } + try (Stream walk = Files.walk(root, Integer.MAX_VALUE)) { + for (Iterator i = walk.iterator() ; i.hasNext();) { + String candidate = i.next().toString(); + candidate = candidate.substring(candidate.indexOf("/connections")); + if (candidate.endsWith(".xml")) { + result.add(new URL("app:/" + candidate)); + } + } + } + return result; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java b/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java new file mode 100644 index 0000000..442164b --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/RegistryConnectionContent.java @@ -0,0 +1,48 @@ +package gov.nasa.pds.registry.common.connection; + +import java.io.IOException; +import java.net.URL; +import java.util.Arrays; +import org.glassfish.jaxb.runtime.v2.JAXBContextFactory; +import gov.nasa.pds.registry.common.connection.config.CognitoType; +import gov.nasa.pds.registry.common.connection.config.DirectType; +import gov.nasa.pds.registry.common.connection.config.RegistryConnection; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; + +public final class RegistryConnectionContent { + static { KnownRegistryConnections.initialzeAppHandler(); } + public static RegistryConnectionContent from (URL registry_connection) throws IOException, JAXBException { + JAXBContext jaxbContext = new JAXBContextFactory().createContext(new Class[]{RegistryConnection.class}, null); + RegistryConnection xml; + String acceptable[] = { "app", "file", "jar", "http", "https" }; + if (!Arrays.stream(acceptable).anyMatch(registry_connection.getProtocol()::equalsIgnoreCase)) { + throw new IllegalArgumentException("URL protocol must be one of " + acceptable + " not the one specified: " + registry_connection); + } + if (registry_connection.getProtocol().equalsIgnoreCase("app")) { + registry_connection = RegistryConnectionContent.class.getResource + ("/" + registry_connection.getAuthority() + registry_connection.getPath()); + } + xml = (RegistryConnection)jaxbContext.createUnmarshaller().unmarshal(registry_connection); + return new RegistryConnectionContent(xml); + } + final private RegistryConnection content; + private RegistryConnectionContent (RegistryConnection xml) { + this.content = xml; + } + public CognitoType getCognitoClientId() { + return this.content.getCognitoClientId(); + } + public String getIndex() { + return this.content.getIndex(); + } + public DirectType getServerUrl() { + return this.content.getServerUrl(); + } + public boolean isCognitoConnection() { + return this.content.getCognitoClientId() != null; + } + public boolean isDirectConnection() { + return this.content.getServerUrl() != null; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/SSLUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/SSLUtils.java similarity index 90% rename from src/main/java/gov/nasa/pds/registry/common/es/client/SSLUtils.java rename to src/main/java/gov/nasa/pds/registry/common/connection/SSLUtils.java index 193b3ff..eaddc09 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/SSLUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/SSLUtils.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection; import java.security.SecureRandom; @@ -12,7 +12,7 @@ * * @author karpenko */ -public class SSLUtils +class SSLUtils { /** * Create "trust all" SSL context to support self-signed certificates. diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/TrustAllManager.java b/src/main/java/gov/nasa/pds/registry/common/connection/TrustAllManager.java similarity index 88% rename from src/main/java/gov/nasa/pds/registry/common/es/client/TrustAllManager.java rename to src/main/java/gov/nasa/pds/registry/common/connection/TrustAllManager.java index 2c0ecc2..6bce074 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/TrustAllManager.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/TrustAllManager.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection; import java.security.cert.CertificateException; @@ -12,7 +12,7 @@ * * @author karpenko */ -public class TrustAllManager implements X509TrustManager +class TrustAllManager implements X509TrustManager { private X509Certificate[] certs; diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java new file mode 100644 index 0000000..e6c6156 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK1.java @@ -0,0 +1,87 @@ +package gov.nasa.pds.registry.common.connection; + +import java.net.URL; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import org.apache.http.HttpHost; +import org.apache.http.client.CredentialsProvider; +import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.RestClient; +import gov.nasa.pds.registry.common.connection.config.DirectType; +import gov.nasa.pds.registry.common.connection.es.RestClientWrapper; + +public class UseOpensearchSDK1 implements ConnectionFactory { + final private boolean veryTrusting; + final private AuthContent auth; + final private HttpHost host; + final private org.apache.hc.core5.http.HttpHost host5; + final private URL service; + private String index = null; + + public static UseOpensearchSDK1 build (DirectType url, AuthContent auth) throws Exception { + URL service = new URL(url.getValue()); + // Trust self-signed certificates + if(url.isTrustSelfSigned()) + { + SSLContext sslCtx = SSLUtils.createTrustAllContext(); + HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory()); + } + return new UseOpensearchSDK1 (service, auth, url.isTrustSelfSigned()); + } + + private UseOpensearchSDK1 (URL service, AuthContent auth, boolean trustSelfSigned) { + this.auth = auth; + this.host = new HttpHost(service.getHost(), service.getPort(), service.getProtocol()); + this.host5 = new org.apache.hc.core5.http.HttpHost(service.getProtocol(), service.getHost(), service.getPort()); + this.service = service; + this.veryTrusting = trustSelfSigned; + } + @Override + public ConnectionFactory clone() { + return new UseOpensearchSDK1(this.service, this.auth, this.veryTrusting).setIndexName(this.index); + } + @Override + public RestClient createRestClient() throws Exception { + return new RestClientWrapper(this); + } + @Override + public org.apache.hc.client5.http.auth.CredentialsProvider getCredentials5() { + return this.auth.getCredentials5(this.getHost5()); + } + @Override + public HttpHost getHost() { + return this.host; + } + @Override + public String getHostName() { + return this.host.getHostName(); + } + @Override + public org.apache.hc.core5.http.HttpHost getHost5() { + return this.host5; + } + @Override + public String getIndexName() { + return this.index; + } + @Override + public ConnectionFactory setIndexName(String idxName) { + this.index = idxName; + return this; + } + @Override + public String toString() { + String me = "Direct to " + this.service.getProtocol() + "://" + this.service.getHost(); + if (0 <= this.service.getPort()) me += ":" + this.service.getPort(); + me += " using index '" + String.valueOf(this.index) + "'"; + return me; + } + @Override + public CredentialsProvider getCredentials() { + return this.auth.getCredentials(); + } + @Override + public boolean isTrustingSelfSigned() { + return this.veryTrusting; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK2.java b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK2.java new file mode 100644 index 0000000..d61f67c --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/UseOpensearchSDK2.java @@ -0,0 +1,143 @@ +package gov.nasa.pds.registry.common.connection; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.net.URI; +import java.net.URL; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse; +import java.util.Map; +import java.util.Properties; +import org.apache.http.HttpHost; +import org.apache.http.client.CredentialsProvider; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.RestClient; +import gov.nasa.pds.registry.common.connection.aws.RestClientWrapper; +import gov.nasa.pds.registry.common.connection.config.CognitoType; +import gov.nasa.pds.registry.common.connection.config.DirectType; + +public final class UseOpensearchSDK2 implements ConnectionFactory { + final private boolean isServerless; + final private boolean veryTrusting; + final private AuthContent auth; + final private HttpHost host; + final private org.apache.hc.core5.http.HttpHost host5; + final private URL endpoint; + private String index = null; + public static UseOpensearchSDK2 build (CognitoType cog, AuthContent auth) throws IOException, InterruptedException { + boolean expectedContent = true; + Gson gson = new Gson(); + HttpClient client = HttpClient.newHttpClient(); + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(cog.getIDP())) + .POST(BodyPublishers.ofString("{\"AuthFlow\":\"USER_PASSWORD_AUTH\",\"AuthParameters\":{" + + "\"USERNAME\":\"" + auth.getUser() + "\"," + + "\"PASSWORD\":\"" + auth.getPassword() + "\"" + + "},\"ClientId\":\"" + cog.getValue() + "\"" + + "}")) + .setHeader("X-Amz-Target", "AWSCognitoIdentityProviderService.InitiateAuth") + .setHeader("Content-Type", "application/x-amz-json-1.1") + .build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + Map> content; + Properties awsCreds = new Properties(System.getProperties()); // initialize properties as oracle suggests + Type contentType = new TypeToken>>(){}.getType(); + + expectedContent &= response.body().contains("AuthenticationResult"); + expectedContent &= response.body().contains("AccessToken"); + expectedContent &= response.body().contains("ExpiresIn"); + expectedContent &= response.body().contains("IdToken"); + expectedContent &= response.body().contains("RefreshToken"); + expectedContent &= response.body().contains("TokenType"); + expectedContent &= response.body().contains("ChallengeParameters"); + if (!expectedContent) { + throw new IOException("Received an unexpected response of: " + response.toString() + + " ->\n" + response.body()); + } + content = gson.fromJson(response.body(), contentType); + client = HttpClient.newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .build(); + request = HttpRequest.newBuilder() + .uri(URI.create(cog.getGateway())) + .GET() + .setHeader("Authorization", content.get("AuthenticationResult").get("TokenType") + " " + content.get("AuthenticationResult").get("AccessToken")) + .setHeader("IDToken", content.get("AuthenticationResult").get("IdToken")) + .build(); + response = client.send(request, HttpResponse.BodyHandlers.ofString()); + if (299 < response.statusCode()) { + throw new IOException("Could not obtain signed URL: " + response.toString()); + } + expectedContent &= response.body().contains("IdentityId"); + expectedContent &= response.body().contains("Credentials"); + expectedContent &= response.body().contains("AccessKeyId"); + expectedContent &= response.body().contains("SecretKey"); + expectedContent &= response.body().contains("SessionToken"); + expectedContent &= response.body().contains("Expiration"); + expectedContent &= response.body().contains("ResponseMetadata"); + contentType = new TypeToken>(){}.getType(); + content = gson.fromJson(response.body(), contentType); + // fill then set system properties as oracle suggests (init happened above) + awsCreds.setProperty("aws.accessKeyId", content.get("Credentials").get("AccessKeyId")); + awsCreds.setProperty("aws.secretAccessKey", content.get("Credentials").get("SecretKey")); + awsCreds.setProperty("aws.sessionToken", content.get("Credentials").get("SessionToken")); + System.setProperties(awsCreds); + return new UseOpensearchSDK2(auth, new URL(cog.getEndpoint()), true, false); + } + public static UseOpensearchSDK2 build (DirectType url, AuthContent auth) throws Exception { + return new UseOpensearchSDK2(auth, new URL(url.getValue()), false, url.isTrustSelfSigned()); + } + private UseOpensearchSDK2 (AuthContent auth, URL opensearchEndpoint, boolean isServerless, boolean veryTrusting) { + this.auth = auth; + this.endpoint = opensearchEndpoint; + this.host = new HttpHost(this.endpoint.getHost(), this.endpoint.getPort(), this.endpoint.getProtocol()); + this.host5 = new org.apache.hc.core5.http.HttpHost(this.endpoint.getProtocol(), this.endpoint.getHost(), this.endpoint.getPort()); + this.isServerless = isServerless; + this.veryTrusting = veryTrusting; + } + @Override + public ConnectionFactory clone() { + return new UseOpensearchSDK2(this.auth, this.endpoint, this.isServerless, this.veryTrusting).setIndexName(this.index); + } + @Override + public RestClient createRestClient() throws Exception { + return new RestClientWrapper(this, this.isServerless); + } + @Override + public CredentialsProvider getCredentials() { + return this.auth.getCredentials(this.getHost()); + } + @Override + public org.apache.hc.client5.http.auth.CredentialsProvider getCredentials5() { + return this.auth.getCredentials5(this.getHost5()); + } + @Override + public HttpHost getHost() { + return this.host; + } + @Override + public org.apache.hc.core5.http.HttpHost getHost5() { + return this.host5; + } + @Override + public String getHostName() { + return this.host.getHostName(); + } + @Override + public String getIndexName() { + return this.index; + } + @Override + public boolean isTrustingSelfSigned() { + return this.veryTrusting; + } + @Override + public ConnectionFactory setIndexName(String idxName) { + this.index = idxName; + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java new file mode 100644 index 0000000..adca455 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkImpl.java @@ -0,0 +1,108 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.lang.reflect.Type; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.client.opensearch.core.bulk.CreateOperation; +import org.opensearch.client.opensearch.core.bulk.IndexOperation; +import org.opensearch.client.opensearch.core.bulk.UpdateOperation; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import gov.nasa.pds.registry.common.Request.Bulk; +import gov.nasa.pds.registry.common.meta.Metadata; + +class BulkImpl implements Bulk { + final BulkRequest.Builder craftsman = new BulkRequest.Builder(); + final private boolean isServerless; + BulkImpl (boolean isServerless) { + this.isServerless = isServerless; + } + @SuppressWarnings("unchecked") + @Override + public void add(String statement, String document) { + BulkOperation.Builder journeyman = new BulkOperation.Builder(); + Gson gson = new Gson(); + Map> cmd; + Map params; + Object doc = gson.fromJson(document, Object.class); + Type contentType = new TypeToken>>(){}.getType(); + cmd = gson.fromJson(statement, contentType); + + if (cmd.containsKey("create")) { + CreateOperation.Builder apprentice = new CreateOperation.Builder().document(doc); + params = cmd.get("create"); + if (params.containsKey("_id")) { + apprentice.id(params.get("_id")); + } + if (params.containsKey("_index")) { + apprentice.index(params.get("_index")); + } + journeyman.create(apprentice.build()); + } else if (cmd.containsKey("index")) { + IndexOperation.Builder apprentice = new IndexOperation.Builder().document(doc); + params = cmd.get("index"); + if (params.containsKey("_id")) { + apprentice.id(params.get("_id")); + } + if (params.containsKey("_index")) { + apprentice.index(params.get("_index")); + } + journeyman.index(apprentice.build()); + } else if (cmd.containsKey("update")) { + doc = ((Map)doc).get("doc"); + UpdateOperation.Builder apprentice = new UpdateOperation.Builder().document(doc); + params = cmd.get("update"); + if (params.containsKey("_id")) { + apprentice.id(params.get("_id")); + } + if (params.containsKey("_index")) { + apprentice.index(params.get("_index")); + } + journeyman.update(apprentice.build()); + } else { + throw new RuntimeException("Received a statement that did not contain one of the expected keys: create, index, or update."); + } + this.craftsman.operations(journeyman.build()); + } + @Override + public Bulk buildUpdateStatus(Collection lidvids, String status) { + ArrayList updates = new ArrayList(); + Map doc; + for (String lidvid : lidvids) { + doc = new HashMap(); + doc.put(Metadata.FLD_ARCHIVE_STATUS, status); + updates.add(new BulkOperation.Builder().update(new UpdateOperation.Builder>().document(doc).id(lidvid).build()).build()); + } + this.craftsman.operations(updates); + return this; + } + @Override + public Bulk setIndex(String name) { + this.craftsman.index(name); + return this; + } + @Override + public Bulk setRefresh(Refresh type) { + switch (type) { + case False: + this.craftsman.refresh(org.opensearch.client.opensearch._types.Refresh.False); + break; + case True: + this.craftsman.refresh(org.opensearch.client.opensearch._types.Refresh.True); + break; + case WaitFor: + if (this.isServerless) { + // No OP because not supported in serverless realm + } + else { + this.craftsman.refresh(org.opensearch.client.opensearch._types.Refresh.WaitFor); + } + break; + } + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkRespWrap.java new file mode 100644 index 0000000..04c3f30 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/BulkRespWrap.java @@ -0,0 +1,80 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.ArrayList; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.opensearch.core.BulkResponse; +import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; +import gov.nasa.pds.registry.common.Response; + +class BulkRespWrap implements Response.Bulk { + private class ItemWrap implements Response.Bulk.Item { + final BulkResponseItem parent; + ItemWrap (BulkResponseItem parent) { + this.parent = parent; + } + @Override + public String id() { + return this.parent.id(); + } + @Override + public String index() { + return this.parent.index(); + } + @Override + public String result() { + return this.parent.result(); + } + @Override + public int status() { + return this.parent.status(); + } + @Override + public boolean error() { + return this.parent.error() != null; + } + @Override + public String operation() { + return this.parent.operationType().jsonValue(); + } + @Override + public String reason() { + return this.error() ? this.parent.error().reason() : ""; + } + }; + final private ArrayList items = new ArrayList(); + final private BulkResponse parent; + final private Logger log; + BulkRespWrap(BulkResponse parent) { + this.log = LogManager.getLogger(this.getClass()); + this.parent = parent; + } + @Override + public boolean errors() { + return this.parent.errors(); + } + @Override + public synchronized List items() { + if (this.parent.items().size() != this.items.size()) { + for (BulkResponseItem item : this.parent.items()) { + this.items.add(new ItemWrap(item)); + } + } + return this.items; + } + @Override + public long took() { + return this.parent.took(); + } + @Override + public void logErrors() { + if (this.parent.errors()) { + for (BulkResponseItem item : this.parent.items()) { + if (item.error() != null && item.error().reason() != null && !item.error().reason().isBlank()) { + log.error(item.error().reason()); + } + } + } + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/CountImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/CountImpl.java new file mode 100644 index 0000000..9e3f413 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/CountImpl.java @@ -0,0 +1,18 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.opensearch.client.opensearch.core.CountRequest; +import gov.nasa.pds.registry.common.Request.Count; + +class CountImpl implements Count { + final CountRequest.Builder craftsman = new CountRequest.Builder(); + @Override + public Count setIndex(String name) { + this.craftsman.index(name); + return this; + } + @Override + public Count setQuery(String q) { + this.craftsman.q(q); + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexConfigWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexConfigWrap.java new file mode 100644 index 0000000..4113a53 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexConfigWrap.java @@ -0,0 +1,117 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.opensearch.client.opensearch._types.analysis.Analyzer; +import org.opensearch.client.opensearch._types.analysis.CustomAnalyzer; +import org.opensearch.client.opensearch._types.mapping.DynamicMapping; +import org.opensearch.client.opensearch._types.mapping.DynamicTemplate; +import org.opensearch.client.opensearch._types.mapping.Property; +import org.opensearch.client.opensearch._types.mapping.TextProperty; +import org.opensearch.client.opensearch._types.mapping.TypeMapping; +import org.opensearch.client.opensearch.indices.CreateIndexRequest; +import org.opensearch.client.opensearch.indices.IndexSettings; +import org.opensearch.client.opensearch.indices.IndexSettingsAnalysis; +import org.opensearch.client.opensearch.indices.IndexSettingsMapping; +import org.opensearch.client.opensearch.indices.IndexSettingsMappingLimit; +import com.google.gson.Gson; + +@SuppressWarnings("unchecked") // evil but necessary because of JSON heterogeneous structures +class CreateIndexConfigWrap { + static CreateIndexRequest.Builder update (CreateIndexRequest.Builder builder, String withJsonConfig) { + Map config = (Map)new Gson().fromJson(withJsonConfig, Object.class); + for (String pk : config.keySet()) { + if (pk.equalsIgnoreCase("mappings")) updateMappings (builder, (Map)config.get(pk)); + else if (pk.equalsIgnoreCase("settings")) updateSettings (builder, (Map)config.get(pk)); + else throw new RuntimeException("Unknown config key '" + pk + "' requiring fix to JSON or CreateIndexConfigWrap.update()"); + } + return builder; + } + private static void updateAnalysis (IndexSettings.Builder builder, Map analysis) { + IndexSettingsAnalysis.Builder craftsman = new IndexSettingsAnalysis.Builder(); + for (String pk : analysis.keySet()) { + if (pk.equalsIgnoreCase("normalizer")) { + Analyzer.Builder journeyman = new Analyzer.Builder(); + Map analyzers = (Map)analysis.get(pk); + for (String ak : analyzers.keySet()) { + Map analyzer = (Map)analyzers.get(ak); + String atype = (String)analyzer.get("type"); + if (atype.equalsIgnoreCase("custom")) + journeyman.custom(new CustomAnalyzer.Builder().filter((List)analyzer.get("filter")).tokenizer(ak.equalsIgnoreCase("keyword_lowercase") ? "keyword" : ak).build()); + else throw new RuntimeException("Unknown analyzer type '" + atype + "' requiring fix to JSON or CreateIndexConfigWrap.updateAnalysis()"); + } + craftsman.analyzer(pk, journeyman.build()); + } + else throw new RuntimeException("Unknown analysis key '" + pk + "' requiring fix to JSON or CreateIndexConfigWrap.updateAnalysis()"); + } + builder.analysis(craftsman.build()); + } + private static void updateMappings (CreateIndexRequest.Builder builder, Map mappings) { + TypeMapping.Builder craftsman = new TypeMapping.Builder(); + for (String pk : mappings.keySet()) { + if (pk.equalsIgnoreCase("dynamic")) craftsman.dynamic((Boolean)mappings.get(pk) ? DynamicMapping.True : DynamicMapping.False); + else if (pk.equalsIgnoreCase("dynamic_templates")) { + ArrayList> templates = new ArrayList>(); + List> templateList = (List>)mappings.get(pk); + for (Map namedTemplate : templateList) { + HashMap templateMap = new HashMap(); + for (String name : namedTemplate.keySet()) { + DynamicTemplate.Builder journeyman = new DynamicTemplate.Builder(); + Map template = (Map)namedTemplate.get(name); + for (String sk : template.keySet()) { + if (sk.equalsIgnoreCase("mapping")) + journeyman.mapping(PropertyHelper.setType(new Property.Builder(), + ((Map)template.get(sk)).get("type")).build()); + else if (sk.equalsIgnoreCase("match_mapping_type")) journeyman.matchMappingType((String)template.get(sk)); + else throw new RuntimeException("Unknown template key '" + pk + "' requiring fix to JSON or CreateIndexConfigWrap.updateMappings()"); + } + templateMap.put(name, journeyman.build()); + } + templates.add(templateMap); + } + craftsman.dynamicTemplates(templates); + } + else if (pk.equalsIgnoreCase("properties")) { + Map> propertyMap = (Map>)mappings.get(pk); + HashMap properties = new HashMap(); + for (String name : propertyMap.keySet()) { + Property.Builder journeyman = new Property.Builder(); + for (String sk : propertyMap.get(name).keySet()) { + if (sk.equalsIgnoreCase("analyzer")) { + if (propertyMap.get(name).containsKey("type") && propertyMap.get(name).get("type").equalsIgnoreCase("text")) { + journeyman.text(new TextProperty.Builder().analyzer(propertyMap.get(name).get(sk)).build()); + break; + } + } + else if (sk.equalsIgnoreCase("type")) PropertyHelper.setType(journeyman, propertyMap.get(name).get(sk)); + else throw new RuntimeException("Unknown property key '" + pk + "' requiring fix to JSON or CreateIndexConfigWrap.updateMappings()"); + } + properties.put(name, journeyman.build()); + } + craftsman.properties(properties); + } + else throw new RuntimeException("Unknown mapping key '" + pk + "' requiring fix to JSON or CreateIndexConfigWrap.updateMappings()"); + } + builder.mappings(craftsman.build()); + } + private static void updateSettings (CreateIndexRequest.Builder builder, Map settings) { + IndexSettings.Builder craftsman = new IndexSettings.Builder(); + IndexSettingsMapping.Builder journeyman = new IndexSettingsMapping.Builder(); + for (String pk : settings.keySet()) { + if (pk.equalsIgnoreCase("analysis")) updateAnalysis (craftsman, (Map)settings.get(pk)); + else if (pk.equalsIgnoreCase("index.mapping.total_fields.limit")) + journeyman.totalFields(new IndexSettingsMappingLimit.Builder().limit(Long.valueOf((String)settings.get(pk))).build()); + else if (pk.equalsIgnoreCase("index.max_result_window")) + craftsman.maxResultWindow(Integer.valueOf((String)settings.get(pk))); + else if (pk.equalsIgnoreCase("number_of_replicas")) + craftsman.numberOfReplicas(Integer.toString(((Double)settings.get(pk)).intValue())); + else if (pk.equalsIgnoreCase("number_of_shards")) + craftsman.numberOfShards(Integer.toString(((Double)settings.get(pk)).intValue())); + else throw new RuntimeException("Unknown setting key '" + pk + "' requiring fix to JSON or CreateIndexConfigWrap.updateSettings()"); + } + craftsman.mapping(journeyman.build()); + builder.settings(craftsman.build()); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexRespWrap.java new file mode 100644 index 0000000..07f77bb --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/CreateIndexRespWrap.java @@ -0,0 +1,23 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.opensearch.client.opensearch.indices.CreateIndexResponse; +import gov.nasa.pds.registry.common.Response; + +class CreateIndexRespWrap implements Response.CreatedIndex { + final private CreateIndexResponse parent; + CreateIndexRespWrap(CreateIndexResponse parent) { + this.parent = parent; + } + @Override + public boolean acknowledge() { + return this.parent.acknowledged(); + } + @Override + public boolean acknowledgeShards() { + return this.parent.shardsAcknowledged(); + } + @Override + public String getIndex() { + return this.parent.index(); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/DBQImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/DBQImpl.java new file mode 100644 index 0000000..ec4324c --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/DBQImpl.java @@ -0,0 +1,34 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.TermQuery; +import org.opensearch.client.opensearch.core.DeleteByQueryRequest; +import gov.nasa.pds.registry.common.Request; + +class DBQImpl implements Request.DeleteByQuery { + final DeleteByQueryRequest.Builder craftsman = new DeleteByQueryRequest.Builder(); + @Override + public Request.DeleteByQuery createFilterQuery(String key, String value) { + this.craftsman.query(new Query.Builder().term(new TermQuery.Builder() + .field(key) + .value(new FieldValue.Builder().stringValue(value).build()) + .build()).build()); + return this; + } + @Override + public Request.DeleteByQuery createMatchAllQuery() { + this.craftsman.query(new Query.Builder().build()); + return this; + } + @Override + public Request.DeleteByQuery setIndex(String name) { + this.craftsman.index(name); + return this; + } + @Override + public Request.DeleteByQuery setRefresh(boolean state) { + this.craftsman.refresh(state); + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetImpl.java new file mode 100644 index 0000000..ebf58c0 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetImpl.java @@ -0,0 +1,39 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.List; +import org.opensearch.client.opensearch.core.GetRequest; +import gov.nasa.pds.registry.common.Request.Get; + +class GetImpl implements Get { + final GetRequest.Builder craftsman = new GetRequest.Builder(); + @Override + public Get excludeField(String field) { + this.craftsman.sourceExcludes(field); + return this; + } + @Override + public Get excludeFields(List fields) { + this.craftsman.sourceExcludes(fields); + return this; + } + @Override + public Get includeField(String field) { + this.craftsman.sourceIncludes(field); + return this; + } + @Override + public Get includeFields(List fields) { + this.craftsman.sourceIncludes(fields); + return this; + } + @Override + public Get setId(String id) { + this.craftsman.id(id); + return this; + } + @Override + public Get setIndex(String index) { + this.craftsman.index(index); + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetRespWrap.java new file mode 100644 index 0000000..b3fd6a0 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/GetRespWrap.java @@ -0,0 +1,45 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.lang3.NotImplementedException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.client.opensearch.core.GetResponse; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.es.dao.LidvidSet; +import gov.nasa.pds.registry.common.es.dao.dd.DataTypeNotFoundException; +import gov.nasa.pds.registry.common.util.Tuple; + +class GetRespWrap implements Response.Get { + final private GetResponse parent; + final protected Logger log; + GetRespWrap(GetResponse parent) { + this.log = LogManager.getLogger(this.getClass()); + this.parent = parent; + } + @Override + public List dataTypes(boolean stringForMissing) + throws IOException, DataTypeNotFoundException { + throw new RuntimeException("This method is supported via MGet and should never be called here"); + } + @Override + public IdSets ids() { + LidvidSet result = new LidvidSet(null,null); + if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + return result; + } + @Override + public String productClass() { + String result = ""; + if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + return result; + } + @Override + public List refs() { + ArrayList results = new ArrayList(); + if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + return results; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetImpl.java new file mode 100644 index 0000000..95329b1 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetImpl.java @@ -0,0 +1,47 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import org.opensearch.client.opensearch.core.MgetRequest; +import gov.nasa.pds.registry.common.Request.Get; +import gov.nasa.pds.registry.common.Request.MGet; + +class MGetImpl implements MGet { + final MgetRequest.Builder craftsman = new MgetRequest.Builder(); + @Override + public Get excludeField(String field) { + this.craftsman.sourceExcludes(field); + return this; + } + @Override + public Get excludeFields(List fields) { + this.craftsman.sourceExcludes(fields); + return this; + } + @Override + public Get includeField(String field) { + this.craftsman.sourceIncludes(field); + return this; + } + @Override + public Get includeFields(List fields) { + this.craftsman.sourceIncludes(fields); + return this; + } + @Override + public Get setId(String id) { + this.craftsman.ids(id); + return this; + } + @Override + public Get setIndex(String index) { + this.craftsman.index(index); + return this; + } + @Override + public MGet setIds(Collection ids) { + this.craftsman.ids(new ArrayList(ids)); + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetRespWrap.java new file mode 100644 index 0000000..e2054e4 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MGetRespWrap.java @@ -0,0 +1,41 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.opensearch.client.opensearch.core.MgetResponse; +import org.opensearch.client.opensearch.core.mget.MultiGetResponseItem; +import gov.nasa.pds.registry.common.es.dao.dd.DataTypeNotFoundException; +import gov.nasa.pds.registry.common.util.Tuple; + +class MGetRespWrap extends GetRespWrap { + final private MgetResponse parent; + MGetRespWrap (MgetResponse parent) { + super(null); + this.parent = parent; + } + @Override + public List dataTypes(boolean stringForMissing) throws IOException, DataTypeNotFoundException { + ArrayList results = new ArrayList(); + for (MultiGetResponseItem doc : this.parent.docs()) { + Tuple t = null; + if (doc.isResult()) { + @SuppressWarnings("unchecked") + Map src = (Map)doc.result().source(); + if (src != null && src.containsKey("es_data_type")) t = new Tuple(doc.result().id(), src.get("es_data_type")); + } else if (stringForMissing) { + if (doc.result().id().startsWith("ref_lid_") || doc.result().id().startsWith("ref_lidvid_") + || doc.result().id().endsWith("_Area")) { + t = new Tuple(doc.result().id(), "keyword"); + } else { + log.error("Could not find datatype for field " + doc.result().id()); + } + } else { + throw new DataTypeNotFoundException(); + } + if (t != null) results.add(t); + } + return results; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java new file mode 100644 index 0000000..92d429e --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingImpl.java @@ -0,0 +1,37 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.Collection; +import java.util.HashMap; +import org.opensearch.client.opensearch._types.mapping.Property; +import org.opensearch.client.opensearch.indices.GetMappingRequest; +import org.opensearch.client.opensearch.indices.PutMappingRequest; +import gov.nasa.pds.registry.common.Request.Mapping; +import gov.nasa.pds.registry.common.util.Tuple; + +class MappingImpl implements Mapping { + boolean isGet = true; + final GetMappingRequest.Builder craftsman_get = new GetMappingRequest.Builder(); + final PutMappingRequest.Builder craftsman_set = new PutMappingRequest.Builder(); + @Override + public Mapping buildUpdateFieldSchema(Collection pairs) { + HashMap mapping = new HashMap(); + for (Tuple t : pairs) { + Property.Builder journeyman = new Property.Builder(); + String fieldName = t.item1; + String fieldType = t.item2; + PropertyHelper.setType(journeyman, fieldType); + mapping.put(fieldName, journeyman.build()); + } + this.craftsman_set.properties(mapping); + this.isGet = false; + return this; + } + + @Override + public Mapping setIndex(String name) { + this.craftsman_get.index(name); + this.craftsman_set.index(name); + return this; + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingRespImpl.java new file mode 100644 index 0000000..d4f2a86 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/MappingRespImpl.java @@ -0,0 +1,21 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.HashSet; +import java.util.Set; +import org.opensearch.client.opensearch.indices.GetMappingResponse; +import org.opensearch.client.opensearch.indices.PutMappingResponse; +import gov.nasa.pds.registry.common.Response; + +class MappingRespImpl implements Response.Mapping { + final Set fieldNames; + MappingRespImpl (GetMappingResponse response) { + this.fieldNames = response.result().keySet(); + } + MappingRespImpl (PutMappingResponse response) { + this.fieldNames = new HashSet(); + } + @Override + public Set fieldNames() { + return new HashSet(this.fieldNames); // make a copy so data is never corrupted + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/PropertyHelper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/PropertyHelper.java new file mode 100644 index 0000000..426d281 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/PropertyHelper.java @@ -0,0 +1,49 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.opensearch.client.opensearch._types.mapping.BinaryProperty; +import org.opensearch.client.opensearch._types.mapping.BooleanProperty; +import org.opensearch.client.opensearch._types.mapping.DateProperty; +import org.opensearch.client.opensearch._types.mapping.DoubleNumberProperty; +import org.opensearch.client.opensearch._types.mapping.FloatNumberProperty; +import org.opensearch.client.opensearch._types.mapping.IntegerNumberProperty; +import org.opensearch.client.opensearch._types.mapping.KeywordProperty; +import org.opensearch.client.opensearch._types.mapping.LongNumberProperty; +import org.opensearch.client.opensearch._types.mapping.Property; +import org.opensearch.client.opensearch._types.mapping.TextProperty; + +final class PropertyHelper { + static Property.Builder setType (Property.Builder builder, String fieldType) { + switch (fieldType) { + case "binary": + builder.binary(new BinaryProperty.Builder().build()); + break; + case "boolean": + builder.boolean_(new BooleanProperty.Builder().build()); + break; + case "date": + builder.date(new DateProperty.Builder().build()); + break; + case "double": + builder.double_(new DoubleNumberProperty.Builder().build()); + break; + case "float": + builder.float_(new FloatNumberProperty.Builder().build()); + break; + case "integer": + builder.integer(new IntegerNumberProperty.Builder().build()); + break; + case "keyword": + builder.keyword(new KeywordProperty.Builder().build()); + break; + case "long": + builder.long_(new LongNumberProperty.Builder().build()); + break; + case "text": + builder.text(new TextProperty.Builder().build()); + break; + default: + throw new RuntimeException("Cannot map type '" + fieldType + "' yet. Please review PropertyHelper.setType() code and fix."); + } + return builder; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java new file mode 100644 index 0000000..1c63cb3 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/RestClientWrapper.java @@ -0,0 +1,164 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.io.IOException; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import javax.net.ssl.SSLContext; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManager; +import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; +import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.opensearch.indices.CreateIndexRequest; +import org.opensearch.client.opensearch.indices.DeleteIndexRequest; +import org.opensearch.client.opensearch.indices.ExistsRequest; +import org.opensearch.client.transport.aws.AwsSdk2Transport; +import org.opensearch.client.transport.aws.AwsSdk2TransportOptions; +import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; +import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.Request.Bulk; +import gov.nasa.pds.registry.common.Request.Count; +import gov.nasa.pds.registry.common.Request.DeleteByQuery; +import gov.nasa.pds.registry.common.Request.Get; +import gov.nasa.pds.registry.common.Request.MGet; +import gov.nasa.pds.registry.common.Request.Mapping; +import gov.nasa.pds.registry.common.Request.Search; +import gov.nasa.pds.registry.common.Request.Setting; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.http.apache.ApacheHttpClient; +import software.amazon.awssdk.regions.Region; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.ResponseException; +import gov.nasa.pds.registry.common.RestClient; + +public class RestClientWrapper implements RestClient { + final private boolean isServerless; + final private OpenSearchClient client; + final private SdkHttpClient httpClient; + public RestClientWrapper(ConnectionFactory conFact, boolean isServerless) { + this.httpClient = ApacheHttpClient.builder().build(); + this.isServerless = isServerless; + if (isServerless) { + this.client = new OpenSearchClient( + new AwsSdk2Transport( + httpClient, + conFact.getHostName(), + "aoss", + Region.US_WEST_2, // signing service region that we should probably get from host name?? + AwsSdk2TransportOptions.builder().build() + ) + ); + } else { + OpenSearchClient localClient = null; + try { + SSLContext sslcontext = SSLContextBuilder + .create() + .loadTrustMaterial((chains, authType) -> true) + .build(); + final ApacheHttpClient5TransportBuilder builder = ApacheHttpClient5TransportBuilder.builder(conFact.getHost5()); + builder.setHttpClientConfigCallback(httpClientBuilder -> { + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() + .setSslContext(sslcontext) + .build(); + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder + .create() + .setTlsStrategy(tlsStrategy) + .build(); + return httpClientBuilder + .setDefaultCredentialsProvider(conFact.getCredentials5()) + .setConnectionManager(connectionManager); + }); + localClient = new OpenSearchClient(builder.build()); + } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + finally { + this.client = localClient; + } + } + } + @Override + public void close() throws IOException { + this.client.shutdown(); + this.httpClient.close(); + } + @Override + public Bulk createBulkRequest() { + return new BulkImpl(this.isServerless); + } + @Override + public Count createCountRequest() { + return new CountImpl(); + } + @Override + public Get createGetRequest() { + return new GetImpl(); + } + @Override + public Mapping createMappingRequest() { + return new MappingImpl(); + } + @Override + public MGet createMGetRequest() { + return new MGetImpl(); + } + @Override + public Search createSearchRequest() { + return new SearchImpl(); + } + @Override + public Setting createSettingRequest() { + return new SettingImpl(); + } + @Override + public Response.CreatedIndex create(String indexName, String configAsJson) throws IOException, ResponseException { + return new CreateIndexRespWrap(this.client.indices().create(CreateIndexConfigWrap.update(new CreateIndexRequest.Builder(), configAsJson).index(indexName).build())); + } + @Override + public void delete(String indexName) throws IOException, ResponseException { + this.client.indices().delete(new DeleteIndexRequest.Builder().index(indexName).build()); + } + @Override + public boolean exists(String indexName) throws IOException, ResponseException { + return this.client.indices().exists(new ExistsRequest.Builder().index(indexName).build()).value(); + } + @Override + public Response.Bulk performRequest(Bulk request) throws IOException, ResponseException { + return new BulkRespWrap(this.client.bulk(((BulkImpl)request).craftsman.build())); + } + @Override + public long performRequest(Count request) throws IOException, ResponseException { + return this.client.count(((CountImpl)request).craftsman.build()).count(); + } + @Override + public Response.Get performRequest(Get request) throws IOException, ResponseException { + if (request instanceof MGet) + return new MGetRespWrap(this.client.mget(((MGetImpl)request).craftsman.build(), Object.class)); + return new GetRespWrap(this.client.get(((GetImpl)request).craftsman.build(), Object.class)); + } + @Override + public Response.Mapping performRequest(Mapping request) throws IOException, ResponseException { + MappingImpl req = (MappingImpl)request; + return req.isGet ? new MappingRespImpl(this.client.indices().getMapping(req.craftsman_get.build())) : + new MappingRespImpl(this.client.indices().putMapping(req.craftsman_set.build())); + } + @Override + public Response.Search performRequest(Search request) throws IOException, ResponseException { + return new SearchRespWrap(this.client.search(((SearchImpl)request).craftsman.build(), Object.class)); + } + @Override + public Response.Settings performRequest(Setting request) throws IOException, ResponseException { + return new SettingRespImpl(this.client.indices().getSettings(((SettingImpl)request).craftsman.build())); + } + @Override + public DeleteByQuery createDeleteByQuery() { + return new DBQImpl(); + } + @Override + public long performRequest(DeleteByQuery request) throws IOException, ResponseException { + return this.client.deleteByQuery(((DBQImpl)request).craftsman.build()).deleted(); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java new file mode 100644 index 0000000..7c256d6 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchImpl.java @@ -0,0 +1,125 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.util.ArrayList; +import java.util.Collection; +import org.opensearch.client.opensearch._types.FieldSort; +import org.opensearch.client.opensearch._types.FieldValue; +import org.opensearch.client.opensearch._types.SortOptions; +import org.opensearch.client.opensearch._types.SortOrder; +import org.opensearch.client.opensearch._types.query_dsl.BoolQuery; +import org.opensearch.client.opensearch._types.query_dsl.IdsQuery; +import org.opensearch.client.opensearch._types.query_dsl.MatchQuery; +import org.opensearch.client.opensearch._types.query_dsl.Query; +import org.opensearch.client.opensearch._types.query_dsl.Query.Builder; +import org.opensearch.client.opensearch._types.query_dsl.TermQuery; +import org.opensearch.client.opensearch._types.query_dsl.TermsQuery; +import org.opensearch.client.opensearch._types.query_dsl.TermsQueryField; +import org.opensearch.client.opensearch.core.SearchRequest; +import org.opensearch.client.opensearch.core.search.SourceConfig; +import org.opensearch.client.opensearch.core.search.SourceFilter; +import gov.nasa.pds.registry.common.Request.Search; + +class SearchImpl implements Search { + final SearchRequest.Builder craftsman = new SearchRequest.Builder(); + private void buildIds (Collection lids, boolean alt) { + SourceConfig.Builder journeyman = new SourceConfig.Builder(); + if (alt) { + journeyman.filter(new SourceFilter.Builder().includes("alternate_ids").build()); + } + this.craftsman.query(new Query.Builder().ids(new IdsQuery.Builder().values(new ArrayList(lids)).build()).build()); + this.craftsman.size(lids.size()); + this.craftsman.source(journeyman.fetch(alt).build()); + } + private Query.Builder matchQuery (String fieldname, String fieldvalue) { + return (Builder)new Query.Builder().match(new MatchQuery.Builder().field(fieldname).query(new FieldValue.Builder().stringValue(fieldvalue).build()).build()); + } + @Override + public Search buildAlternativeIds(Collection lids) { + this.buildIds(lids, true); + return this; + } + @Override + public Search buildLatestLidVids(Collection lids) { + ArrayList terms = new ArrayList(lids.size()); + for (String lid : lids) { + terms.add(new FieldValue.Builder().stringValue(lid).build()); + } + // FIXME: need to work out aggregates + // this.craftsman.aggregations("latest", ); + this.craftsman.query(new Query.Builder().terms(new TermsQuery.Builder().field("lid").terms(new TermsQueryField.Builder().value(terms).build()).build()).build()); + this.craftsman.size(0); + this.craftsman.source(new SourceConfig.Builder().fetch(false).build()); + return this; + } + @Override + public Search buildListFields(String dataType) { + this.craftsman.query(new Query.Builder().bool(new BoolQuery.Builder().must(this.matchQuery("es_data_type", dataType).build()).build()).build()); + this.craftsman.size(1000); // have no idea why hardcoded but it is (.es.JsonHelper:217 + this.craftsman.source(new SourceConfig.Builder().filter(new SourceFilter.Builder().includes("es_field_name").build()).build()); + return this; + } + @Override + public Search buildListLdds(String namespace) { + BoolQuery.Builder journeyman = new BoolQuery.Builder() + .must(this.matchQuery("class_ns", "registry").build(), + this.matchQuery("class_name", "LDD_Info").build(), + this.matchQuery("attr_ns", namespace).build()); + this.craftsman.query(new Query.Builder().bool(journeyman.build()).build()); + this.craftsman.size(1000); // have no idea why hardcoded but it is (.es.JsonHelper:265 + this.craftsman.source(new SourceConfig.Builder().filter(new SourceFilter.Builder().includes("date", "attr_name").build()).build()); + return this; + } + @Override + public Search buildTheseIds(Collection lids) { + this.buildIds(lids, false); + return this; + } + @Override + public Search setIndex(String name) { + this.craftsman.index(name); + return this; + } + @Override + public Search setPretty(boolean pretty) { + // ignored because Java v2 returns a document not JSON + return this; + } + @Override + public Search all(String sortField, int size, String searchAfter) { + this.craftsman.searchAfter(searchAfter); + this.craftsman.sort(new SortOptions.Builder() + .field(new FieldSort.Builder().field(sortField).order(SortOrder.Asc).build()) + .build()); + return this; + } + @Override + public Search all(String filterField, String filterValue, String sortField, int size, + String searchAfter) { + this.all(sortField, size, searchAfter); + this.buildGetField(filterField, filterValue); + return this; + } + @Override + public Search buildGetField(String field_name, String field_value) { + BoolQuery.Builder journeyman = new BoolQuery.Builder() + .filter(new Query.Builder().term(new TermQuery.Builder() + .field(field_name) + .value(new FieldValue.Builder().stringValue(field_value).build()) + .build()).build()); + this.craftsman.query(new Query.Builder().bool(journeyman.build()).build()); + return this; + } + @Override + public Search buildTermQuery(String fieldname, String value) { + TermQuery.Builder journeyman = new TermQuery.Builder() + .field(fieldname) + .value(new FieldValue.Builder().stringValue(value).build()); + this.craftsman.query(new Query.Builder().term(journeyman.build()).build()); + return this; + } + @Override + public Search setSize(int hitsperpage) { + this.craftsman.size(hitsperpage); + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java new file mode 100644 index 0000000..89056f5 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SearchRespWrap.java @@ -0,0 +1,91 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.NotImplementedException; +import org.opensearch.client.opensearch.core.SearchResponse; +import org.opensearch.client.opensearch.core.search.Hit; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.es.dao.dd.LddInfo; +import gov.nasa.pds.registry.common.es.dao.dd.LddVersions; + +@SuppressWarnings("unchecked") // JSON heterogenous structures requires raw casting +class SearchRespWrap implements Response.Search { + final private SearchResponse parent; + SearchRespWrap(SearchResponse parent) { + this.parent = parent; + } + @Override + public Map> altIds() throws UnsupportedOperationException, IOException { + HashMap> results = new HashMap>(); + if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + return results; + } + @Override + public Set fields() throws UnsupportedOperationException, IOException { + Set results = new HashSet(); + for (Hit hit : this.parent.hits().hits()) { + for (String value : ((Map)hit.source()).values()) { + results.add(value); + } + } + return results; + } + @Override + public List latestLidvids() { + ArrayList lidvids = new ArrayList(); + if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + return lidvids; + } + @Override + public LddVersions lddInfo() throws UnsupportedOperationException, IOException { + LddVersions result = new LddVersions(); + for (Hit hit : this.parent.hits().hits()) { + Map source = (Map)hit.source(); + if (source.containsKey("attr_name") && source.containsKey("date")) { + result.addSchemaFile(source.get("attr_name")); + result.updateDate(source.get("date")); + } else { + throw new UnsupportedOperationException("Either date or attr_name or both are missing from hit."); + } + } + return result; + } + @Override + public List ldds() throws UnsupportedOperationException, IOException { + ArrayList results = new ArrayList(); + if (true) throw new NotImplementedException("Need to fill this out when have a return value"); + return results; + } + @Override + public Set nonExistingIds(Collection from_ids) + throws UnsupportedOperationException, IOException { + HashSet results = new HashSet(from_ids); + for (Hit hit : this.parent.hits().hits()) { + if (from_ids.contains(hit.id())) from_ids.remove(hit.id()); + } + return results; + } + @Override + public List batch() throws UnsupportedOperationException, IOException { + return this.parent.documents(); + } + @Override + public String field(String name) throws NoSuchFieldException { + return ((Map)this.parent.documents().get(0)).get(name).toString(); + } + @Override + public List> documents() { + ArrayList> docs = new ArrayList>(); + for (Object doc : this.parent.documents()) { + docs.add((Map)doc); + } + return docs; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java new file mode 100644 index 0000000..37b7146 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingImpl.java @@ -0,0 +1,13 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.opensearch.client.opensearch.indices.GetIndicesSettingsRequest; +import gov.nasa.pds.registry.common.Request.Setting; + +class SettingImpl implements Setting { + final GetIndicesSettingsRequest.Builder craftsman = new GetIndicesSettingsRequest.Builder(); + @Override + public Setting setIndex(String name) { + this.craftsman.index(name); + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingRespImpl.java new file mode 100644 index 0000000..cbf9911 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/aws/SettingRespImpl.java @@ -0,0 +1,29 @@ +package gov.nasa.pds.registry.common.connection.aws; + +import org.opensearch.client.opensearch.indices.GetIndicesSettingsResponse; +import org.opensearch.client.opensearch.indices.IndexState; +import gov.nasa.pds.registry.common.Response; + +class SettingRespImpl implements Response.Settings { + final private int replicas; + final private int shards; + SettingRespImpl (GetIndicesSettingsResponse response) { + int r = -1, s = -1; + for (IndexState state : response.result().values()) { + // should only ever be one + r = Integer.valueOf(state.settings().numberOfReplicas()); + s = Integer.valueOf(state.settings().numberOfShards()); + } + this.replicas = r; + this.shards = s; + } + @Override + public int replicas() { + return this.replicas; + } + + @Override + public int shards() { + return this.shards; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/config/CognitoType.java b/src/main/java/gov/nasa/pds/registry/common/connection/config/CognitoType.java new file mode 100644 index 0000000..d7834c9 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/config/CognitoType.java @@ -0,0 +1,183 @@ +// +// This file was generated by the Eclipse Implementation of JAXB, v4.0.3 +// See https://eclipse-ee4j.github.io/jaxb-ri +// Any modifications to this file will be lost upon recompilation of the source schema. +// + + +package gov.nasa.pds.registry.common.connection.config; + +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlSchemaType; +import jakarta.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlValue; +import jakarta.xml.bind.annotation.adapters.NormalizedStringAdapter; +import jakarta.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + + +/** + * + * Currently, cognito to serverless opensearh requires three bits of + * information: + * 1. clientID: magic ID which must be given and is not a secret but + * sure looks like one + * 3. endpoint: the endpoint for serverless opensearch + * 2. @gateway: an endpoint that exchanges the cognito authentication + * to an IAM role used for access control with serverless + * opensearch on AWS + * 3. @IDP: the cognito authentication endpoint that exchanges the + * username/password to a set of tokens used by the gateway. + * + * + *

Java class for cognito_type complex type. + * + *

The following schema fragment specifies the expected content contained within this class. + * + *

{@code
+ * 
+ *   
+ *     
+ *       
+ *       
+ *       
+ *     
+ *   
+ * 
+ * }
+ * + * + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "cognito_type", propOrder = { + "value" +}) +public class CognitoType { + + @XmlValue + @XmlJavaTypeAdapter(NormalizedStringAdapter.class) + @XmlSchemaType(name = "normalizedString") + protected String value; + @XmlAttribute(name = "endpoint") + @XmlJavaTypeAdapter(NormalizedStringAdapter.class) + @XmlSchemaType(name = "normalizedString") + protected String endpoint; + @XmlAttribute(name = "gateway") + @XmlJavaTypeAdapter(NormalizedStringAdapter.class) + @XmlSchemaType(name = "normalizedString") + protected String gateway; + @XmlAttribute(name = "IDP") + @XmlJavaTypeAdapter(NormalizedStringAdapter.class) + @XmlSchemaType(name = "normalizedString") + protected String idp; + + /** + * Gets the value of the value property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getValue() { + return value; + } + + /** + * Sets the value of the value property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setValue(String value) { + this.value = value; + } + + /** + * Gets the value of the endpoint property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getEndpoint() { + if (endpoint == null) { + return "https://p5qmxrldysl1gy759hqf.us-west-2.aoss.amazonaws.com"; + } else { + return endpoint; + } + } + + /** + * Sets the value of the endpoint property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setEndpoint(String value) { + this.endpoint = value; + } + + /** + * Gets the value of the gateway property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getGateway() { + if (gateway == null) { + return "https://c8u1zk30u5.execute-api.us-west-2.amazonaws.com/dev/signed-url"; + } else { + return gateway; + } + } + + /** + * Sets the value of the gateway property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setGateway(String value) { + this.gateway = value; + } + + /** + * Gets the value of the idp property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getIDP() { + if (idp == null) { + return "cognito-idp.us-west-2.amazonaws.com"; + } else { + return idp; + } + } + + /** + * Sets the value of the idp property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setIDP(String value) { + this.idp = value; + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/config/DirectType.java b/src/main/java/gov/nasa/pds/registry/common/connection/config/DirectType.java new file mode 100644 index 0000000..2228301 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/config/DirectType.java @@ -0,0 +1,150 @@ +// +// This file was generated by the Eclipse Implementation of JAXB, v4.0.3 +// See https://eclipse-ee4j.github.io/jaxb-ri +// Any modifications to this file will be lost upon recompilation of the source schema. +// + + +package gov.nasa.pds.registry.common.connection.config; + +import java.math.BigInteger; +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlSchemaType; +import jakarta.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.XmlValue; +import jakarta.xml.bind.annotation.adapters.NormalizedStringAdapter; +import jakarta.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + + +/** + * + * Currently, connecting to an opensearch service directly requires just + * two pieces of information: + * 1. URL: the URL to directly contact an opensearch instance and its + * API to include authentication + * 2. @trustSelfSigned: when running locally for testing or development + * it is often necessary to use self signed + * certificates. Setting this option to true + * will all for self signed certificates. + * + * An optional 3rd piece of information @sdk is the Java SDK version to use + * for the connection. The traditional SDK is 1. The newer SDK 2 was + * introduced with the serverless opensearch development and beyond can + * also be used. The default is 2 as we are deprecating the use of 1. + * + * + *

Java class for direct_type complex type. + * + *

The following schema fragment specifies the expected content contained within this class. + * + *

{@code
+ * 
+ *   
+ *     
+ *       
+ *       
+ *     
+ *   
+ * 
+ * }
+ * + * + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "direct_type", propOrder = { + "value" +}) +public class DirectType { + + @XmlValue + @XmlJavaTypeAdapter(NormalizedStringAdapter.class) + @XmlSchemaType(name = "normalizedString") + protected String value; + @XmlAttribute(name = "sdk") + protected BigInteger sdk; + @XmlAttribute(name = "trust_self_signed") + protected Boolean trustSelfSigned; + + /** + * Gets the value of the value property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getValue() { + return value; + } + + /** + * Sets the value of the value property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setValue(String value) { + this.value = value; + } + + /** + * Gets the value of the sdk property. + * + * @return + * possible object is + * {@link BigInteger } + * + */ + public BigInteger getSdk() { + if (sdk == null) { + return new BigInteger("2"); + } else { + return sdk; + } + } + + /** + * Sets the value of the sdk property. + * + * @param value + * allowed object is + * {@link BigInteger } + * + */ + public void setSdk(BigInteger value) { + this.sdk = value; + } + + /** + * Gets the value of the trustSelfSigned property. + * + * @return + * possible object is + * {@link Boolean } + * + */ + public boolean isTrustSelfSigned() { + if (trustSelfSigned == null) { + return false; + } else { + return trustSelfSigned; + } + } + + /** + * Sets the value of the trustSelfSigned property. + * + * @param value + * allowed object is + * {@link Boolean } + * + */ + public void setTrustSelfSigned(Boolean value) { + this.trustSelfSigned = value; + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/config/ObjectFactory.java b/src/main/java/gov/nasa/pds/registry/common/connection/config/ObjectFactory.java new file mode 100644 index 0000000..f25274b --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/config/ObjectFactory.java @@ -0,0 +1,68 @@ +// +// This file was generated by the Eclipse Implementation of JAXB, v4.0.3 +// See https://eclipse-ee4j.github.io/jaxb-ri +// Any modifications to this file will be lost upon recompilation of the source schema. +// + + +package gov.nasa.pds.registry.common.connection.config; + +import jakarta.xml.bind.annotation.XmlRegistry; + + +/** + * This object contains factory methods for each + * Java content interface and Java element interface + * generated in the gov.nasa.pds.registry.common.connection.config package. + *

An ObjectFactory allows you to programmatically + * construct new instances of the Java representation + * for XML content. The Java representation of XML + * content can consist of schema derived interfaces + * and classes representing the binding of schema + * type definitions, element declarations and model + * groups. Factory methods for each of these are + * provided in this class. + * + */ +@XmlRegistry +public class ObjectFactory { + + + /** + * Create a new ObjectFactory that can be used to create new instances of schema derived classes for package: gov.nasa.pds.registry.common.connection.config + * + */ + public ObjectFactory() { + } + + /** + * Create an instance of {@link RegistryConnection } + * + * @return + * the new instance of {@link RegistryConnection } + */ + public RegistryConnection createRegistryConnection() { + return new RegistryConnection(); + } + + /** + * Create an instance of {@link CognitoType } + * + * @return + * the new instance of {@link CognitoType } + */ + public CognitoType createCognitoType() { + return new CognitoType(); + } + + /** + * Create an instance of {@link DirectType } + * + * @return + * the new instance of {@link DirectType } + */ + public DirectType createDirectType() { + return new DirectType(); + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/config/RegistryConnection.java b/src/main/java/gov/nasa/pds/registry/common/connection/config/RegistryConnection.java new file mode 100644 index 0000000..3fe0462 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/config/RegistryConnection.java @@ -0,0 +1,142 @@ +// +// This file was generated by the Eclipse Implementation of JAXB, v4.0.3 +// See https://eclipse-ee4j.github.io/jaxb-ri +// Any modifications to this file will be lost upon recompilation of the source schema. +// + + +package gov.nasa.pds.registry.common.connection.config; + +import jakarta.xml.bind.annotation.XmlAccessType; +import jakarta.xml.bind.annotation.XmlAccessorType; +import jakarta.xml.bind.annotation.XmlAttribute; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlRootElement; +import jakarta.xml.bind.annotation.XmlSchemaType; +import jakarta.xml.bind.annotation.XmlType; +import jakarta.xml.bind.annotation.adapters.NormalizedStringAdapter; +import jakarta.xml.bind.annotation.adapters.XmlJavaTypeAdapter; + + +/** + * + * This terrible construct is so that xjc can autodetect this as the + * root node for processing. Many things would be better but this is + * the most workable solution especially if the making of the binding + * code is automated in the pom. The only other real solution is to + * modify one of the classes generated by hand. + * + * + *

Java class for anonymous complex type. + * + *

The following schema fragment specifies the expected content contained within this class. + * + *

{@code
+ * 
+ *   
+ *     
+ *       
+ *         
+ *         
+ *       
+ *       
+ *     
+ *   
+ * 
+ * }
+ * + * + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlType(name = "", propOrder = { + "cognitoClientId", + "serverUrl" +}) +@XmlRootElement(name = "registry_connection") +public class RegistryConnection { + + protected CognitoType cognitoClientId; + @XmlElement(name = "server_url") + protected DirectType serverUrl; + @XmlAttribute(name = "index") + @XmlJavaTypeAdapter(NormalizedStringAdapter.class) + @XmlSchemaType(name = "normalizedString") + protected String index; + + /** + * Gets the value of the cognitoClientId property. + * + * @return + * possible object is + * {@link CognitoType } + * + */ + public CognitoType getCognitoClientId() { + return cognitoClientId; + } + + /** + * Sets the value of the cognitoClientId property. + * + * @param value + * allowed object is + * {@link CognitoType } + * + */ + public void setCognitoClientId(CognitoType value) { + this.cognitoClientId = value; + } + + /** + * Gets the value of the serverUrl property. + * + * @return + * possible object is + * {@link DirectType } + * + */ + public DirectType getServerUrl() { + return serverUrl; + } + + /** + * Sets the value of the serverUrl property. + * + * @param value + * allowed object is + * {@link DirectType } + * + */ + public void setServerUrl(DirectType value) { + this.serverUrl = value; + } + + /** + * Gets the value of the index property. + * + * @return + * possible object is + * {@link String } + * + */ + public String getIndex() { + if (index == null) { + return "registry"; + } else { + return index; + } + } + + /** + * Sets the value of the index property. + * + * @param value + * allowed object is + * {@link String } + * + */ + public void setIndex(String value) { + this.index = value; + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkImpl.java new file mode 100644 index 0000000..c45798a --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkImpl.java @@ -0,0 +1,50 @@ +package gov.nasa.pds.registry.common.connection.es; + + +import java.util.Collection; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.Request.Bulk; + +class BulkImpl implements Request.Bulk { + private String index = null; + private String refresh = null; + String json = ""; + //@Override + public void add(String statement) { + this.json += statement + "\n"; + } + @Override + public void add(String statement, String document) { + this.json += statement + "\n"; + this.json += document + "\n"; + } + @Override + public Request.Bulk buildUpdateStatus(Collection lidvids, String status) { + this.json = JsonHelper.buildUpdateStatusJson(lidvids, status); + return this; + } + @Override + public Bulk setIndex(String name) { + this.index = name; + return this; + } + @Override + public Bulk setRefresh(Refresh type) { + switch (type) { + case False: + this.refresh = "false"; + break; + case True: + this.refresh = "true"; + break; + case WaitFor: + this.refresh = "wait_for"; + break; + } + return this; + } + @Override + public String toString() { + return (this.index == null ? "" : "/" + this.index) + "/_bulk" + (this.refresh == null ? "" : "?refresh=" + this.refresh); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java new file mode 100644 index 0000000..3a383e0 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkRespImpl.java @@ -0,0 +1,106 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.NotImplementedException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import com.google.gson.Gson; +import gov.nasa.pds.registry.common.Response; + +class BulkRespImpl implements Response.Bulk { + final int errorCount; + final private Logger log; + final private org.elasticsearch.client.Response response; + BulkRespImpl (org.elasticsearch.client.Response response) { + this.log = LogManager.getLogger(this.getClass()); + this.errorCount = this.parse(response.toString()); + this.response = response; + } + @SuppressWarnings("rawtypes") // necessary evil to manipulate heterogenous structures + private int parse (String resp) { + int numErrors = 0; + try + { + // TODO: Use streaming parser. Stop parsing if there are no errors. + // Parse JSON response + Gson gson = new Gson(); + Map json = (Map)gson.fromJson(resp, Object.class); + Boolean hasErrors = (Boolean)json.get("errors"); + if(hasErrors) + { + @SuppressWarnings("unchecked") + List list = (List)json.get("items"); + + // List size = batch size (one item per document) + for(Object item: list) + { + Map action = (Map)((Map)item).get("index"); + if(action == null) + { + action = (Map)((Map)item).get("create"); + if(action != null) + { + String status = String.valueOf(action.get("status")); + // For "create" requests status=409 means that the record already exists. + // It is not an error. We use "create" action to insert records which don't exist + // and keep existing records as is. We do this when loading an old LDD and more + // recent version of the LDD is already loaded. + // NOTE: Gson JSON parser stores numbers as floats. + // The string value is usually "409.0". Can it be something else? + if(status.startsWith("409")) + { + // Increment to properly report number of processed records. + numErrors++; + continue; + } + } + } + if(action == null) continue; + + String id = (String)action.get("_id"); + Map error = (Map)action.get("error"); + if(error != null) + { + String message = (String)error.get("reason"); + String sanitizedLidvid = id.replace('\r', ' ').replace('\n', ' '); // protect vs log spoofing see code-scanning alert #37 + String sanitizedMessage = message.replace('\r', ' ').replace('\n', ' '); // protect vs log spoofing + log.error("LIDVID = " + sanitizedLidvid + ", Message = " + sanitizedMessage); + numErrors++; + } + } + } + + return numErrors; + } + catch(Exception ex) + { + return 0; + } + } + @Override + public boolean errors() { + return errorCount != 0; + } + @Override + public List items() { + throw new NotImplementedException(); + } + @Override + public long took() { + throw new NotImplementedException(); + } + @Override + public void logErrors() { + BulkResponseParser parser = new BulkResponseParser(); + try (InputStream is = this.response.getEntity().getContent()) { + InputStreamReader reader = new InputStreamReader(is); + parser.parse(reader); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("Some weird JSON parsing exception and should never get here."); + } + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponseParser.java similarity index 95% rename from src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponseParser.java index 24ca799..ff1441b 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/BulkResponseParser.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/BulkResponseParser.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.dao; +package gov.nasa.pds.registry.common.connection.es; import java.io.IOException; import java.io.Reader; @@ -8,7 +8,6 @@ import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; - import gov.nasa.pds.registry.common.util.CloseUtils; /** @@ -27,13 +26,12 @@ public BulkResponseParser() log = LogManager.getLogger(this.getClass()); } - /** * Parse JSON string * @param reader bulk API response stream * @throws IOException an exception */ - public void parse(Reader reader) throws IOException + public void parse(Reader reader) throws IOException // needs to be public for testing { if(reader == null) return; diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/ClientConfigCB.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConfigCB.java similarity index 72% rename from src/main/java/gov/nasa/pds/registry/common/es/client/ClientConfigCB.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConfigCB.java index fb563e2..5065daa 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/ClientConfigCB.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/ClientConfigCB.java @@ -1,12 +1,9 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; import javax.net.ssl.SSLContext; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; @@ -19,12 +16,17 @@ * * @author karpenko */ -public class ClientConfigCB implements RestClientBuilder.HttpClientConfigCallback +class ClientConfigCB implements RestClientBuilder.HttpClientConfigCallback { private boolean trustSelfSignedCert = false; private CredentialsProvider credProvider; + public void setCredProvider(CredentialsProvider credProvider) { + this.credProvider = credProvider; + } + + /** * Constructor */ @@ -43,20 +45,6 @@ public void setTrustSelfSignedCert(boolean b) } - /** - * Set user name and password for basic authentication. - * @param user user name - * @param pass password - */ - public void setUserPass(String user, String pass) - { - if(user == null || pass == null) return; - - credProvider = new BasicCredentialsProvider(); - credProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, pass)); - } - - @Override public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/CountImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/CountImpl.java new file mode 100644 index 0000000..b8e3e3a --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/CountImpl.java @@ -0,0 +1,22 @@ +package gov.nasa.pds.registry.common.connection.es; + +import gov.nasa.pds.registry.common.Request.Count; + +class CountImpl implements Count { + private String index; + private String query; + @Override + public Count setIndex(String name) { + this.index = name; + return this; + } + @Override + public Count setQuery(String q) { + this.query = q; + return this; + } + @Override + public String toString() { + return "/" + this.index + "/_count?q=" + this.query; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/DeleteByQueryImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/DeleteByQueryImpl.java new file mode 100644 index 0000000..0b151ad --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/DeleteByQueryImpl.java @@ -0,0 +1,52 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.Map; +import com.google.gson.Gson; +import gov.nasa.pds.registry.common.Request.DeleteByQuery; + +class DeleteByQueryImpl implements DeleteByQuery { + private boolean refresh = false; + private String index = "undefnned"; + String query = ""; + @Override + public DeleteByQuery setIndex(String name) { + this.index = name; + return this; + } + @Override + public DeleteByQuery createFilterQuery(String key, String value) { + this.query = new RegistryRequestBuilder().createFilterQuery(key, value); + return this; + } + @Override + public DeleteByQuery createMatchAllQuery() { + this.query = new RegistryRequestBuilder().createMatchAllQuery(); + return this; + } + @Override + public String toString() { + return "/" + this.index + "/_delete_by_query" + (this.refresh ? "?refresh=true" : ""); + } + @SuppressWarnings("rawtypes") + long extractNumDeleted(org.elasticsearch.client.Response resp) { + try { + InputStream is = resp.getEntity().getContent(); + Reader rd = new InputStreamReader(is); + Gson gson = new Gson(); + Object obj = gson.fromJson(rd, Object.class); + rd.close(); + obj = ((Map) obj).get("deleted"); + return ((Double)obj).longValue(); + } catch (Exception ex) { + return 0; + } + } + @Override + public DeleteByQuery setRefresh(boolean state) { + this.refresh = state; + return this; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/EsQueryUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsQueryUtils.java new file mode 100644 index 0000000..67a36bf --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsQueryUtils.java @@ -0,0 +1,97 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; + +import com.google.gson.stream.JsonWriter; + +/** + * Helper methods for building Elasticsearch queries. + * + * @author karpenko + */ +class EsQueryUtils +{ + /** + * Append "match_all" object. + * @param writer JSON writer + * @throws IOException an exception + */ + public static void appendMatchAll(JsonWriter writer) throws IOException + { + writer.name("match_all"); + writer.beginObject(); + writer.endObject(); + } + + + /** + * Append match all query. + * @param writer JSON writer + * @throws IOException an exception + */ + public static void appendMatchAllQuery(JsonWriter writer) throws IOException + { + writer.name("query"); + writer.beginObject(); + appendMatchAll(writer); + writer.endObject(); + } + + + /** + * Append filter query + * @param writer JSON writer + * @param field field name + * @param value field value + * @throws IOException an exception + */ + public static void appendFilterQuery(JsonWriter writer, String field, String value) throws IOException + { + writer.name("query"); + writer.beginObject(); + + writer.name("bool"); + writer.beginObject(); + appendMustMatchAll(writer); + appendTermFilter(writer, field, value); + writer.endObject(); + + writer.endObject(); + } + + + /** + * Append must match all criterion + * @param writer JSON writer + * @throws IOException an exception + */ + private static void appendMustMatchAll(JsonWriter writer) throws IOException + { + writer.name("must"); + writer.beginObject(); + appendMatchAll(writer); + writer.endObject(); + } + + + /** + * Append term filter + * @param writer JSON writer + * @param field field name + * @param value field value + * @throws IOException an exception + */ + private static void appendTermFilter(JsonWriter writer, String field, String value) throws IOException + { + writer.name("filter"); + writer.beginObject(); + + writer.name("term"); + writer.beginObject(); + writer.name(field).value(value); + writer.endObject(); + + writer.endObject(); + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/EsUtils.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java similarity index 71% rename from src/main/java/gov/nasa/pds/registry/common/es/client/EsUtils.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java index 5aefcdf..7e4ebf4 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/EsUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/EsUtils.java @@ -1,26 +1,22 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; import java.util.List; -import java.util.Map; import org.apache.http.HttpHost; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import com.google.gson.Gson; - - /** * Elasticsearch utility methods. * * @author karpenko */ -public class EsUtils +class EsUtils { /** * Parse Elasticsearch URL - * @param url Elasticsearch URL, e.g., "http://localhost:9200" + * @param url Elasticsearch URL, e.g., "app:/connections/direct/localhost.xml" * @return HTTP host information * @throws Exception an exception */ @@ -99,48 +95,12 @@ public static String extractErrorMessage(ResponseException ex) String lines[] = msg.split("\n"); if(lines.length < 2) return msg; - String reason = extractReasonFromJson(lines[1]); + String reason = SearchResponseParser.extractReasonFromJson(lines[1]); if(reason == null) return msg; return reason; } - - /** - * Extract error message from Elasticsearch response JSON. - * @param json response JSON - * @return error message - */ - @SuppressWarnings("rawtypes") - public static String extractReasonFromJson(String json) - { - try - { - Gson gson = new Gson(); - Object obj = gson.fromJson(json, Object.class); - - obj = ((Map)obj).get("error"); - - Object rc = ((Map)obj).get("root_cause"); - if(rc != null) - { - List list = (List)rc; - obj = ((Map)list.get(0)).get("reason"); - } - else - { - obj = ((Map)obj).get("reason"); - } - - return obj.toString(); - } - catch(Exception ex) - { - return null; - } - } - - /** * Print Elasticsearch response warnings. * @param resp HTTP response diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/GetAltIdsParser.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/GetAltIdsParser.java new file mode 100644 index 0000000..82f6da9 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/GetAltIdsParser.java @@ -0,0 +1,55 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +class GetAltIdsParser implements SearchResponseParser.Callback +{ + private Map> map; + + + GetAltIdsParser() + { + map = new TreeMap<>(); + } + + + public Map> getIdMap() + { + return map; + } + + + @Override + public void onRecord(String id, Object rec) + { + if(rec instanceof Map) + { + Object obj = ((Map)rec).get("alternate_ids"); + if(obj == null) return; + + // Multiple values + if(obj instanceof List) + { + Set altIds = new TreeSet<>(); + for(Object item: (List)obj) + { + altIds.add(item.toString()); + } + + map.put(id, altIds); + } + // Single value + else if(obj instanceof String) + { + Set altIds = new TreeSet<>(); + altIds.add((String)obj); + map.put(id, altIds); + } + } + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/GetImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/GetImpl.java new file mode 100644 index 0000000..f715aa2 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/GetImpl.java @@ -0,0 +1,62 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import gov.nasa.pds.registry.common.Request.Get; +import gov.nasa.pds.registry.common.Request.MGet; + +class GetImpl implements MGet { + final private ArrayList excludes = new ArrayList(); + final private ArrayList includes = new ArrayList(); + private String id = null; + private String index; + String json = null; + @Override + public Get excludeField(String field) { + this.excludes.add(field); + return this; + } + @Override + public Get excludeFields(List fields) { + this.excludes.addAll(fields); + return this; + } + @Override + public Get includeField(String field) { + this.includes.add(field); + return this; + } + @Override + public Get includeFields(List fields) { + this.includes.addAll(fields); + return this; + } + @Override + public Get setId(String id) { + this.setId(id); + return this; + } + @Override + public MGet setIds(Collection ids) { + this.json = JsonHelper.buildIdList(ids); + return this; + } + @Override + public Get setIndex(String index) { + this.setIndex(index); + return this; + } + @Override + public String toString() { + String constraints = ""; + if (!this.excludes.isEmpty() || !this.includes.isEmpty()) { + constraints += "?_source="; + for (String field : this.includes) { + constraints += field + ","; + } + constraints = constraints.substring(0, constraints.length()-1); + } + return "/" + this.index + "/_doc" + (this.id == null ? "" : "/" + this.id) + constraints; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/GetRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/GetRespImpl.java new file mode 100644 index 0000000..e410660 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/GetRespImpl.java @@ -0,0 +1,169 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.es.dao.DaoUtils; +import gov.nasa.pds.registry.common.es.dao.LidvidSet; +import gov.nasa.pds.registry.common.es.dao.dd.DataTypeNotFoundException; +import gov.nasa.pds.registry.common.es.dao.dd.GetDataTypesResponseParser; +import gov.nasa.pds.registry.common.util.LidVidUtils; +import gov.nasa.pds.registry.common.util.Tuple; + +class GetRespImpl implements Response.Get { + final private Logger log; + final private org.elasticsearch.client.Response response; + GetRespImpl(org.elasticsearch.client.Response response) { + this.log = LogManager.getLogger(this.getClass()); + this.response = response; + } + private LidvidSet parseCollectionIdsSource(JsonReader rd) throws IOException { + LidvidSet ids = new LidvidSet(null,null); + rd.beginObject(); + while (rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if ("ref_lid_collection".equals(name)) { + ids.lids = DaoUtils.parseSet(rd); + } else if ("ref_lidvid_collection".equals(name)) { + ids.lidvids = DaoUtils.parseSet(rd); + } else { + rd.skipValue(); + } + } + rd.endObject(); + return ids; + } + private String parseProductClassSource(JsonReader rd) throws IOException { + rd.beginObject(); + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if("product_class".equals(name)) { + return rd.nextString(); + } else { + rd.skipValue(); + } + } + rd.endObject(); + return null; + } + private List parseRefs(JsonReader rd) throws IOException { + rd.beginObject(); + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if("product_lidvid".equals(name)) { + return DaoUtils.parseList(rd); + } else { + rd.skipValue(); + } + } + rd.endObject(); + return null; + } + @Override + public String productClass() { + try (InputStream is = this.response.getEntity().getContent()) { + JsonReader rd = new JsonReader(new InputStreamReader(is)); + rd.beginObject(); + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if("_source".equals(name)) { + return parseProductClassSource(rd); + } else { + rd.skipValue(); + } + } + rd.endObject(); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("Weird JSON parsing error because should never reach this branch"); + } + return null; + } + @Override + public List refs() { + try (InputStream is = this.response.getEntity().getContent()) { + JsonReader rd = new JsonReader(new InputStreamReader(is)); + rd.beginObject(); + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if("_source".equals(name)) { + return parseRefs(rd); + } else { + rd.skipValue(); + } + } + rd.endObject(); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("Weird JSON parsing problem that should never occur"); + } + return null; + } + @Override + public IdSets ids() { + LidvidSet collectionIds = null; + try (InputStream is = this.response.getEntity().getContent()) { + JsonReader rd = new JsonReader(new InputStreamReader(is)); + rd.beginObject(); + while (rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if ("_source".equals(name)) { + collectionIds = parseCollectionIdsSource(rd); + } else { + rd.skipValue(); + } + } + rd.endObject(); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("Weird JSON parsing problem that should never get here"); + } + if (collectionIds == null || collectionIds.lidvids == null || collectionIds.lids == null) + return collectionIds; + // Harvest converts LIDVIDs to LIDs, so let's delete those converted LIDs. + for (String lidvid : collectionIds.lidvids) { + String lid = LidVidUtils.lidvidToLid(lidvid); + if (lid != null) { + collectionIds.lids.remove(lid); + } + } + return collectionIds; + } + @Override + public List dataTypes(boolean stringForMissing) throws IOException, DataTypeNotFoundException { + List dtInfo = new ArrayList(); + GetDataTypesResponseParser parser = new GetDataTypesResponseParser(); + List records = parser.parse(this.response.getEntity()); + // Process response (list of fields) + boolean missing = false; + for (GetDataTypesResponseParser.Record rec : records) { + if (rec.found) { + dtInfo.add(new Tuple(rec.id, rec.esDataType)); + } + // There is no data type for this field in ES registry-dd index + else { + // Automatically assign data type for known fields + if (rec.id.startsWith("ref_lid_") || rec.id.startsWith("ref_lidvid_") + || rec.id.endsWith("_Area")) { + dtInfo.add(new Tuple(rec.id, "keyword")); + continue; + } + if (stringForMissing) { + log.warn("Could not find datatype for field " + rec.id + ". Will use 'keyword'"); + dtInfo.add(new Tuple(rec.id, "keyword")); + } else { + log.error("Could not find datatype for field " + rec.id); + missing = true; + } + } + } + if (stringForMissing == false && missing == true) + throw new DataTypeNotFoundException(); + + return dtInfo; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/JsonHelper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/JsonHelper.java new file mode 100644 index 0000000..bad1610 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/JsonHelper.java @@ -0,0 +1,388 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.StringWriter; +import java.util.Collection; +import org.apache.http.HttpEntity; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import com.google.gson.stream.JsonWriter; +import gov.nasa.pds.registry.common.meta.Metadata; +import gov.nasa.pds.registry.common.util.Tuple; + +/** + * Builds Elasticsearch JSON queries + * + * @author karpenko + */ +class JsonHelper { + /** + * Build update product archive status JSON request + * + * @param lidvids list of LIDVIDs to update + * @param status new status + * @return JSON + */ + static String buildUpdateStatusJson(Collection lidvids, String status) { + if (lidvids == null || lidvids.isEmpty()) + return null; + if (status == null || status.isEmpty()) + throw new IllegalArgumentException("Status could not be null or empty."); + + StringBuilder bld = new StringBuilder(); + String dataLine = + "{ \"doc\" : {\"" + Metadata.FLD_ARCHIVE_STATUS + "\" : \"" + status + "\"} }\n"; + + // Build NJSON (new-line delimited JSON) + for (String lidvid : lidvids) { + // Line 1: Elasticsearch document ID + bld.append("{ \"update\" : {\"_id\" : \"" + lidvid + "\" } }\n"); + // Line 2: Data + bld.append(dataLine); + } + + return bld.toString(); + } + + + /** + * Build aggregation query to select latest versions of lids + * + * @param lids list of LIDs + * @return JSON + */ + static String buildGetLatestLidVidsJson(Collection lids) { + if (lids == null || lids.isEmpty()) + return null; + + StringWriter strWriter = new StringWriter(); + try (JsonWriter jw = new JsonWriter(strWriter)) { + jw.beginObject(); + + jw.name("_source").value(false); + jw.name("size").value(0); + + // Query + jw.name("query"); + jw.beginObject(); + + jw.name("terms"); + jw.beginObject(); + + jw.name("lid"); + jw.beginArray(); + for (String lid : lids) { + jw.value(lid); + } + jw.endArray(); + + jw.endObject(); // terms + jw.endObject(); // query + + // Aggs + jw.name("aggs"); + jw.beginObject(); + + jw.name("lids"); + jw.beginObject(); + + jw.name("terms"); + jw.beginObject(); + jw.name("field").value("lid"); + jw.name("size").value(5000); + jw.endObject(); + + jw.name("aggs"); + jw.beginObject(); + jw.name("latest"); + jw.beginObject(); + jw.name("top_hits"); + jw.beginObject(); + + jw.name("sort"); + jw.beginArray(); + jw.beginObject(); + jw.name("vid"); + jw.beginObject(); + jw.name("order").value("desc"); + jw.endObject(); + jw.endObject(); + jw.endArray(); + + jw.name("_source").value(false); + jw.name("size").value(1); + + jw.endObject(); // top_hits + jw.endObject(); // latest + jw.endObject(); // aggs + + jw.endObject(); // lids + jw.endObject(); // aggs + + jw.endObject(); + + return strWriter.toString(); + } catch (IOException ioe) { + // not sure how we got here but fail in a large way + throw new RuntimeException("some weird internal JSON writing problem"); + } + } + + static String buildSearchIdsRequest(Collection ids, int pageSize, boolean isAlt) { + StringWriter strWriter = new StringWriter(); + try (JsonWriter writer = new JsonWriter(strWriter)) { + // Create ids query + writer.beginObject(); + + // Exclude source from response + JsonWriter src = writer.name("_source"); + if (isAlt) + src.value("alternate_ids"); + else + src.value(false); + writer.name("size").value(pageSize); + + writer.name("query"); + writer.beginObject(); + writer.name("ids"); + writer.beginObject(); + + writer.name("values"); + JsonWriter a = writer.beginArray(); + for (String id : ids) { + writer.value(id); + } + writer.endArray(); + + writer.endObject(); + writer.endObject(); + writer.endObject(); + + a.close(); + writer.close(); + } catch (IOException ioe) { + // cannot imagine ever getting here + throw new RuntimeException( + "some weird JSON serialization problem that should not be happening"); + } + return strWriter.toString(); + } + + static String buildListLddsRequest(String namespace) { + StringWriter strWriter = new StringWriter(); + try (JsonWriter jw = new JsonWriter(strWriter)) { + jw.beginObject(); + // Size (number of records to return) + jw.name("size").value(1000); + + // Start query + jw.name("query"); + jw.beginObject(); + jw.name("bool"); + jw.beginObject(); + + jw.name("must"); + jw.beginArray(); + appendMatch(jw, "class_ns", "registry"); + appendMatch(jw, "class_name", "LDD_Info"); + if (namespace != null) { + appendMatch(jw, "attr_ns", namespace); + } + jw.endArray(); + + jw.endObject(); + jw.endObject(); + // End query + + // Start source + jw.name("_source"); + jw.beginArray(); + jw.value("date").value("attr_name").value("attr_ns").value("im_version"); + jw.endArray(); + // End source + + jw.endObject(); + jw.close(); + } catch (IOException ioe) { + // cannot imagine ever getting here + throw new RuntimeException( + "some weird JSON serialization problem that should not be happening"); + } + + return strWriter.toString(); + } + + static String buildListFieldsRequest(String dataType) { + StringWriter strWriter = new StringWriter(); + try (JsonWriter jw = new JsonWriter(strWriter)) { + jw.beginObject(); + // Size (number of records to return) + jw.name("size").value(1000); + + // Start query + jw.name("query"); + jw.beginObject(); + jw.name("bool"); + jw.beginObject(); + + jw.name("must"); + jw.beginArray(); + appendMatch(jw, "es_data_type", dataType); + jw.endArray(); + + jw.endObject(); + jw.endObject(); + // End query + + // Start source + jw.name("_source"); + jw.beginArray(); + jw.value("es_field_name"); + jw.endArray(); + // End source + + jw.endObject(); + jw.close(); + } catch (IOException ioe) { + // cannot imagine ever getting here + throw new RuntimeException( + "some weird JSON serialization problem that should not be happening"); + } + + return strWriter.toString(); + } + + + /** + * Create get data dictionary (LDD) info request. + * + * @param namespace LDD namespace ID, such as 'pds', 'cart', etc. + * @return Elasticsearch query in JSON format + * @throws IOException an exception + */ + static String buildGetLddInfoRequest(String namespace) throws IOException { + StringWriter strWriter = new StringWriter(); + try (JsonWriter jw = new JsonWriter(strWriter)) { + jw.beginObject(); + // Size (number of records to return) + jw.name("size").value(1000); + + // Start query + jw.name("query"); + jw.beginObject(); + jw.name("bool"); + jw.beginObject(); + + jw.name("must"); + jw.beginArray(); + appendMatch(jw, "class_ns", "registry"); + appendMatch(jw, "class_name", "LDD_Info"); + appendMatch(jw, "attr_ns", namespace); + jw.endArray(); + + jw.endObject(); + jw.endObject(); + // End query + + // Start source + jw.name("_source"); + jw.beginArray(); + jw.value("date").value("attr_name"); + jw.endArray(); + // End source + + jw.endObject(); + jw.close(); + } catch (IOException ioe) { + // cannot imagine ever getting here + throw new RuntimeException( + "some weird JSON serialization problem that should not be happening"); + } + + return strWriter.toString(); + } + + + private static void appendMatch(JsonWriter jw, String field, String value) throws IOException { + jw.beginObject(); + jw.name("match"); + jw.beginObject(); + jw.name(field).value(value); + jw.endObject(); + jw.endObject(); + } + + static String buildIdList(Collection ids) { + StringWriter strWriter = new StringWriter(); + try (JsonWriter jw = new JsonWriter(strWriter)) { + jw.beginObject(); + jw.name("ids"); + + JsonWriter a = jw.beginArray(); + for (String id : ids) { + jw.value(id); + } + jw.endArray(); + + jw.endObject(); + a.close(); + jw.close(); + } catch (IOException ioe) { + // cannot imagine ever getting here + throw new RuntimeException( + "some weird JSON serialization problem that should not be happening"); + } + + return strWriter.toString(); + } + + static String buildUpdateSchemaRequest(Collection fields) { + StringWriter strWriter = new StringWriter(); + try (JsonWriter jw = new JsonWriter(strWriter)) { + jw.beginObject(); + + jw.name("properties"); + JsonWriter o = jw.beginObject(); + for (Tuple field : fields) { + jw.name(field.item1); + jw.beginObject(); + jw.name("type").value(field.item2); + jw.endObject(); + } + jw.endObject(); + + jw.endObject(); + o.close(); + jw.close(); + } catch (IOException ioe) { + // cannot imagine ever getting here + throw new RuntimeException( + "some weird JSON serialization problem that should not be happening"); + } + + return strWriter.toString(); + } + + static long findCount (HttpEntity entity) { + long count = 0; + try (InputStream is = entity.getContent()) { + JsonReader rd = new JsonReader(new InputStreamReader(is)); + rd.beginObject(); + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + String name = rd.nextName(); + if("count".equals(name)) { + count = rd.nextInt(); + break; + } else { + rd.skipValue(); + } + } + rd.endObject(); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("some weird JSON serialization problem that should not be happening"); + } + return count; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingImpl.java new file mode 100644 index 0000000..a98d7a7 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingImpl.java @@ -0,0 +1,24 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.util.Collection; +import gov.nasa.pds.registry.common.Request.Mapping; +import gov.nasa.pds.registry.common.util.Tuple; + +class MappingImpl implements Mapping { + String index; + String json = null; + @Override + public Mapping buildUpdateFieldSchema(Collection pairs) { + this.json = JsonHelper.buildUpdateSchemaRequest(pairs); + return this; + } + @Override + public Mapping setIndex(String name) { + this.index = name; + return this; + } + @Override + public String toString() { + return "/" + this.index + (this.json == null ? "/_mappings" : "/_mapping"); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingRespImpl.java new file mode 100644 index 0000000..2ac8c4f --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingRespImpl.java @@ -0,0 +1,17 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.util.HashSet; +import java.util.Set; +import gov.nasa.pds.registry.common.Response; + +class MappingRespImpl implements Response.Mapping { + final private Set fieldNames; + MappingRespImpl (org.elasticsearch.client.Response response, String index) { + this.fieldNames = new MappingsParser(index).parse(response.getEntity()); + } + @Override + public Set fieldNames() { + return new HashSet(fieldNames); + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/MappingsParser.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingsParser.java similarity index 80% rename from src/main/java/gov/nasa/pds/registry/common/es/dao/schema/MappingsParser.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/MappingsParser.java index d5c2d7a..50e3161 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/MappingsParser.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/MappingsParser.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.dao.schema; +package gov.nasa.pds.registry.common.connection.es; import java.io.IOException; import java.io.InputStream; @@ -18,7 +18,7 @@ * * @author karpenko */ -public class MappingsParser +class MappingsParser { private String indexName; private JsonReader rd; @@ -41,35 +41,27 @@ public MappingsParser(String indexName) * @return a collection of field names from a given Elasticsearch index. * @throws IOException an exception */ - public Set parse(HttpEntity entity) throws IOException + public Set parse(HttpEntity entity) { - InputStream is = entity.getContent(); + fields = new TreeSet<>(); + try (InputStream is = entity.getContent()) { rd = new JsonReader(new InputStreamReader(is)); - - fields = new TreeSet<>(); - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - // Usually there is only one root element = index name. - String name = rd.nextName(); - if(indexName.equals(name)) - { - parseMappings(); - } - // Usually we should not go here. - else - { - rd.skipValue(); - } + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) { + // Usually there is only one root element = index name. + String name = rd.nextName(); + if(indexName.equals(name)) { + parseMappings(); + } else { + rd.skipValue(); + } } - rd.endObject(); - rd.close(); - - return fields; + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("some strange exception because we should never get here"); + } + return fields; } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/NonExistingIdsResponse.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/NonExistingIdsResponse.java new file mode 100644 index 0000000..c7ff219 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/NonExistingIdsResponse.java @@ -0,0 +1,47 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.util.Collection; +import java.util.Set; +import java.util.TreeSet; + +/** + * Helper class to process Elasticsearch response from "search IDs" query. + * + * @author karpenko + */ +class NonExistingIdsResponse implements SearchResponseParser.Callback +{ + private Set retIds; + + + /** + * Constructor + * @param ids Product IDs (lidvids) sent to Elasticsearch in "search IDs" query. + * IDs are copied to internal collection. + * After processing Elasticsearch response, all IDs existing in Elasticsearch + * "registry" index will be removed from this internal collection. + */ + NonExistingIdsResponse(Collection ids) + { + retIds = new TreeSet<>(ids); + } + + /** + * Return collection of product IDs (lidvids) non-existing in Elasticsearch. + * @return a collection of product IDs (lidvids) + */ + public Set getIds() + { + return retIds; + } + + + /** + * This method is called for each record in Elasticsearch response + */ + @Override + public void onRecord(String id, Object src) + { + retIds.remove(id); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RegistryRequestBuilder.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RegistryRequestBuilder.java new file mode 100644 index 0000000..15157af --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RegistryRequestBuilder.java @@ -0,0 +1,202 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import java.io.StringWriter; +import java.io.Writer; + +import com.google.gson.stream.JsonWriter; + +/** + * A class to build Elasticsearch API JSON requests. + * + * @author karpenko + */ +class RegistryRequestBuilder +{ + private boolean pretty; + + + /** + * Constructor + * @param pretty Pretty-format JSON requests + */ + RegistryRequestBuilder(boolean pretty) + { + this.pretty = pretty; + } + + + /** + * Constructor + */ + public RegistryRequestBuilder() + { + this(false); + } + + + private JsonWriter createJsonWriter(Writer writer) + { + JsonWriter jw = new JsonWriter(writer); + if (pretty) + { + jw.setIndent(" "); + } + + return jw; + } + + /** + * Build export data request + * @param filterField Filter field name, such as "lidvid". + * @param filterValue Filter value. + * @param sortField Sort field is required to paginate data and use "search_after" field. + * @param size Batch / page size + * @param searchAfter "search_after" field to perform pagination + * @return JSON + * @throws IOException an exception + */ + public String createExportDataRequest(String filterField, String filterValue, String sortField, int size, String searchAfter) { + StringWriter out = new StringWriter(); + try (JsonWriter writer = createJsonWriter(out)) { + writer.beginObject(); + // Size (number of records to return) + writer.name("size").value(size); + // Filter query + EsQueryUtils.appendFilterQuery(writer, filterField, filterValue); + // "search_after" parameter is used for pagination + if (searchAfter != null) { + writer.name("search_after").value(searchAfter); + } + // Sort is required by pagination + writer.name("sort"); + writer.beginObject(); + writer.name(sortField).value("asc"); + writer.endObject(); + writer.endObject(); + writer.close(); + } catch (IOException e) { + throw new RuntimeException("Should never get here"); + } + return out.toString(); + } + + + /** + * Build export all data request + * @param sortField Sort field is required to paginate data and use "search_after" field. + * @param size Batch / page size + * @param searchAfter "search_after" field to perform pagination + * @return JSON + * @throws IOException an exception + */ + public String createExportAllDataRequest(String sortField, int size, String searchAfter) { + StringWriter out = new StringWriter(); + try (JsonWriter writer = createJsonWriter(out)) { + writer.beginObject(); + // Size (number of records to return) + writer.name("size").value(size); + // Match all query + EsQueryUtils.appendMatchAllQuery(writer); + // "search_after" parameter is used for pagination + if (searchAfter != null) { + writer.name("search_after"); + writer.beginArray(); + writer.value(searchAfter); + writer.endArray(); + } + // Sort is required by pagination + writer.name("sort"); + writer.beginObject(); + writer.name(sortField).value("asc"); + writer.endObject(); + writer.endObject(); + writer.close(); + } catch (IOException e) { + throw new RuntimeException("Should never get here."); + } + return out.toString(); + } + + + /** + * Build get BLOB request + * @param lidvid a LidVid + * @return JSON + * @throws IOException an exception + */ + public String createGetBlobRequest(String fieldName, String lidvid) + { + StringWriter out = new StringWriter(); + try (JsonWriter writer = createJsonWriter(out)){ + + writer.beginObject(); + + // Return only BLOB + writer.name("_source"); + writer.beginArray(); + writer.value(fieldName); + writer.endArray(); + + // Query + EsQueryUtils.appendFilterQuery(writer, "lidvid", lidvid); + writer.endObject(); + + writer.close(); + } catch (IOException e) { + throw new RuntimeException("Should never get here"); + } + return out.toString(); + } + + + /** + * Create Elasticsearch filter query + * @param field filter field name + * @param value filter value + * @return JSON + * @throws IOException an exception + */ + public String createFilterQuery(String field, String value) + { + StringWriter out = new StringWriter(); + try (JsonWriter writer = createJsonWriter(out)) { + + writer.beginObject(); + EsQueryUtils.appendFilterQuery(writer, field, value); + writer.endObject(); + + writer.close(); + } catch (IOException e) { + throw new RuntimeException("Should never get here"); + } + return out.toString(); + } + + + /** + * Build match all query + * @return JSON + * @throws IOException an exception + */ + public String createMatchAllQuery() + { + StringWriter out = new StringWriter(); + try (JsonWriter writer = createJsonWriter(out)) { + + writer.beginObject(); + + writer.name("query"); + writer.beginObject(); + EsQueryUtils.appendMatchAll(writer); + writer.endObject(); + + writer.endObject(); + + writer.close(); + } catch (IOException e) { + throw new RuntimeException("Should never get here"); + } + return out.toString(); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/RequestConfigCB.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestConfigCB.java similarity index 88% rename from src/main/java/gov/nasa/pds/registry/common/es/client/RequestConfigCB.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/RequestConfigCB.java index 9ed11b3..460ac60 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/RequestConfigCB.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RequestConfigCB.java @@ -1,4 +1,4 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; import org.apache.http.client.config.RequestConfig; @@ -11,7 +11,7 @@ * * @author karpenko */ -public class RequestConfigCB implements RestClientBuilder.RequestConfigCallback +class RequestConfigCB implements RestClientBuilder.RequestConfigCallback { private int connectTimeoutSec = 5; private int socketTimeoutSec = 10; diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java new file mode 100644 index 0000000..e08e4ba --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseExceptionWrapper.java @@ -0,0 +1,79 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.PrintStream; +import java.io.PrintWriter; +import gov.nasa.pds.registry.common.ResponseException; + +final class ResponseExceptionWrapper extends ResponseException { + private static final long serialVersionUID = -5116172984798822803L; + final org.elasticsearch.client.ResponseException real_exception; + ResponseExceptionWrapper (org.elasticsearch.client.ResponseException real_exception){ + this.real_exception = real_exception; + } + @Override + public String getMessage() { + return this.real_exception == null ? super.getMessage() : this.real_exception.getMessage(); + } + @Override + public String getLocalizedMessage() { + return this.real_exception == null ? super.getLocalizedMessage() : this.real_exception.getLocalizedMessage(); + } + @Override + public synchronized Throwable getCause() { + return this.real_exception == null ? super.getCause() : this.real_exception.getCause(); + } + @Override + public synchronized Throwable initCause(Throwable cause) { + return this.real_exception == null ? super.initCause(cause) : this.real_exception.initCause(cause); + } + @Override + public String toString() { + return this.real_exception == null ? super.toString() : this.real_exception.toString(); + } + @Override + public void printStackTrace() { + if (this.real_exception == null) super.printStackTrace(); else this.real_exception.printStackTrace(); + } + @Override + public void printStackTrace(PrintStream s) { + if (this.real_exception == null) super.printStackTrace(s); else this.real_exception.printStackTrace(s); + } + @Override + public void printStackTrace(PrintWriter s) { + if (this.real_exception == null) super.printStackTrace(s); else this.real_exception.printStackTrace(s); + } + @Override + public synchronized Throwable fillInStackTrace() { + return this.real_exception == null ? super.fillInStackTrace() : this.real_exception.fillInStackTrace(); + } + @Override + public StackTraceElement[] getStackTrace() { + return this.real_exception == null ? super.getStackTrace() : this.real_exception.getStackTrace(); + } + @Override + public void setStackTrace(StackTraceElement[] stackTrace) { + if (this.real_exception == null) super.setStackTrace(stackTrace); else this.real_exception.setStackTrace(stackTrace); + } + @Override + public int statusCode() { + return this.real_exception.getResponse().getStatusLine().getStatusCode(); + } + /* + public Response getResponse() { + return new ResponseWrapper(this.real_exception.getResponse()); + } + */ + @Override + public String extractErrorMessage() { + String msg = this.real_exception.getMessage(); + if(msg == null) return "Unknown error"; + + String lines[] = msg.split("\n"); + if(lines.length < 2) return msg; + + String reason = SearchResponseParser.extractReasonFromJson(lines[1]); + if(reason == null) return msg; + + return reason; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseNotImplYet.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseNotImplYet.java new file mode 100644 index 0000000..cd1a12b --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/ResponseNotImplYet.java @@ -0,0 +1,18 @@ +package gov.nasa.pds.registry.common.connection.es; + +import gov.nasa.pds.registry.common.Response; + +class ResponseNotImplYet implements Response.CreatedIndex { + @Override + public boolean acknowledge() { + throw new RuntimeException("This method needs to be implemented for old style SDK"); + } + @Override + public boolean acknowledgeShards() { + throw new RuntimeException("This method needs to be implemented for old style SDK"); + } + @Override + public String getIndex() { + throw new RuntimeException("This method needs to be implemented for old style SDK"); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java new file mode 100644 index 0000000..f9387ea --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/RestClientWrapper.java @@ -0,0 +1,143 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import java.util.List; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RestClientBuilder; +import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.Request.Bulk; +import gov.nasa.pds.registry.common.Request.Count; +import gov.nasa.pds.registry.common.Request.DeleteByQuery; +import gov.nasa.pds.registry.common.Request.Get; +import gov.nasa.pds.registry.common.Request.Mapping; +import gov.nasa.pds.registry.common.Request.MGet; +import gov.nasa.pds.registry.common.Request.Search; +import gov.nasa.pds.registry.common.Request.Setting; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.ResponseException; +import gov.nasa.pds.registry.common.RestClient; + + +/** + * Utility class to build Elasticsearch rest client. + * + * @author karpenko + */ +public class RestClientWrapper implements RestClient +{ + final org.elasticsearch.client.RestClient real_client; + /** + * Constructor. + * @throws Exception an exception + */ + public RestClientWrapper(ConnectionFactory conFact) throws Exception + { + ClientConfigCB clientCB = new ClientConfigCB(); + RequestConfigCB reqCB = new RequestConfigCB(); + RestClientBuilder bld = org.elasticsearch.client.RestClient.builder(conFact.getHost()); + clientCB.setCredProvider(conFact.getCredentials()); + clientCB.setTrustSelfSignedCert(conFact.isTrustingSelfSigned()); + bld.setHttpClientConfigCallback(clientCB); + bld.setRequestConfigCallback(reqCB); + this.real_client = bld.build(); + } + private org.elasticsearch.client.Response performRequest(String endpoint, String json, String method) throws IOException,ResponseException { + try { + Request request = new Request(method, endpoint); + if (json != null) request.setJsonEntity(json); + return this.real_client.performRequest(request); + } catch (org.elasticsearch.client.ResponseException e) { + throw new ResponseExceptionWrapper(e); + } + } + private void printWarnings(org.elasticsearch.client.Response resp) { + List warnings = resp.getWarnings(); + if(warnings != null) + { + for(String warn: warnings) + { + System.out.println("[WARN] " + warn); + } + } + } + @Override + public void close() throws IOException { + this.real_client.close(); + } + @Override + public Bulk createBulkRequest() { + return new BulkImpl(); + } + @Override + public Count createCountRequest() { + return new CountImpl(); + } + @Override + public Get createGetRequest() { + return new GetImpl(); + } + @Override + public Mapping createMappingRequest() { + return new MappingImpl(); + } + @Override + public MGet createMGetRequest() { + return new GetImpl(); + } + @Override + public Search createSearchRequest() { + return new SearchImpl(); + } + @Override + public Setting createSettingRequest() { + return new SettingImpl(); + } + @Override + public Response.CreatedIndex create (String indexName, String configAsJSON) throws IOException,ResponseException { + this.printWarnings(this.performRequest("/" + indexName, configAsJSON, "PUT")); + return new ResponseNotImplYet(); + } + @Override + public void delete (String indexName) throws IOException,ResponseException { + this.printWarnings(this.performRequest(indexName, null, "DELETE")); + } + @Override + public boolean exists (String indexName) throws IOException,ResponseException { + return this.performRequest ("/" + indexName, null, "HEAD").getStatusLine().getStatusCode() == 200; + } + @Override + public Response.Bulk performRequest(Bulk request) throws IOException,ResponseException { + return new BulkRespImpl(this.performRequest(request.toString(), ((BulkImpl)request).json, "POST")); + } + @Override + public long performRequest(Count request) throws IOException,ResponseException { + return JsonHelper.findCount(this.performRequest(request.toString(), null, "GET").getEntity()); + } + @Override + public Response.Get performRequest(Get request) throws IOException,ResponseException { + return new GetRespImpl(this.performRequest(request.toString(), ((GetImpl)request).json, "GET")); + } + @Override + public Response.Mapping performRequest(Mapping request) throws IOException,ResponseException { + String index = ((MappingImpl)request).index; + String json = ((MappingImpl)request).json; + return new MappingRespImpl(this.performRequest(request.toString(), json, json == null ? "GET" : "PUT"), index); + } + @Override + public Response.Search performRequest(Search request) throws IOException,ResponseException { + return new SearchRespImpl(this.performRequest(request.toString(), ((SearchImpl)request).json, "GET")); + } + @Override + public Response.Settings performRequest(Setting request) throws IOException,ResponseException { + return new SettingsRespImpl(this.performRequest(request.toString(), null, "GET")); + } + @Override + public DeleteByQuery createDeleteByQuery() { + return null; + } + @Override + public long performRequest(DeleteByQuery request) throws IOException,ResponseException { + return ((DeleteByQueryImpl)request).extractNumDeleted( + this.performRequest(request.toString(), ((DeleteByQueryImpl)request).query, "POST")); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java new file mode 100644 index 0000000..6394780 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchImpl.java @@ -0,0 +1,73 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.util.Collection; +import org.apache.commons.lang3.NotImplementedException; +import gov.nasa.pds.registry.common.Request.Search; + +class SearchImpl implements Search { + private boolean pretty = false; + private String index; + String json = null; + @Override + public Search buildAlternativeIds(Collection lids) { + this.json = JsonHelper.buildSearchIdsRequest(lids, lids.size(), true); + return this; + } + @Override + public Search buildLatestLidVids(Collection lids) { + this.json = JsonHelper.buildGetLatestLidVidsJson(lids); + return this; + } + @Override + public Search buildListFields(String dataType) { + this.json = JsonHelper.buildListFieldsRequest(dataType); + return this; + } + @Override + public Search buildListLdds(String namespace) { + this.json = JsonHelper.buildListLddsRequest(namespace); + return this; + } + @Override + public Search buildTheseIds(Collection lids) { + this.json = JsonHelper.buildSearchIdsRequest(lids, lids.size(), true); + return this; + } + @Override + public Search setIndex(String name) { + this.index = name; + return this; + } + @Override + public String toString() { + return "/" + this.index + "/_search" + (this.pretty ? "?pretty" : ""); + } + @Override + public Search setPretty(boolean pretty) { + this.pretty = pretty; + return this; + } + @Override + public Search buildGetField(String field_name, String lidvid) { + this.json = new RegistryRequestBuilder().createGetBlobRequest(field_name, lidvid); + return this; + } + @Override + public Search all(String sortField, int size, String searchAfter) { + this.json = new RegistryRequestBuilder().createExportAllDataRequest(sortField, size, searchAfter); + return this; + } + @Override + public Search all(String filterField, String filterValue, String sortField, int size, String searchAfter) { + this.json = new RegistryRequestBuilder().createExportDataRequest(filterField, filterValue, sortField, size, searchAfter); + return this; + } + @Override + public Search buildTermQuery(String fieldname, String value) { + throw new NotImplementedException(); + } + @Override + public Search setSize(int hitsperpage) { + throw new NotImplementedException(); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java new file mode 100644 index 0000000..08629b8 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchRespImpl.java @@ -0,0 +1,182 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.lang3.NotImplementedException; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.es.dao.LatestLidsResponseParser; +import gov.nasa.pds.registry.common.es.dao.dd.LddInfo; +import gov.nasa.pds.registry.common.es.dao.dd.LddVersions; + +class SearchRespImpl implements Response.Search { + private static class BatchObjects implements SearchResponseParser.Callback { + final ArrayList content = new ArrayList(); + public void onRecord(String id, Object rec) { + content.add(rec); + } + } + private static class FieldNameFinder implements SearchResponseParser.Callback { + final private String fieldName; + boolean found = false; + String blob = null; + @Override + @SuppressWarnings("rawtypes") + public void onRecord(String id, Object rec) { + found = true; + this.blob = ((Map) rec).get(this.fieldName).toString(); + } + public FieldNameFinder(String fieldName) { + super(); + this.fieldName = fieldName; + } + } + + /** + * Inner private class to parse LDD information response from Elasticsearch. + * + * @author karpenko + */ + private static class ListFieldsParser extends SearchResponseParser + implements SearchResponseParser.Callback { + public Set list; + public ListFieldsParser() { + list = new HashSet<>(200); + } + @Override + public void onRecord(String id, Object rec) { + if (rec instanceof Map) { + @SuppressWarnings("rawtypes") + Map map = (Map) rec; + String fieldName = (String) map.get("es_field_name"); + list.add(fieldName); + } + } + } + /** + * Inner private class to parse LDD information response from Elasticsearch. + * + * @author karpenko + */ + private static class GetLddInfoRespParser extends SearchResponseParser + implements SearchResponseParser.Callback { + public LddVersions info; + public GetLddInfoRespParser() { + info = new LddVersions(); + } + @Override + public void onRecord(String id, Object rec) { + if (rec instanceof Map) { + @SuppressWarnings("rawtypes") + Map map = (Map) rec; + String strDate = (String) map.get("date"); + info.updateDate(strDate); + String file = (String) map.get("attr_name"); + info.addSchemaFile(file); + } + } + } + /** + * Inner private class to parse LDD information response from Elasticsearch. + * + * @author karpenko + */ + private static class ListLddsParser extends SearchResponseParser + implements SearchResponseParser.Callback { + public List list; + public ListLddsParser() { + list = new ArrayList<>(); + } + @Override + public void onRecord(String id, Object rec) { + if (rec instanceof Map) { + @SuppressWarnings("rawtypes") + Map map = (Map) rec; + LddInfo info = new LddInfo(); + // Namespace + info.namespace = (String) map.get("attr_ns"); + // Date + String str = (String) map.get("date"); + if (str != null && !str.isEmpty()) { + info.date = Instant.parse(str); + } + // Versions + info.imVersion = (String) map.get("im_version"); + // File name + info.file = (String) map.get("attr_name"); + list.add(info); + } + } + } + + final private org.elasticsearch.client.Response response; + SearchRespImpl(org.elasticsearch.client.Response response) { + this.response = response; + } + @Override + public List latestLidvids() { + try (InputStream is = this.response.getEntity().getContent()) { + LatestLidsResponseParser parser = new LatestLidsResponseParser(); + parser.parse(new InputStreamReader(is)); + return parser.getLidvids(); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("Weird JSON parsing error and should never get here"); + } + } + @Override + public LddVersions lddInfo() throws UnsupportedOperationException, IOException { + GetLddInfoRespParser parser = new GetLddInfoRespParser(); + parser.parseResponse(this.response, parser); + return parser.info; + } + @Override + public List ldds() throws UnsupportedOperationException, IOException { + ListLddsParser parser = new ListLddsParser(); + parser.parseResponse(this.response, parser); + return parser.list; + } + @Override + public Set fields() throws UnsupportedOperationException, IOException { + ListFieldsParser parser = new ListFieldsParser(); + parser.parseResponse(this.response, parser); + return parser.list; + } + @Override + public Map> altIds() throws UnsupportedOperationException, IOException { + GetAltIdsParser cb = new GetAltIdsParser(); + SearchResponseParser parser = new SearchResponseParser(); + parser.parseResponse(this.response, cb); + return cb.getIdMap(); + } + @Override + public Set nonExistingIds(Collection from_ids) throws UnsupportedOperationException, IOException { + NonExistingIdsResponse idsResp = new NonExistingIdsResponse(from_ids); + SearchResponseParser parser = new SearchResponseParser(); + parser.parseResponse(this.response, idsResp); + return idsResp.getIds(); + } + @Override + public String field(String name) throws NoSuchFieldException { + FieldNameFinder fnf = new FieldNameFinder(name); + if (!fnf.found) throw new NoSuchFieldException(); + return fnf.blob; + } + @Override + public List batch() throws UnsupportedOperationException, IOException { + BatchObjects content = new BatchObjects(); + SearchResponseParser parser = new SearchResponseParser(); + parser.parseResponse(this.response, content); + return content.content; + } + @Override + public List> documents() { + throw new NotImplementedException(); + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/SearchResponseParser.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchResponseParser.java similarity index 70% rename from src/main/java/gov/nasa/pds/registry/common/es/client/SearchResponseParser.java rename to src/main/java/gov/nasa/pds/registry/common/connection/es/SearchResponseParser.java index 21e9f48..2c2dc76 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/SearchResponseParser.java +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SearchResponseParser.java @@ -1,11 +1,14 @@ -package gov.nasa.pds.registry.common.es.client; +package gov.nasa.pds.registry.common.connection.es; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; - +import java.util.List; +import java.util.Map; import org.elasticsearch.client.Response; - import com.google.gson.Gson; +import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; @@ -15,7 +18,7 @@ * * @author karpenko */ -public class SearchResponseParser +class SearchResponseParser { /** * Inner callback interface @@ -29,7 +32,7 @@ public static interface Callback * @param rec Parsed content of "_source" field. Usually it is a field name-value map. * @throws Exception an exception */ - public void onRecord(String id, Object rec) throws Exception; + public void onRecord(String id, Object rec); } @@ -42,7 +45,7 @@ public static interface Callback /** * Constructor */ - public SearchResponseParser() + SearchResponseParser() { } @@ -71,9 +74,11 @@ public int getNumDocs() * Parse response. Callback.onRecord() will be called for each record. * @param resp Elasticsearch rest client's response object. * @param cb Callback interface. + * @throws IOException + * @throws UnsupportedOperationException * @throws Exception an exception */ - public void parseResponse(Response resp, Callback cb) throws Exception + public void parseResponse(Response resp, Callback cb) throws UnsupportedOperationException, IOException { if(cb == null) throw new IllegalArgumentException("Callback is null"); this.cb = cb; @@ -108,9 +113,10 @@ public void parseResponse(Response resp, Callback cb) throws Exception /** * Parse "hits" array in JSON response. * @param rd JSON reader + * @throws IOException * @throws Exception an exception */ - private void parseHits(JsonReader rd) throws Exception + private void parseHits(JsonReader rd) throws IOException { rd.beginObject(); @@ -139,9 +145,12 @@ private void parseHits(JsonReader rd) throws Exception /** * Parse a hit from "hits" array in JSON response. * @param rd JSON reader + * @throws IOException + * @throws JsonSyntaxException + * @throws JsonIOException * @throws Exception an exception */ - private void parseHit(JsonReader rd) throws Exception + private void parseHit(JsonReader rd) throws JsonIOException, JsonSyntaxException, IOException { Object src = null; @@ -172,4 +181,37 @@ else if("_source".equals(name)) cb.onRecord(lastId, src); } + /** + * Extract error message from search response JSON. + * @param json response JSON + * @return error message + */ + @SuppressWarnings("rawtypes") + public static String extractReasonFromJson(String json) + { + try + { + Gson gson = new Gson(); + Object obj = gson.fromJson(json, Object.class); + + obj = ((Map)obj).get("error"); + + Object rc = ((Map)obj).get("root_cause"); + if(rc != null) + { + List list = (List)rc; + obj = ((Map)list.get(0)).get("reason"); + } + else + { + obj = ((Map)obj).get("reason"); + } + + return obj.toString(); + } + catch(Exception ex) + { + return null; + } + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SettingImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SettingImpl.java new file mode 100644 index 0000000..610c748 --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SettingImpl.java @@ -0,0 +1,16 @@ +package gov.nasa.pds.registry.common.connection.es; + +import gov.nasa.pds.registry.common.Request.Setting; + +class SettingImpl implements Setting { + private String index; + @Override + public Setting setIndex(String name) { + this.index = name; + return this; + } + @Override + public String toString() { + return "/" + this.index + "/_settings"; + } +} diff --git a/src/main/java/gov/nasa/pds/registry/common/connection/es/SettingsRespImpl.java b/src/main/java/gov/nasa/pds/registry/common/connection/es/SettingsRespImpl.java new file mode 100644 index 0000000..50fcb7b --- /dev/null +++ b/src/main/java/gov/nasa/pds/registry/common/connection/es/SettingsRespImpl.java @@ -0,0 +1,102 @@ +package gov.nasa.pds.registry.common.connection.es; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import org.apache.http.HttpEntity; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import gov.nasa.pds.registry.common.Response; + +/** + * Parse Elasticsearch "/index_name/_settings" response. + * @author karpenko + */ +class SettingsRespImpl implements Response.Settings +{ + private int replicas; + private int shards; + /** + * Constructor + */ + public SettingsRespImpl(org.elasticsearch.client.Response response){ + this.parse(response.getEntity()); + } + + + /** + * Parse Elasticsearch response + * @param resp Elasticsearch response + * @return index settings + * @throws Exception an exception + */ + public void parse(HttpEntity entity) + { + try (InputStream is = entity.getContent()) { + JsonReader rd = new JsonReader(new InputStreamReader(is)); + rd.beginObject(); // root + rd.nextName(); // index name + rd.beginObject(); + rd.nextName(); // "settings" + rd.beginObject(); + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) + { + String name = rd.nextName(); + if("index".equals(name)) + { + parseIndex(rd); + } + else + { + rd.skipValue(); + } + } + + rd.endObject(); + rd.endObject(); + rd.endObject(); + + rd.close(); + } catch (UnsupportedOperationException | IOException e) { + throw new RuntimeException("some weird json error because should never get here"); + } + } + private void parseIndex(JsonReader rd) throws IOException + { + if(rd == null) throw new IllegalArgumentException("JsonReader is null"); + + rd.beginObject(); + + while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) + { + String name = rd.nextName(); + if("number_of_shards".equals(name)) + { + this.shards = rd.nextInt(); + } + else if("number_of_replicas".equals(name)) + { + this.replicas = rd.nextInt(); + } + else + { + rd.skipValue(); + } + } + + rd.endObject(); + } + + + @Override + public int replicas() { + return this.replicas; + } + + + @Override + public int shards() { + return this.shards; + } + +} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/DebugUtils.java b/src/main/java/gov/nasa/pds/registry/common/es/client/DebugUtils.java deleted file mode 100644 index ffc1e5d..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/DebugUtils.java +++ /dev/null @@ -1,46 +0,0 @@ -package gov.nasa.pds.registry.common.es.client; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; - -import org.elasticsearch.client.Response; - - -/** - * Debug utilities. - * - * @author karpenko - */ -public class DebugUtils -{ - /** - * Print Elasticsearch API response. - * @param resp HTTP response - * @throws IOException an exception - */ - public static void dumpResponseBody(Response resp) throws IOException - { - InputStream is = resp.getEntity().getContent(); - dump(is); - is.close(); - } - - - /** - * Print content of an input stream. - * @param is input stream - * @throws IOException an exception - */ - public static void dump(InputStream is) throws IOException - { - BufferedReader rd = new BufferedReader(new InputStreamReader(is)); - - String line; - while((line = rd.readLine()) != null) - { - System.out.println(line); - } - } -} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/EsClientFactory.java b/src/main/java/gov/nasa/pds/registry/common/es/client/EsClientFactory.java deleted file mode 100644 index e6730ba..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/EsClientFactory.java +++ /dev/null @@ -1,35 +0,0 @@ -package gov.nasa.pds.registry.common.es.client; - - -import org.elasticsearch.client.RestClient; -import gov.nasa.pds.registry.common.util.JavaProps; - - -/** - * A factory class to create Elasticsearch Rest client instances. - * - * @author karpenko - */ -public class EsClientFactory -{ - /** - * Create Elasticsearch rest client. - * @param esUrl Elasticsearch URL, e.g., "http://localhost:9200" - * @param authPath Path to authentication configuration file. - * @return Elasticsearch rest client instance. - * @throws Exception an exception - */ - public static RestClient createRestClient(String esUrl, String authPath) throws Exception - { - EsRestClientBld bld = new EsRestClientBld(esUrl); - - if(authPath != null) - { - JavaProps props = new JavaProps(authPath); - bld.configureAuth(props); - } - - return bld.build(); - } - -} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java b/src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java deleted file mode 100644 index 791d542..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/EsRestClientBld.java +++ /dev/null @@ -1,73 +0,0 @@ -package gov.nasa.pds.registry.common.es.client; - -import org.apache.http.HttpHost; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestClientBuilder; - -import gov.nasa.pds.registry.common.util.JavaProps; - - -/** - * Utility class to build Elasticsearch rest client. - * - * @author karpenko - */ -public class EsRestClientBld -{ - private RestClientBuilder bld; - private ClientConfigCB clientCB; - private RequestConfigCB reqCB; - - - /** - * Constructor. - * @param url Elasticsearch URL, e.g., "http://localhost:9200" - * @throws Exception an exception - */ - public EsRestClientBld(String url) throws Exception - { - HttpHost host = EsUtils.parseEsUrl(url); - bld = RestClient.builder(host); - - clientCB = new ClientConfigCB(); - reqCB = new RequestConfigCB(); - } - - - /** - * Build the Elasticsearch rest client - * @return Elasticsearch rest client - */ - public RestClient build() - { - bld.setHttpClientConfigCallback(clientCB); - bld.setRequestConfigCallback(reqCB); - - return bld.build(); - } - - - /** - * Configure authentication - * @param props properties - * @throws Exception an exception - */ - public void configureAuth(JavaProps props) throws Exception - { - if(props == null) return; - - // Trust self-signed certificates - if(Boolean.TRUE.equals(props.getBoolean(ClientConstants.AUTH_TRUST_SELF_SIGNED))) - { - clientCB.setTrustSelfSignedCert(true); - } - - // Basic authentication - String user = props.getProperty("user"); - String pass = props.getProperty("password"); - if(user != null && pass != null) - { - clientCB.setUserPass(user, pass); - } - } -} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/client/HttpConnectionFactory.java b/src/main/java/gov/nasa/pds/registry/common/es/client/HttpConnectionFactory.java deleted file mode 100644 index 991d7f7..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/es/client/HttpConnectionFactory.java +++ /dev/null @@ -1,127 +0,0 @@ -package gov.nasa.pds.registry.common.es.client; - - -import java.net.HttpURLConnection; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.Base64; - -import javax.net.ssl.HttpsURLConnection; -import javax.net.ssl.SSLContext; - -import org.apache.http.HttpHost; - -import gov.nasa.pds.registry.common.util.JavaProps; - - -/** - * Factory class to create HTTP connections. - * - * @author karpenko - */ -public class HttpConnectionFactory -{ - private int timeout = 5000; - private URL url; - private HttpHost host; - private String authHeader; - - - /** - * Constructor. - * @param esUrl Elasticsearch URL, e.g., "http://localhost:9200" - * @param indexName Elasticsearch index name. - * @param api API name, e.g., "_bulk". - * @throws Exception an exception - */ - public HttpConnectionFactory(String esUrl, String indexName, String api) throws Exception - { - HttpHost host = EsUtils.parseEsUrl(esUrl); - this.url = new URL(host.toURI() + "/" + indexName + "/" + api); - } - - - /** - * Create HTTP connection - * @return HTTP connection - * @throws Exception an exception - */ - public HttpURLConnection createConnection() throws Exception - { - HttpURLConnection con = (HttpURLConnection)url.openConnection(); - con.setConnectTimeout(timeout); - con.setReadTimeout(timeout); - con.setAllowUserInteraction(false); - - if(authHeader != null) - { - con.setRequestProperty("Authorization", authHeader); - } - - return con; - } - - - /** - * Set connection timeout in seconds. - * - * @param timeoutSec timeout in seconds - */ - public void setTimeoutSec(int timeoutSec) - { - if(timeoutSec <= 0) throw new IllegalArgumentException("Timeout should be > 0"); - this.timeout = timeoutSec * 1000; - } - - - /** - * Get host name - * @return host name - */ - public String getHostName() - { - return host.getHostName(); - } - - - /** - * Set user name and password for basic authentication - * @param user user name - * @param pass password - */ - public void setBasicAuthentication(String user, String pass) - { - String auth = user + ":" + pass; - String b64auth = Base64.getEncoder().encodeToString(auth.getBytes(StandardCharsets.UTF_8)); - this.authHeader = "Basic " + b64auth; - } - - - /** - * Setup authentication parameters and TLS/SSL. - * @param authConfigFile Authentication configuration file. - * @throws Exception an exception - */ - public void initAuth(String authConfigFile) throws Exception - { - if(authConfigFile == null) return; - - JavaProps props = new JavaProps(authConfigFile); - - // Trust self-signed certificates - if(Boolean.TRUE.equals(props.getBoolean(ClientConstants.AUTH_TRUST_SELF_SIGNED))) - { - SSLContext sslCtx = SSLUtils.createTrustAllContext(); - HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory()); - } - - // Basic authentication - String user = props.getProperty("user"); - String pass = props.getProperty("password"); - if(user != null && pass != null) - { - setBasicAuthentication(user, pass); - } - } - -} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/BaseRequestBuilder.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/BaseRequestBuilder.java deleted file mode 100644 index b680873..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/BaseRequestBuilder.java +++ /dev/null @@ -1,68 +0,0 @@ -package gov.nasa.pds.registry.common.es.dao; - -import java.io.IOException; -import java.io.StringWriter; -import java.io.Writer; -import java.util.Collection; - -import com.google.gson.stream.JsonWriter; - -/** - * Methods to build JSON requests for Elasticsearch APIs. - * @author karpenko - */ -public class BaseRequestBuilder -{ - protected boolean pretty; - - /** - * Constructor - * @param pretty Format JSON for humans to read. - */ - public BaseRequestBuilder(boolean pretty) - { - this.pretty = pretty; - } - - - protected JsonWriter createJsonWriter(Writer writer) - { - JsonWriter jw = new JsonWriter(writer); - if (pretty) - { - jw.setIndent(" "); - } - - return jw; - } - - - /** - * Create multi get (_mget) request. - * @param ids list of IDs - * @return JSON - * @throws IOException an exception - */ - public String createMgetRequest(Collection ids) throws IOException - { - StringWriter wr = new StringWriter(); - JsonWriter jw = createJsonWriter(wr); - - jw.beginObject(); - jw.name("ids"); - - jw.beginArray(); - for(String id: ids) - { - jw.value(id); - } - jw.endArray(); - - jw.endObject(); - jw.close(); - - return wr.toString(); - } - - -} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DaoUtils.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DaoUtils.java index 681f06d..daf0b32 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DaoUtils.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DaoUtils.java @@ -1,6 +1,7 @@ package gov.nasa.pds.registry.common.es.dao; import java.io.BufferedReader; +import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.ArrayList; @@ -54,7 +55,7 @@ public static String getLastLine(InputStream is) - public static List parseList(JsonReader rd) throws Exception + public static List parseList(JsonReader rd) throws IOException { List list = new ArrayList<>(); @@ -71,7 +72,7 @@ public static List parseList(JsonReader rd) throws Exception } - public static Set parseSet(JsonReader rd) throws Exception + public static Set parseSet(JsonReader rd) throws IOException { Set set = new TreeSet<>(); diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java index 7d9bd91..134fb38 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/DataLoader.java @@ -4,13 +4,10 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.HttpURLConnection; import java.net.UnknownHostException; +import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -18,10 +15,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.google.gson.Gson; - -import gov.nasa.pds.registry.common.es.client.EsUtils; -import gov.nasa.pds.registry.common.es.client.HttpConnectionFactory; +import gov.nasa.pds.registry.common.ConnectionFactory; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.Response; import gov.nasa.pds.registry.common.util.CloseUtils; @@ -41,22 +37,21 @@ public class DataLoader private int totalRecords; private Logger log; - private HttpConnectionFactory conFactory; + private ConnectionFactory conFactory; /** * Constructor - * @param esUrl Elasticsearch URL, e.g., "http://localhost:9200" + * @param esUrl Elasticsearch URL, e.g., "app:/connections/direct/localhost.xml" * @param indexName Elasticsearch index name * @param authConfigFile Elasticsearch authentication configuration file * (see Registry Manager documentation for more info) * @throws Exception an exception */ - public DataLoader(String esUrl, String indexName, String authConfigFile) throws Exception + public DataLoader(ConnectionFactory conFactory) throws Exception { log = LogManager.getLogger(this.getClass()); - conFactory = new HttpConnectionFactory(esUrl, indexName, "_bulk?refresh=wait_for"); - conFactory.initAuth(authConfigFile); + this.conFactory = conFactory; } @@ -162,43 +157,25 @@ private String loadBatch(BufferedReader fileReader, String firstLine) throws Exc */ private String loadBatch(BufferedReader fileReader, String firstLine, int retries) throws Exception { - HttpURLConnection con = null; - OutputStreamWriter writer = null; - + ArrayList statements = new ArrayList(); try { - con = conFactory.createConnection(); - con.setDoInput(true); - con.setDoOutput(true); - con.setRequestMethod("POST"); - con.setRequestProperty("content-type", "application/x-ndjson; charset=utf-8"); - - writer = new OutputStreamWriter(con.getOutputStream(), "UTF-8"); - // First record String line1 = firstLine; String line2 = fileReader.readLine(); if(line2 == null) throw new Exception("Premature end of file"); - - writer.write(line1); - writer.write("\n"); - writer.write(line2); - writer.write("\n"); + statements.add(line1); + statements.add(line2); int numRecords = 1; while(numRecords < batchSize) { line1 = fileReader.readLine(); if(line1 == null) break; - line2 = fileReader.readLine(); if(line2 == null) throw new Exception("Premature end of file"); - - writer.write(line1); - writer.write("\n"); - writer.write(line2); - writer.write("\n"); - + statements.add(line1); + statements.add(line2); numRecords++; } @@ -208,53 +185,18 @@ private String loadBatch(BufferedReader fileReader, String firstLine, int retrie line1 = fileReader.readLine(); if(line1 != null && line1.isEmpty()) line1 = null; } - - writer.flush(); - writer.close(); - - // Check for Elasticsearch errors. - String respJson = getLastLine(con.getInputStream()); - log.debug(respJson); - - if(responseHasErrors(respJson)) - { - throw new Exception("Could not load data."); + int uploaded = this.loadBatch(statements); + totalRecords += uploaded; + + if (uploaded != numRecords) { + throw new Exception ("failed to upload all documents"); } - - totalRecords += numRecords; - return line1; } catch(UnknownHostException ex) { throw new Exception("Unknown host " + conFactory.getHostName()); } - catch(IOException ex) - { - if (retries > 0) { - String msg = ex.getMessage(); - log.warn("DataLoader.loadBatch() request failed due to \"" + msg + "\" ("+ retries +" retries remaining)"); - return loadBatch(fileReader, firstLine, retries - 1); - } - - // Get HTTP response code - int respCode = getResponseCode(con); - if(respCode <= 0) throw ex; - - // Try extracting JSON from multi-line error response (last line) - String json = getLastLine(con.getErrorStream()); - if(json == null) throw ex; - - // Parse error JSON to extract reason. - String msg = EsUtils.extractReasonFromJson(json); - if(msg == null) msg = json; - - throw new Exception(msg); - } - finally - { - CloseUtils.close(writer); - } } @@ -283,40 +225,19 @@ public int loadBatch(List data, Set errorLidvids, int retries) t if(data == null || data.isEmpty()) return 0; if(data.size() % 2 != 0) throw new Exception("Data list size should be an even number."); - HttpURLConnection con = null; - OutputStreamWriter writer = null; - try { - con = conFactory.createConnection(); - con.setDoInput(true); - con.setDoOutput(true); - con.setRequestMethod("POST"); - con.setRequestProperty("content-type", "application/x-ndjson; charset=utf-8"); - - writer = new OutputStreamWriter(con.getOutputStream(), "UTF-8"); - - for(int i = 0; i < data.size(); i+=2) - { - writer.write(data.get(i)); - writer.write("\n"); - writer.write(data.get(i+1)); - writer.write("\n"); - } - - writer.flush(); - writer.close(); - - // Read Elasticsearch response. - String respJson = DaoUtils.getLastLine(con.getInputStream()); - log.debug(respJson); - - // Check for Elasticsearch errors. - int failedCount = processErrors(respJson, errorLidvids); - // Calculate number of successfully saved records - // NOTE: data list has two lines per record (primary key + data) - int loadedCount = data.size() / 2 - failedCount; - return loadedCount; + Request.Bulk bulk = this.conFactory.createRestClient().createBulkRequest().setRefresh(Request.Bulk.Refresh.WaitFor).setIndex(this.conFactory.getIndexName()); + for (int index = 0 ; index < data.size() ; index++) { + bulk.add(data.get(index), data.get(++index)); + } + Response.Bulk response = this.conFactory.createRestClient().performRequest(bulk); + // Check for Elasticsearch errors. + int failedCount = processErrors(response, errorLidvids); + // Calculate number of successfully saved records + // NOTE: data list has two lines per record (primary key + data) + int loadedCount = data.size() / 2 - failedCount; + return loadedCount; } catch(UnknownHostException ex) { @@ -329,7 +250,8 @@ public int loadBatch(List data, Set errorLidvids, int retries) t log.warn("DataLoader.loadBatch() request failed due to \"" + msg + "\" ("+ retries +" retries remaining)"); return loadBatch(data, errorLidvids, retries - 1); } - + throw ex; +/* // Get HTTP response code int respCode = getResponseCode(con); if(respCode <= 0) throw ex; @@ -339,14 +261,11 @@ public int loadBatch(List data, Set errorLidvids, int retries) t if(json == null) throw ex; // Parse error JSON to extract reason. - String msg = EsUtils.extractReasonFromJson(json); + String msg = SearchResponseParser.extractReasonFromJson(json); if(msg == null) msg = json; throw new Exception(msg); - } - finally - { - CloseUtils.close(writer); + */ } } @@ -362,161 +281,24 @@ public int loadBatch(List data) throws Exception return loadBatch(data, null); } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private int processErrors(String resp, Set errorLidvids) - { - int numErrors = 0; - - try - { - // TODO: Use streaming parser. Stop parsing if there are no errors. - // Parse JSON response - Gson gson = new Gson(); - Map json = (Map)gson.fromJson(resp, Object.class); - - Boolean hasErrors = (Boolean)json.get("errors"); - if(hasErrors) - { - List list = (List)json.get("items"); - - // List size = batch size (one item per document) - for(Object item: list) - { - Map action = (Map)((Map)item).get("index"); - if(action == null) - { - action = (Map)((Map)item).get("create"); - if(action != null) - { - String status = String.valueOf(action.get("status")); - // For "create" requests status=409 means that the record already exists. - // It is not an error. We use "create" action to insert records which don't exist - // and keep existing records as is. We do this when loading an old LDD and more - // recent version of the LDD is already loaded. - // NOTE: Gson JSON parser stores numbers as floats. - // The string value is usually "409.0". Can it be something else? - if(status.startsWith("409")) - { - // Increment to properly report number of processed records. - numErrors++; - continue; - } - } - } - if(action == null) continue; - - String id = (String)action.get("_id"); - Map error = (Map)action.get("error"); - if(error != null) - { - String message = (String)error.get("reason"); - String sanitizedLidvid = id.replace('\r', ' ').replace('\n', ' '); // protect vs log spoofing see code-scanning alert #37 - String sanitizedMessage = message.replace('\r', ' ').replace('\n', ' '); // protect vs log spoofing - log.error("LIDVID = " + sanitizedLidvid + ", Message = " + sanitizedMessage); - numErrors++; - if(errorLidvids != null) errorLidvids.add(id); - } - } - } - - return numErrors; - } - catch(Exception ex) - { - return 0; - } - } - - - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private boolean responseHasErrors(String resp) - { - try - { - // Parse JSON response - Gson gson = new Gson(); - Map json = (Map)gson.fromJson(resp, Object.class); - - Boolean hasErrors = (Boolean)json.get("errors"); - if(hasErrors) - { - List list = (List)json.get("items"); - - // List size = batch size (one item per document) - // NOTE: Only few items in the list could have errors - for(Object item: list) - { - Map index = (Map)((Map)item).get("index"); - Map error = (Map)index.get("error"); - if(error != null) - { - String message = (String)error.get("reason"); - log.error(message); - return true; - } - } - } - - return false; - } - catch(Exception ex) - { - return false; - } - } - - - /** - * Get HTTP response code, e.g., 200 (OK) - * @param con HTTP connection - * @return HTTP response code, e.g., 200 (OK) - */ - private static int getResponseCode(HttpURLConnection con) - { - if(con == null) return -1; - - try - { - return con.getResponseCode(); - } - catch(Exception ex) - { - return -1; - } - } - - - /** - * This method is used to parse multi-line Elasticsearch error responses. - * JSON error response is on the last line of a message. - * @param is input stream - * @return Last line - */ - private String getLastLine(InputStream is) - { - String lastLine = null; - - try - { - BufferedReader rd = new BufferedReader(new InputStreamReader(is)); - - String line; - while((line = rd.readLine()) != null) - { - lastLine = line; + private int processErrors(Response.Bulk resp, Set errorLidvids) { + int numErrors = 0; + if (resp.errors()) { + for (Response.Bulk.Item item : resp.items()) { + if (item.error()) { + if (item.operation() == "create" && item.status() == 409) { + numErrors++; + } else { + String message = item.reason(); + String sanitizedLidvid = item.id().replace('\r', ' ').replace('\n', ' '); // protect vs log spoofing see code-scanning alert #37 + String sanitizedMessage = message.replace('\r', ' ').replace('\n', ' '); // protect vs log spoofing + log.error("LIDVID = " + sanitizedLidvid + ", Message = " + sanitizedMessage); + numErrors++; + if(errorLidvids != null) errorLidvids.add(item.id()); } + } } - catch(Exception ex) - { - log.info("Exception thrown in DataLoader.getLastLine() - please inform developer", ex); - } - finally - { - CloseUtils.close(is); - } - - return lastLine; + } + return numErrors; } } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/LidvidSet.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/LidvidSet.java index cb6c33f..a1de85c 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/LidvidSet.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/LidvidSet.java @@ -1,9 +1,26 @@ package gov.nasa.pds.registry.common.es.dao; import java.util.Set; +import gov.nasa.pds.registry.common.Response; -public class LidvidSet +public class LidvidSet implements Response.Get.IdSets { public Set lidvids; public Set lids; + public LidvidSet(Set lids, Set lidvids) { + this.lids = lids; + this.lidvids = lidvids; + } + public LidvidSet(Response.Get.IdSets ids) { + this.lids = ids.lids(); + this.lidvids = ids.lidvids(); + } + @Override + public Set lids() { + return this.lids; + } + @Override + public Set lidvids() { + return this.lidvids; + } } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java index d8c3dee..52a82d8 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductDao.java @@ -1,23 +1,12 @@ package gov.nasa.pds.registry.common.es.dao; -import java.io.InputStream; -import java.io.InputStreamReader; import java.net.URLEncoder; import java.util.Collection; import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; -import org.elasticsearch.client.RestClient; - -import com.google.gson.stream.JsonReader; -import com.google.gson.stream.JsonToken; - -import gov.nasa.pds.registry.common.util.CloseUtils; -import gov.nasa.pds.registry.common.util.LidVidUtils; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.Response; +import gov.nasa.pds.registry.common.ResponseException; +import gov.nasa.pds.registry.common.RestClient; /** * Product data access object. @@ -25,9 +14,7 @@ * @author karpenko */ public class ProductDao -{ - private Logger log; - +{ private RestClient client; private String indexName; @@ -38,8 +25,6 @@ public class ProductDao */ public ProductDao(RestClient client, String indexName) { - log = LogManager.getLogger(this.getClass()); - this.client = client; this.indexName = indexName; } @@ -55,18 +40,20 @@ public String getProductClass(String lidvid) throws Exception { if(lidvid == null) return null; - String reqUrl = "/" + indexName + "/_doc/" + lidvid + "?_source=product_class"; - Request req = new Request("GET", reqUrl); - Response resp = null; + Request.Get req = client.createGetRequest() + .includeField("product_class") + .setId(lidvid) + .setIndex(this.indexName); + Response.Get resp = null; try { resp = client.performRequest(req); + return resp.productClass(); } catch(ResponseException ex) { - resp = ex.getResponse(); - int code = resp.getStatusLine().getStatusCode(); + int code = ex.statusCode(); // Invalid LIDVID if(code == 404 || code == 405) { @@ -77,37 +64,6 @@ public String getProductClass(String lidvid) throws Exception throw ex; } } - - InputStream is = null; - - try - { - is = resp.getEntity().getContent(); - JsonReader rd = new JsonReader(new InputStreamReader(is)); - - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("_source".equals(name)) - { - return parseProductClassSource(rd); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); - } - finally - { - CloseUtils.close(is); - } - - return null; } @@ -141,18 +97,16 @@ else if(type == 'S') query = URLEncoder.encode(query, "UTF-8"); // Request URL - String reqUrl = "/" + indexName + "-refs/_count?q=" + query; - Request req = new Request("GET", reqUrl); - Response resp = null; - + Request.Count req = client.createCountRequest() + .setIndex(this.indexName + "-refs") + .setQuery(query); try { - resp = client.performRequest(req); + return (int)client.performRequest(req); } catch(ResponseException ex) { - resp = ex.getResponse(); - int code = resp.getStatusLine().getStatusCode(); + int code = ex.statusCode(); // Invalid LIDVID if(code == 404 || code == 405) { @@ -163,37 +117,6 @@ else if(type == 'S') throw ex; } } - - InputStream is = null; - - try - { - is = resp.getEntity().getContent(); - JsonReader rd = new JsonReader(new InputStreamReader(is)); - - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("count".equals(name)) - { - return rd.nextInt(); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); - } - finally - { - CloseUtils.close(is); - } - - return 0; } @@ -211,18 +134,17 @@ public List getRefs(String collectionLidVid, char type, int page) throws if(collectionLidVid == null) return null; String docId = collectionLidVid + "::" + type + page; - String reqUrl = "/" + indexName + "-refs/_doc/" + docId + "?_source=product_lidvid"; - Request req = new Request("GET", reqUrl); - Response resp = null; - + Request.Get req = client.createGetRequest() + .includeField("product_lidvid") + .setId(docId) + .setIndex(this.indexName + "-refs"); try { - resp = client.performRequest(req); + return client.performRequest(req).refs(); } catch(ResponseException ex) { - resp = ex.getResponse(); - int code = resp.getStatusLine().getStatusCode(); + int code = ex.statusCode(); // Invalid LIDVID if(code == 404 || code == 405) { @@ -233,37 +155,6 @@ public List getRefs(String collectionLidVid, char type, int page) throws throw ex; } } - - InputStream is = null; - - try - { - is = resp.getEntity().getContent(); - JsonReader rd = new JsonReader(new InputStreamReader(is)); - - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("_source".equals(name)) - { - return parseRefs(rd); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); - } - finally - { - CloseUtils.close(is); - } - - return null; } @@ -277,79 +168,12 @@ public void updateArchiveStatus(Collection lidvids, String status) throw { if(lidvids == null || status == null) return; - String json = ProductRequestBuilder.buildUpdateStatusJson(lidvids, status); - log.debug("Request:\n" + json); - - String reqUrl = "/" + indexName + "/_bulk"; //?refresh=wait_for"; - Request req = new Request("POST", reqUrl); - req.setJsonEntity(json); - - Response resp = client.performRequest(req); - - // Check for Elasticsearch errors. - InputStream is = null; - InputStreamReader rd = null; - try - { - is = resp.getEntity().getContent(); - rd = new InputStreamReader(is); - - BulkResponseParser parser = new BulkResponseParser(); - parser.parse(rd); - } - finally - { - CloseUtils.close(rd); - CloseUtils.close(is); - } - } - - - private static String parseProductClassSource(JsonReader rd) throws Exception - { - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("product_class".equals(name)) - { - return rd.nextString(); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); - - return null; - } - - - private static List parseRefs(JsonReader rd) throws Exception - { - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("product_lidvid".equals(name)) - { - return DaoUtils.parseList(rd); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); + Request.Bulk req = client.createBulkRequest() + .buildUpdateStatus(lidvids, status) + .setIndex(this.indexName); - return null; + client.performRequest(req).logErrors();; } - /** * Get collection references of a bundle. References can be either LIDs, LIDVIDs or both. @@ -361,18 +185,18 @@ public LidvidSet getCollectionIds(String bundleLidvid) throws Exception { if(bundleLidvid == null) return null; - String reqUrl = "/" + indexName + "/_doc/" + bundleLidvid + "?_source=ref_lidvid_collection,ref_lid_collection"; - Request req = new Request("GET", reqUrl); - Response resp = null; - + Request.Get req = client.createGetRequest() + .includeField("ref_lidvid_collection") + .includeField("ref_lid_collection") + .setId(bundleLidvid) + .setIndex(this.indexName); try { - resp = client.performRequest(req); + return new LidvidSet(client.performRequest(req).ids()); } catch(ResponseException ex) { - resp = ex.getResponse(); - int code = resp.getStatusLine().getStatusCode(); + int code = ex.statusCode(); // Invalid LIDVID if(code == 404 || code == 405) { @@ -383,83 +207,7 @@ public LidvidSet getCollectionIds(String bundleLidvid) throws Exception throw ex; } } - - LidvidSet collectionIds = null; - InputStream is = null; - - try - { - is = resp.getEntity().getContent(); - JsonReader rd = new JsonReader(new InputStreamReader(is)); - - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("_source".equals(name)) - { - collectionIds = parseCollectionIdsSource(rd); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); - } - finally - { - CloseUtils.close(is); - } - - if(collectionIds == null || collectionIds.lidvids == null - || collectionIds.lids == null) return collectionIds; - - // Harvest converts LIDVIDs to LIDs, so let's delete those converted LIDs. - for(String lidvid: collectionIds.lidvids) - { - String lid = LidVidUtils.lidvidToLid(lidvid); - if(lid != null) - { - collectionIds.lids.remove(lid); - } - } - - - return collectionIds; } - - - private static LidvidSet parseCollectionIdsSource(JsonReader rd) throws Exception - { - LidvidSet ids = new LidvidSet(); - - rd.beginObject(); - - while(rd.hasNext() && rd.peek() != JsonToken.END_OBJECT) - { - String name = rd.nextName(); - if("ref_lid_collection".equals(name)) - { - ids.lids = DaoUtils.parseSet(rd); - } - else if("ref_lidvid_collection".equals(name)) - { - ids.lidvids = DaoUtils.parseSet(rd); - } - else - { - rd.skipValue(); - } - } - - rd.endObject(); - - return ids; - } - /** * Given a list of LIDs, find latest versions @@ -471,45 +219,17 @@ public List getLatestLidVids(Collection lids) throws Exception { if(lids == null || lids.isEmpty()) return null; - String json = ProductRequestBuilder.buildGetLatestLidVidsJson(lids); - log.debug("getGetLatestLidVids() request: " + json); - - if(json == null) return null; - - String reqUrl = "/" + indexName + "/_search/"; - Request req = new Request("GET", reqUrl); - req.setJsonEntity(json); - - Response resp = null; - + Request.Search req = client.createSearchRequest() + .setIndex(this.indexName) + .buildLatestLidVids(lids); try { - resp = client.performRequest(req); + return client.performRequest(req).latestLidvids(); } catch(ResponseException ex) { throw ex; } - - //DebugUtils.dumpResponseBody(resp); - - InputStream is = null; - InputStreamReader rd = null; - - try - { - is = resp.getEntity().getContent(); - rd = new InputStreamReader(is); - - LatestLidsResponseParser parser = new LatestLidsResponseParser(); - parser.parse(rd); - return parser.getLidvids(); - } - finally - { - CloseUtils.close(rd); - CloseUtils.close(is); - } } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductRequestBuilder.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductRequestBuilder.java deleted file mode 100644 index 3522170..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/ProductRequestBuilder.java +++ /dev/null @@ -1,134 +0,0 @@ -package gov.nasa.pds.registry.common.es.dao; - -import java.io.IOException; -import java.io.StringWriter; -import java.util.Collection; - -import com.google.gson.stream.JsonWriter; - -import gov.nasa.pds.registry.common.meta.Metadata; -import gov.nasa.pds.registry.common.util.CloseUtils; - -/** - * Builds Elasticsearch JSON queries - * @author karpenko - */ -public class ProductRequestBuilder -{ - /** - * Build update product archive status JSON request - * @param lidvids list of LIDVIDs to update - * @param status new status - * @return JSON - */ - public static String buildUpdateStatusJson(Collection lidvids, String status) - { - if(lidvids == null || lidvids.isEmpty()) return null; - if(status == null || status.isEmpty()) throw new IllegalArgumentException("Status could not be null or empty."); - - StringBuilder bld = new StringBuilder(); - String dataLine = "{ \"doc\" : {\"" + Metadata.FLD_ARCHIVE_STATUS + "\" : \"" + status + "\"} }\n"; - - // Build NJSON (new-line delimited JSON) - for(String lidvid: lidvids) - { - // Line 1: Elasticsearch document ID - bld.append("{ \"update\" : {\"_id\" : \"" + lidvid + "\" } }\n"); - // Line 2: Data - bld.append(dataLine); - } - - return bld.toString(); - } - - - /** - * Build aggregation query to select latest versions of lids - * @param lids list of LIDs - * @return JSON - */ - public static String buildGetLatestLidVidsJson(Collection lids) throws IOException - { - if(lids == null || lids.isEmpty()) return null; - - JsonWriter jw = null; - - try - { - StringWriter strWriter = new StringWriter(); - jw = new JsonWriter(strWriter); - - jw.beginObject(); - - jw.name("_source").value(false); - jw.name("size").value(0); - - // Query - jw.name("query"); - jw.beginObject(); - - jw.name("terms"); - jw.beginObject(); - - jw.name("lid"); - jw.beginArray(); - for(String lid: lids) - { - jw.value(lid); - } - jw.endArray(); - - jw.endObject(); // terms - jw.endObject(); // query - - // Aggs - jw.name("aggs"); - jw.beginObject(); - - jw.name("lids"); - jw.beginObject(); - - jw.name("terms"); - jw.beginObject(); - jw.name("field").value("lid"); - jw.name("size").value(5000); - jw.endObject(); - - jw.name("aggs"); - jw.beginObject(); - jw.name("latest"); - jw.beginObject(); - jw.name("top_hits"); - jw.beginObject(); - - jw.name("sort"); - jw.beginArray(); - jw.beginObject(); - jw.name("vid"); - jw.beginObject(); - jw.name("order").value("desc"); - jw.endObject(); - jw.endObject(); - jw.endArray(); - - jw.name("_source").value(false); - jw.name("size").value(1); - - jw.endObject(); // top_hits - jw.endObject(); // latest - jw.endObject(); // aggs - - jw.endObject(); // lids - jw.endObject(); // aggs - - jw.endObject(); - - return strWriter.toString(); - } - finally - { - CloseUtils.close(jw); - } - } - -} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DDRequestBuilder.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DDRequestBuilder.java deleted file mode 100644 index 4484e95..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DDRequestBuilder.java +++ /dev/null @@ -1,178 +0,0 @@ -package gov.nasa.pds.registry.common.es.dao.dd; - -import java.io.IOException; -import java.io.StringWriter; - -import com.google.gson.stream.JsonWriter; - -import gov.nasa.pds.registry.common.es.dao.BaseRequestBuilder; - - -/** - * Methods to build JSON requests for Elasticsearch APIs. - * @author karpenko - */ -public class DDRequestBuilder extends BaseRequestBuilder -{ - /** - * Constructor - * @param pretty Format JSON for humans to read. - */ - public DDRequestBuilder(boolean pretty) - { - super(pretty); - } - - /** - * Constructor - */ - public DDRequestBuilder() - { - this(false); - } - - - /** - * Create get data dictionary (LDD) info request. - * @param namespace LDD namespace ID, such as 'pds', 'cart', etc. - * @return Elasticsearch query in JSON format - * @throws IOException an exception - */ - public String createListLddsRequest(String namespace) throws IOException - { - StringWriter wr = new StringWriter(); - JsonWriter jw = createJsonWriter(wr); - - jw.beginObject(); - // Size (number of records to return) - jw.name("size").value(1000); - - // Start query - jw.name("query"); - jw.beginObject(); - jw.name("bool"); - jw.beginObject(); - - jw.name("must"); - jw.beginArray(); - appendMatch(jw, "class_ns", "registry"); - appendMatch(jw, "class_name", "LDD_Info"); - if(namespace != null) - { - appendMatch(jw, "attr_ns", namespace); - } - jw.endArray(); - - jw.endObject(); - jw.endObject(); - // End query - - // Start source - jw.name("_source"); - jw.beginArray(); - jw.value("date").value("attr_name").value("attr_ns").value("im_version"); - jw.endArray(); - // End source - - jw.endObject(); - jw.close(); - - return wr.toString(); - } - - - public String createListFieldsRequest(String dataType) throws IOException - { - StringWriter wr = new StringWriter(); - JsonWriter jw = createJsonWriter(wr); - - jw.beginObject(); - // Size (number of records to return) - jw.name("size").value(1000); - - // Start query - jw.name("query"); - jw.beginObject(); - jw.name("bool"); - jw.beginObject(); - - jw.name("must"); - jw.beginArray(); - appendMatch(jw, "es_data_type", dataType); - jw.endArray(); - - jw.endObject(); - jw.endObject(); - // End query - - // Start source - jw.name("_source"); - jw.beginArray(); - jw.value("es_field_name"); - jw.endArray(); - // End source - - jw.endObject(); - jw.close(); - - return wr.toString(); - } - - - /** - * Create get data dictionary (LDD) info request. - * @param namespace LDD namespace ID, such as 'pds', 'cart', etc. - * @return Elasticsearch query in JSON format - * @throws IOException an exception - */ - public String createGetLddInfoRequest(String namespace) throws IOException - { - StringWriter wr = new StringWriter(); - JsonWriter jw = createJsonWriter(wr); - - jw.beginObject(); - // Size (number of records to return) - jw.name("size").value(1000); - - // Start query - jw.name("query"); - jw.beginObject(); - jw.name("bool"); - jw.beginObject(); - - jw.name("must"); - jw.beginArray(); - appendMatch(jw, "class_ns", "registry"); - appendMatch(jw, "class_name", "LDD_Info"); - appendMatch(jw, "attr_ns", namespace); - jw.endArray(); - - jw.endObject(); - jw.endObject(); - // End query - - // Start source - jw.name("_source"); - jw.beginArray(); - jw.value("date").value("attr_name"); - jw.endArray(); - // End source - - jw.endObject(); - jw.close(); - - return wr.toString(); - } - - - private static void appendMatch(JsonWriter jw, String field, String value) throws IOException - { - jw.beginObject(); - jw.name("match"); - jw.beginObject(); - jw.name(field).value(value); - jw.endObject(); - jw.endObject(); - } - -} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java index 719e57f..48bf1ec 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/dd/DataDictionaryDao.java @@ -1,21 +1,11 @@ package gov.nasa.pds.registry.common.es.dao.dd; -import java.time.Instant; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; - -import gov.nasa.pds.registry.common.es.client.SearchResponseParser; -import gov.nasa.pds.registry.common.es.dao.schema.SchemaRequestBuilder; +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.common.util.Tuple; @@ -26,8 +16,6 @@ */ public class DataDictionaryDao { - private Logger log; - private RestClient client; private String indexName; @@ -39,42 +27,11 @@ public class DataDictionaryDao */ public DataDictionaryDao(RestClient client, String indexName) { - log = LogManager.getLogger(this.getClass()); - this.client = client; this.indexName = indexName; } - /** - * Inner private class to parse LDD information response from Elasticsearch. - * @author karpenko - */ - private static class GetLddInfoRespParser extends SearchResponseParser implements SearchResponseParser.Callback - { - public LddVersions info; - - public GetLddInfoRespParser() - { - info = new LddVersions(); - } - - @Override - public void onRecord(String id, Object rec) throws Exception - { - if(rec instanceof Map) - { - @SuppressWarnings("rawtypes") - Map map = (Map)rec; - - String strDate = (String)map.get("date"); - info.updateDate(strDate); - - String file = (String)map.get("attr_name"); - info.addSchemaFile(file); - } - } - } /** @@ -85,64 +42,13 @@ public void onRecord(String id, Object rec) throws Exception */ public LddVersions getLddInfo(String namespace) throws Exception { - DDRequestBuilder bld = new DDRequestBuilder(); - String json = bld.createListLddsRequest(namespace); - - Request req = new Request("GET", "/" + indexName + "-dd/_search"); - req.setJsonEntity(json); - Response resp = client.performRequest(req); - - GetLddInfoRespParser parser = new GetLddInfoRespParser(); - parser.parseResponse(resp, parser); - return parser.info; + Request.Search req = client.createSearchRequest() + .buildListLdds(namespace) + .setIndex(indexName + "-dd"); + return client.performRequest(req).lddInfo(); } - /** - * Inner private class to parse LDD information response from Elasticsearch. - * @author karpenko - */ - private static class ListLddsParser extends SearchResponseParser implements SearchResponseParser.Callback - { - public List list; - - public ListLddsParser() - { - list = new ArrayList<>(); - } - - @Override - public void onRecord(String id, Object rec) throws Exception - { - if(rec instanceof Map) - { - @SuppressWarnings("rawtypes") - Map map = (Map)rec; - - LddInfo info = new LddInfo(); - - // Namespace - info.namespace = (String)map.get("attr_ns"); - - // Date - String str = (String)map.get("date"); - if(str != null && !str.isEmpty()) - { - info.date = Instant.parse(str); - } - - // Versions - info.imVersion = (String)map.get("im_version"); - - // File name - info.file = (String)map.get("attr_name"); - - list.add(info); - } - } - } - - /** * List registered LDDs * @param namespace if this parameter is null list all LDDs @@ -151,48 +57,12 @@ public void onRecord(String id, Object rec) throws Exception */ public List listLdds(String namespace) throws Exception { - DDRequestBuilder bld = new DDRequestBuilder(); - String json = bld.createListLddsRequest(namespace); - - Request req = new Request("GET", "/" + indexName + "-dd/_search"); - req.setJsonEntity(json); - Response resp = client.performRequest(req); - - ListLddsParser parser = new ListLddsParser(); - parser.parseResponse(resp, parser); - return parser.list; + Request.Search req = client.createSearchRequest() + .buildListLdds(namespace) + .setIndex(this.indexName + "-dd"); + return client.performRequest(req).ldds(); } - - - /** - * Inner private class to parse LDD information response from Elasticsearch. - * @author karpenko - */ - private static class ListFieldsParser extends SearchResponseParser implements SearchResponseParser.Callback - { - public Set list; - - public ListFieldsParser() - { - list = new HashSet<>(200); - } - - @Override - public void onRecord(String id, Object rec) throws Exception - { - if(rec instanceof Map) - { - @SuppressWarnings("rawtypes") - Map map = (Map)rec; - - String fieldName = (String)map.get("es_field_name"); - list.add(fieldName); - } - } - } - - /** * Get field names by Elasticsearch type, such as "boolean" or "date". * @return a set of field names @@ -200,16 +70,10 @@ public void onRecord(String id, Object rec) throws Exception */ public Set getFieldNamesByEsType(String esType) throws Exception { - DDRequestBuilder bld = new DDRequestBuilder(); - String json = bld.createListFieldsRequest(esType); - - Request req = new Request("GET", "/" + indexName + "-dd/_search"); - req.setJsonEntity(json); - Response resp = client.performRequest(req); - - ListFieldsParser parser = new ListFieldsParser(); - parser.parseResponse(resp, parser); - return parser.list; + Request.Search req = client.createSearchRequest() + .buildListFields(esType) + .setIndex(this.indexName + "-dd"); + return client.performRequest(req).fields(); } @@ -226,58 +90,12 @@ public Set getFieldNamesByEsType(String esType) throws Exception public List getDataTypes(Collection ids, boolean stringForMissing) throws Exception { if(ids == null || ids.isEmpty()) return null; - - List dtInfo = new ArrayList(); - // Create request - Request req = new Request("GET", "/" + indexName + "-dd/_mget?_source=es_data_type"); - - // Create request body - SchemaRequestBuilder bld = new SchemaRequestBuilder(); - String json = bld.createMgetRequest(ids); - req.setJsonEntity(json); - - // Call ES - Response resp = client.performRequest(req); - GetDataTypesResponseParser parser = new GetDataTypesResponseParser(); - List records = parser.parse(resp.getEntity()); - - // Process response (list of fields) - boolean missing = false; - - for(GetDataTypesResponseParser.Record rec: records) - { - if(rec.found) - { - dtInfo.add(new Tuple(rec.id, rec.esDataType)); - } - // There is no data type for this field in ES registry-dd index - else - { - // Automatically assign data type for known fields - if(rec.id.startsWith("ref_lid_") || rec.id.startsWith("ref_lidvid_") - || rec.id.endsWith("_Area")) - { - dtInfo.add(new Tuple(rec.id, "keyword")); - continue; - } - - if(stringForMissing) - { - log.warn("Could not find datatype for field " + rec.id + ". Will use 'keyword'"); - dtInfo.add(new Tuple(rec.id, "keyword")); - } - else - { - log.error("Could not find datatype for field " + rec.id); - missing = true; - } - } - } - - if(stringForMissing == false && missing == true) throw new DataTypeNotFoundException(); - - return dtInfo; + Request.Get req = client.createMGetRequest() + .setIds(ids) + .includeField("es_data_type") + .setIndex(this.indexName + "-dd"); + return this.client.performRequest(req).dataTypes(stringForMissing); } } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java index 6b87295..2ad13a0 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaDao.java @@ -2,13 +2,8 @@ import java.util.List; import java.util.Set; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.RestClient; - +import gov.nasa.pds.registry.common.Request; +import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.common.util.Tuple; @@ -20,7 +15,6 @@ */ public class SchemaDao { - private Logger log; private RestClient client; private String indexName; @@ -32,7 +26,6 @@ public class SchemaDao */ public SchemaDao(RestClient client, String indexName) { - log = LogManager.getLogger(this.getClass()); this.client = client; this.indexName = indexName; } @@ -45,11 +38,8 @@ public SchemaDao(RestClient client, String indexName) */ public Set getFieldNames() throws Exception { - Request req = new Request("GET", "/" + indexName + "/_mappings"); - Response resp = client.performRequest(req); - - MappingsParser parser = new MappingsParser(indexName); - return parser.parse(resp.getEntity()); + Request.Mapping req = client.createMappingRequest().setIndex(indexName); + return client.performRequest(req).fieldNames(); } @@ -62,11 +52,9 @@ public void updateSchema(List fields) throws Exception { if(fields == null || fields.isEmpty()) return; - SchemaRequestBuilder bld = new SchemaRequestBuilder(); - String json = bld.createUpdateSchemaRequest(fields); - - Request req = new Request("PUT", "/" + indexName + "/_mapping"); - req.setJsonEntity(json); + Request.Mapping req = client.createMappingRequest() + .buildUpdateFieldSchema(fields) + .setIndex(this.indexName); client.performRequest(req); } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaRequestBuilder.java b/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaRequestBuilder.java deleted file mode 100644 index d66aeb4..0000000 --- a/src/main/java/gov/nasa/pds/registry/common/es/dao/schema/SchemaRequestBuilder.java +++ /dev/null @@ -1,67 +0,0 @@ -package gov.nasa.pds.registry.common.es.dao.schema; - -import java.io.IOException; -import java.io.StringWriter; -import java.util.List; - -import com.google.gson.stream.JsonWriter; - -import gov.nasa.pds.registry.common.es.dao.BaseRequestBuilder; -import gov.nasa.pds.registry.common.util.Tuple; - - -/** - * Methods to build JSON requests for Elasticsearch APIs. - * @author karpenko - */ -public class SchemaRequestBuilder extends BaseRequestBuilder -{ - /** - * Constructor - * @param pretty Format JSON for humans to read. - */ - public SchemaRequestBuilder(boolean pretty) - { - super(pretty); - } - - /** - * Constructor - */ - public SchemaRequestBuilder() - { - this(false); - } - - - /** - * Create update Elasticsearch schema request - * @param fields A list of fields to add. Each field tuple has a name and a data type. - * @return Elasticsearch query in JSON format - * @throws IOException an exception - */ - public String createUpdateSchemaRequest(List fields) throws IOException - { - StringWriter wr = new StringWriter(); - JsonWriter jw = createJsonWriter(wr); - - jw.beginObject(); - - jw.name("properties"); - jw.beginObject(); - for(Tuple field: fields) - { - jw.name(field.item1); - jw.beginObject(); - jw.name("type").value(field.item2); - jw.endObject(); - } - jw.endObject(); - - jw.endObject(); - jw.close(); - - return wr.toString(); - } - -} diff --git a/src/main/java/gov/nasa/pds/registry/common/es/service/CollectionInventoryWriter.java b/src/main/java/gov/nasa/pds/registry/common/es/service/CollectionInventoryWriter.java index a102ade..51c30f0 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/service/CollectionInventoryWriter.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/service/CollectionInventoryWriter.java @@ -6,8 +6,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - -import gov.nasa.pds.registry.common.cfg.RegistryCfg; +import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.es.dao.DataLoader; import gov.nasa.pds.registry.common.meta.InventoryBatchReader; import gov.nasa.pds.registry.common.util.CloseUtils; @@ -44,10 +43,10 @@ public class CollectionInventoryWriter /** * Constructor */ - public CollectionInventoryWriter(RegistryCfg cfg) throws Exception + public CollectionInventoryWriter(ConnectionFactory conFact) throws Exception { log = LogManager.getLogger(this.getClass()); - loader = new DataLoader(cfg.url, cfg.indexName + "-refs", cfg.authFile); + loader = new DataLoader(conFact.clone().setIndexName(conFact.getIndexName()+"-refs")); } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/service/JsonLddLoader.java b/src/main/java/gov/nasa/pds/registry/common/es/service/JsonLddLoader.java index 136342f..4fd6815 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/service/JsonLddLoader.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/service/JsonLddLoader.java @@ -7,7 +7,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; - +import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.dd.LddEsJsonWriter; import gov.nasa.pds.registry.common.dd.LddUtils; import gov.nasa.pds.registry.common.dd.Pds2EsDataTypeMap; @@ -41,12 +41,12 @@ public class JsonLddLoader * @param authFilePath authentication configuration file * @throws Exception an exception */ - public JsonLddLoader(DataDictionaryDao dao, String esUrl, String indexName, String authFilePath) throws Exception + public JsonLddLoader(DataDictionaryDao dao, ConnectionFactory conFact) throws Exception { log = LogManager.getLogger(this.getClass()); dtMap = new Pds2EsDataTypeMap(); - loader = new DataLoader(esUrl, indexName + "-dd", authFilePath); + loader = new DataLoader(conFact.clone().setIndexName(conFact.getIndexName() + "-dd")); this.dao = dao; } diff --git a/src/main/java/gov/nasa/pds/registry/common/es/service/SchemaUpdater.java b/src/main/java/gov/nasa/pds/registry/common/es/service/SchemaUpdater.java index 04339d0..9273df8 100644 --- a/src/main/java/gov/nasa/pds/registry/common/es/service/SchemaUpdater.java +++ b/src/main/java/gov/nasa/pds/registry/common/es/service/SchemaUpdater.java @@ -16,7 +16,7 @@ import gov.nasa.pds.registry.common.util.Tuple; import gov.nasa.pds.registry.common.es.dao.dd.DataDictionaryDao; -import gov.nasa.pds.registry.common.cfg.RegistryCfg; +import gov.nasa.pds.registry.common.ConnectionFactory; import gov.nasa.pds.registry.common.es.dao.schema.SchemaDao; import gov.nasa.pds.registry.common.es.dao.dd.LddVersions; @@ -42,7 +42,7 @@ public class SchemaUpdater * @param cfg Registry (Elasticsearch) configuration * @throws Exception */ - public SchemaUpdater(RegistryCfg cfg, DataDictionaryDao ddDao, SchemaDao schemaDao) throws Exception + public SchemaUpdater(ConnectionFactory conFact, DataDictionaryDao ddDao, SchemaDao schemaDao) throws Exception { log = LogManager.getLogger(this.getClass()); @@ -51,7 +51,7 @@ public SchemaUpdater(RegistryCfg cfg, DataDictionaryDao ddDao, SchemaDao schemaD fileDownloader = new FileDownloader(true); - lddLoader = new JsonLddLoader(ddDao, cfg.url, cfg.indexName, cfg.authFile); + lddLoader = new JsonLddLoader(ddDao, conFact); lddLoader.loadPds2EsDataTypeMap(LddUtils.getPds2EsDataTypeCfgFile("HARVEST_HOME")); } diff --git a/src/main/java/gov/nasa/pds/registry/common/meta/FileMetadataExtractor.java b/src/main/java/gov/nasa/pds/registry/common/meta/FileMetadataExtractor.java index c7b3288..8d355ee 100644 --- a/src/main/java/gov/nasa/pds/registry/common/meta/FileMetadataExtractor.java +++ b/src/main/java/gov/nasa/pds/registry/common/meta/FileMetadataExtractor.java @@ -16,8 +16,6 @@ import java.util.zip.DeflaterOutputStream; import org.apache.commons.codec.binary.Hex; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.apache.tika.Tika; import org.json.JSONObject; import org.json.XML; @@ -30,9 +28,7 @@ * @author karpenko */ public class FileMetadataExtractor -{ - private Logger log; - +{ private MessageDigest md5Digest; private byte[] buf; private Tika tika; @@ -47,9 +43,7 @@ public class FileMetadataExtractor * @throws Exception and exception */ public FileMetadataExtractor() throws Exception - { - log = LogManager.getLogger(this.getClass()); - + { md5Digest = MessageDigest.getInstance("MD5"); buf = new byte[1024 * 16]; tika = new Tika(); @@ -265,6 +259,10 @@ private String getFileRef(File file, List refRules) if(filePath.startsWith(rule.prefix)) { filePath = rule.replacement + filePath.substring(rule.prefix.length()); + String protocol = filePath.substring(0, filePath.indexOf(':')+3); // include :// as part of protocol + String body = filePath.substring(protocol.length()); + while (body.contains("//")) body = body.replace("//", "/"); + filePath = protocol + body; break; } } diff --git a/src/main/java/gov/nasa/pds/registry/common/util/Tuple.java b/src/main/java/gov/nasa/pds/registry/common/util/Tuple.java index e7ea681..027a0ad 100644 --- a/src/main/java/gov/nasa/pds/registry/common/util/Tuple.java +++ b/src/main/java/gov/nasa/pds/registry/common/util/Tuple.java @@ -9,8 +9,8 @@ public class Tuple { public String item1; public String item2; - - + public Tuple() { + } /** * Constructor * @param item1 value 1 diff --git a/src/main/java/gov/nasa/pds/registry/common/util/json/BaseNJsonWriter.java b/src/main/java/gov/nasa/pds/registry/common/util/json/BaseNJsonWriter.java index 339dc4d..31346c1 100644 --- a/src/main/java/gov/nasa/pds/registry/common/util/json/BaseNJsonWriter.java +++ b/src/main/java/gov/nasa/pds/registry/common/util/json/BaseNJsonWriter.java @@ -18,6 +18,7 @@ * * @param A data record to write. */ +@SuppressWarnings("hiding") public abstract class BaseNJsonWriter implements Closeable { protected FileWriter writer; diff --git a/src/main/resources/connections/cognito/development.xml b/src/main/resources/connections/cognito/development.xml new file mode 100644 index 0000000..f9d8d07 --- /dev/null +++ b/src/main/resources/connections/cognito/development.xml @@ -0,0 +1,7 @@ + + + 47d9j6ks9un4errq6pnbu0bc1r + diff --git a/src/main/resources/connections/direct/localhost.xml b/src/main/resources/connections/direct/localhost.xml new file mode 100644 index 0000000..ae8ab14 --- /dev/null +++ b/src/main/resources/connections/direct/localhost.xml @@ -0,0 +1,4 @@ + + + https://localhost:9200 + diff --git a/src/main/resources/connections/direct/localhost1.xml b/src/main/resources/connections/direct/localhost1.xml new file mode 100644 index 0000000..7b5d93e --- /dev/null +++ b/src/main/resources/connections/direct/localhost1.xml @@ -0,0 +1,4 @@ + + + https://localhost:9200 + diff --git a/src/main/resources/connections/direct/localhost2.xml b/src/main/resources/connections/direct/localhost2.xml new file mode 100644 index 0000000..8d78b7e --- /dev/null +++ b/src/main/resources/connections/direct/localhost2.xml @@ -0,0 +1,4 @@ + + + https://localhost:9200 + diff --git a/src/main/resources/registry-connection.xsd b/src/main/resources/registry-connection.xsd new file mode 100644 index 0000000..263ff91 --- /dev/null +++ b/src/main/resources/registry-connection.xsd @@ -0,0 +1,119 @@ + + + + + + + + + Currently, cognito to serverless opensearh requires three bits of + information: + 1. clientID: magic ID which must be given and is not a secret but + sure looks like one + 3. endpoint: the endpoint for serverless opensearch + 2. @gateway: an endpoint that exchanges the cognito authentication + to an IAM role used for access control with serverless + opensearch on AWS + 3. @IDP: the cognito authentication endpoint that exchanges the + username/password to a set of tokens used by the gateway. + + + + + + + + + + + + + + + Currently, connecting to an opensearch service directly requires just + two pieces of information: + 1. URL: the URL to directly contact an opensearch instance and its + API to include authentication + 2. @trustSelfSigned: when running locally for testing or development + it is often necessary to use self signed + certificates. Setting this option to true + will all for self signed certificates. + + An optional 3rd piece of information @sdk is the Java SDK version to use + for the connection. The traditional SDK is 1. The newer SDK 2 was + introduced with the serverless opensearch development and beyond can + also be used. The default is 2 as we are deprecating the use of 1. + + + + + + + + + + + + + + + + + + + + + + This terrible construct is so that xjc can autodetect this as the + root node for processing. Many things would be better but this is + the most workable solution especially if the making of the binding + code is automated in the pom. The only other real solution is to + modify one of the classes generated by hand. + + + + + + Define the connection to the registry, security for the connection, + and the index within the registry. + + @index: the index to be used by harvest whose default is registry + + cognito: the cognito client ID, IDP and gateway for AWS based + instances of opensearch + server_url: the opensearch URL when not using AWS services + + + + + + + + + + diff --git a/src/test/java/dao/TestBulkResponseParser.java b/src/test/java/dao/TestBulkResponseParser.java index 2b91cce..d4c2444 100644 --- a/src/test/java/dao/TestBulkResponseParser.java +++ b/src/test/java/dao/TestBulkResponseParser.java @@ -1,8 +1,7 @@ package dao; import java.io.StringReader; - -import gov.nasa.pds.registry.common.es.dao.BulkResponseParser; +import gov.nasa.pds.registry.common.connection.es.BulkResponseParser; import tt.TestLogConfigurator; public class TestBulkResponseParser diff --git a/src/test/java/dao/TestDataDictionaryDao.java b/src/test/java/dao/TestDataDictionaryDao.java index e4e7597..fef177c 100644 --- a/src/test/java/dao/TestDataDictionaryDao.java +++ b/src/test/java/dao/TestDataDictionaryDao.java @@ -4,9 +4,8 @@ import java.util.List; import java.util.Set; -import org.elasticsearch.client.RestClient; - -import gov.nasa.pds.registry.common.es.client.EsClientFactory; +import gov.nasa.pds.registry.common.EstablishConnectionFactory; +import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.common.es.dao.dd.DataDictionaryDao; import gov.nasa.pds.registry.common.es.dao.dd.LddInfo; import gov.nasa.pds.registry.common.es.dao.dd.LddVersions; @@ -25,30 +24,30 @@ public static void main(String[] args) throws Exception private static void testListBooleanFields() throws Exception { - RestClient esClient = EsClientFactory.createRestClient("http://localhost:9200", null); + RestClient client = EstablishConnectionFactory.from("app:/connections/direct/localhost.xml").createRestClient(); try { - DataDictionaryDao dao = new DataDictionaryDao(esClient, "registry"); + DataDictionaryDao dao = new DataDictionaryDao(client, "registry"); Set list = dao.getFieldNamesByEsType("boolean"); - + System.out.println("Boolean fields count = " + list.size()); list.forEach((name) -> { System.out.println(name); }); } finally { - esClient.close(); + client.close(); } } private static void testListDateFields() throws Exception { - RestClient esClient = EsClientFactory.createRestClient("http://localhost:9200", null); + RestClient client = EstablishConnectionFactory.from("app:/connections/direct/localhost.xml").createRestClient(); try { - DataDictionaryDao dao = new DataDictionaryDao(esClient, "registry"); + DataDictionaryDao dao = new DataDictionaryDao(client, "registry"); Set list = dao.getFieldNamesByEsType("date"); System.out.println("Date fields count = " + list.size()); @@ -56,18 +55,18 @@ private static void testListDateFields() throws Exception } finally { - esClient.close(); + client.close(); } } private static void testListLdds() throws Exception { - RestClient esClient = EsClientFactory.createRestClient("http://localhost:9200", null); + RestClient client = EstablishConnectionFactory.from("app:/connections/direct/localhost.xml").createRestClient(); try { - DataDictionaryDao dao = new DataDictionaryDao(esClient, "registry"); + DataDictionaryDao dao = new DataDictionaryDao(client, "registry"); List list = dao.listLdds(null); Collections.sort(list); @@ -79,24 +78,24 @@ private static void testListLdds() throws Exception } finally { - esClient.close(); + client.close(); } } private static void testGetLddInfo() throws Exception { - RestClient esClient = EsClientFactory.createRestClient("http://localhost:9200", null); + RestClient client = EstablishConnectionFactory.from("app:/connections/direct/localhost.xml").createRestClient(); try { - DataDictionaryDao dao = new DataDictionaryDao(esClient, "registry"); + DataDictionaryDao dao = new DataDictionaryDao(client, "registry"); LddVersions info = dao.getLddInfo("pds"); info.debug(); } finally { - esClient.close(); + client.close(); } } diff --git a/src/test/java/dao/TestProductDao.java b/src/test/java/dao/TestProductDao.java index 7a71c93..66397c7 100644 --- a/src/test/java/dao/TestProductDao.java +++ b/src/test/java/dao/TestProductDao.java @@ -3,9 +3,8 @@ import java.util.Arrays; import java.util.List; -import org.elasticsearch.client.RestClient; - -import gov.nasa.pds.registry.common.es.client.EsClientFactory; +import gov.nasa.pds.registry.common.EstablishConnectionFactory; +import gov.nasa.pds.registry.common.RestClient; import gov.nasa.pds.registry.common.es.dao.ProductDao; import gov.nasa.pds.registry.common.util.CloseUtils; import tt.TestLogConfigurator; @@ -18,19 +17,19 @@ public static void main(String[] args) throws Exception TestLogConfigurator.configureLogger(); - RestClient esClient = null; + RestClient client = null; try { - esClient = EsClientFactory.createRestClient("localhost", null); - ProductDao dao = new ProductDao(esClient, "registry"); + client = EstablishConnectionFactory.from("localhost").createRestClient(); + ProductDao dao = new ProductDao(client, "registry"); //testUpdateArchiveStatus(dao); testGetLatestLidVids(dao); } finally { - CloseUtils.close(esClient); + CloseUtils.close(client); } } diff --git a/src/test/java/tt/TestRestClient.java b/src/test/java/tt/TestRestClient.java index 9f0d1db..dd624f8 100644 --- a/src/test/java/tt/TestRestClient.java +++ b/src/test/java/tt/TestRestClient.java @@ -1,8 +1,7 @@ package tt; -import org.elasticsearch.client.RestClient; - -import gov.nasa.pds.registry.common.es.client.EsClientFactory; +import gov.nasa.pds.registry.common.EstablishConnectionFactory; +import gov.nasa.pds.registry.common.RestClient; public class TestRestClient @@ -10,7 +9,7 @@ public class TestRestClient public static void main(String[] args) throws Exception { - RestClient client = EsClientFactory.createRestClient("localhost", null); + RestClient client = EstablishConnectionFactory.from("localhost").createRestClient(); client.close(); }