Fixes and closes #907

Additional null checks on event types processing Add the data resource UID to annotation index
gbif · Jun 6, 2023 · f74f964 · f74f964
1 parent a53fb1e
commit f74f964
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 6 deletions.
diff --git a/...atlas/pipelines/src/main/java/au/org/ala/pipelines/converters/ALAParentJsonConverter.java b/...atlas/pipelines/src/main/java/au/org/ala/pipelines/converters/ALAParentJsonConverter.java
@@ -213,13 +213,15 @@ private void mapInherited(EventJsonRecord.Builder builder) {
           eventCore.getParentsLineage().stream()
               .sorted(
                   Comparator.comparingInt(org.gbif.pipelines.io.avro.Parent::getOrder).reversed())
+              .filter(e -> e.getEventType() != null)
               .map(e -> e.getEventType())
               .collect(Collectors.toList());
 
       if (builder.getSurveyID() == null) {
         List<org.gbif.pipelines.io.avro.Parent> surveys =
             eventCore.getParentsLineage().stream()
-                .filter(e -> e.getEventType().equalsIgnoreCase("Survey"))
+                .filter(
+                    e -> e.getEventType() != null && e.getEventType().equalsIgnoreCase("Survey"))
                 .collect(Collectors.toList());
         if (!surveys.isEmpty()) {
           builder.setSurveyID(surveys.get(0).getId());

diff --git a/...ngatlas/pipelines/src/main/java/au/org/ala/pipelines/transforms/IndexRecordTransform.java b/...ngatlas/pipelines/src/main/java/au/org/ala/pipelines/transforms/IndexRecordTransform.java
@@ -1358,6 +1358,8 @@ public static SolrInputDocument convertIndexRecordToSolrDoc(
               annotation -> {
                 SolrInputDocument childDoc = new SolrInputDocument();
                 childDoc.setField(ID, indexRecord.getId() + "-" + annotation.getDoi());
+                addStringSafely(
+                    childDoc, IndexFields.DATA_RESOURCE_UID, annotation.getDatasetKey());
                 addStringSafely(childDoc, DcTerm.identifier.simpleName(), annotation.getDoi());
                 addStringSafely(
                     childDoc, DwcTerm.scientificName.simpleName(), annotation.getScientificName());

diff --git a/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/util/DwcaMetaXml.java b/livingatlas/pipelines/src/main/java/au/org/ala/pipelines/util/DwcaMetaXml.java
@@ -7,7 +7,9 @@
 import freemarker.template.DefaultObjectWrapperBuilder;
 import freemarker.template.Template;
 import freemarker.template.TemplateExceptionHandler;
+import java.io.File;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.OutputStreamWriter;
 import java.io.StringReader;
 import java.io.Writer;
@@ -16,6 +18,7 @@
 import lombok.Data;
 import lombok.Getter;
 import lombok.SneakyThrows;
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.fs.FileSystem;
 import org.gbif.pipelines.core.pojo.HdfsConfigs;
 import org.gbif.pipelines.core.utils.FsUtils;
@@ -57,17 +60,20 @@ public class DwcaMetaXml {
   @SneakyThrows
   public void write() {
     Template temp = new Template("meta", new StringReader(META), createConfig());
+    try (Writer out = getWriter()) {
+      temp.process(TemplatePojo.create(coreTerms, multimediaTerms), out);
+    }
+  }
 
-    Writer out;
+  private Writer getWriter() throws IOException {
     if (hdfsSiteConfig == null || hdfsSiteConfig.isEmpty()) {
-      out = new OutputStreamWriter(new FileOutputStream(pathToWrite));
+      FileUtils.forceMkdir(new File(pathToWrite).getParentFile());
+      return new OutputStreamWriter(new FileOutputStream(pathToWrite));
     } else {
       FileSystem fs =
           FsUtils.getFileSystem(HdfsConfigs.create(hdfsSiteConfig, coreSiteConfig), pathToWrite);
-      out = new OutputStreamWriter(ALAFsUtils.openOutputStream(fs, pathToWrite));
+      return new OutputStreamWriter(ALAFsUtils.openOutputStream(fs, pathToWrite));
     }
-
-    temp.process(TemplatePojo.create(coreTerms, multimediaTerms), out);
   }
 
   /** Create Freemarker config */