From 1fed102d0f1a9786c5536c9eb5beea0689989da1 Mon Sep 17 00:00:00 2001
From: Jens Wille <jens.wille@hbz-nrw.de>
Date: Wed, 9 Oct 2024 14:43:55 +0200
Subject: [PATCH] Optionally specify limit for number of entities in a record.

This is a brute-force approach to dealing with OOM situations when Alma records have an excessive number of items (e.g. 99374518570506441: >12000 entities = ~10 GB heap for the Record instance).

Use Metafix instance setter `setMaxEntityCount(int)` or set system property `org.metafacture.metafix.maxEntityCount=<int>`.

Alternative options:

- Increase maximum heap size for JVM.
- Significantly reduce memory requirement for Record instances.
---
 metafix-runner/build.gradle                   | 10 +++++++
 .../java/org/metafacture/metafix/Metafix.java | 29 ++++++++++++++++++-
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/metafix-runner/build.gradle b/metafix-runner/build.gradle
index fe2f8d9b..9ebdd3e7 100644
--- a/metafix-runner/build.gradle
+++ b/metafix-runner/build.gradle
@@ -49,3 +49,13 @@ application {
     ]
   }
 }
+
+tasks.withType(JavaExec) {
+  doFirst {
+    def prefix = project.group + '.'
+
+    System.properties.each { k, v ->
+      if (k.startsWith(prefix)) systemProperties[k] = v
+    }
+  }
+}
diff --git a/metafix/src/main/java/org/metafacture/metafix/Metafix.java b/metafix/src/main/java/org/metafacture/metafix/Metafix.java
index 7d4bd12e..7b306041 100644
--- a/metafix/src/main/java/org/metafacture/metafix/Metafix.java
+++ b/metafix/src/main/java/org/metafacture/metafix/Metafix.java
@@ -101,6 +101,7 @@ public class Metafix implements StreamPipe<StreamReceiver>, Maps {
     private boolean repeatedFieldsToEntities;
     private boolean strictnessHandlesProcessExceptions;
     private int entityCount;
+    private int maxEntityCount = Integer.getInteger("org.metafacture.metafix.maxEntityCount", -1);
 
     public Metafix() {
         this(NO_VARS);
@@ -313,22 +314,36 @@ public void startEntity(final String name) {
             throw new IllegalArgumentException("Entity name must not be null.");
         }
 
+        ++entityCount;
+        if (maxEntityCountExceeded()) {
+            LOG.debug("Maximum number of entities exceeded: {}/{}", entityCount, maxEntityCount);
+            return;
+        }
+
         final Value value = isArrayName(name) ? Value.newArray() : Value.newHash();
         addValue(name, value);
         entities.add(value);
 
-        entityCountStack.push(++entityCount);
+        entityCountStack.push(entityCount);
         flattener.startEntity(name);
     }
 
     @Override
     public void endEntity() {
+        if (maxEntityCountExceeded()) {
+            return;
+        }
+
         entityCountStack.pop();
         flattener.endEntity();
     }
 
     @Override
     public void literal(final String name, final String value) {
+        if (entityCountStack.size() > 1 && maxEntityCountExceeded()) {
+            return;
+        }
+
         LOG.debug("Putting '{}': '{}'", name, value);
         flattener.literal(name, value);
     }
@@ -438,6 +453,18 @@ public String getEntityMemberName() {
         return entityMemberName;
     }
 
+    public void setMaxEntityCount(final int maxEntityCount) {
+        this.maxEntityCount = maxEntityCount;
+    }
+
+    public int getMaxEntityCount() {
+        return maxEntityCount;
+    }
+
+    private boolean maxEntityCountExceeded() {
+        return maxEntityCount >= 0 && entityCount > maxEntityCount;
+    }
+
     public enum Strictness {
 
         /**