Improve Provenance module

InseeFr · Nov 12, 2024 · 8261bc3 · 8261bc3
2 parents f676449 + fa7630a
commit 8261bc3
Show file tree

Hide file tree

Showing 25 changed files with 476 additions and 214 deletions.
diff --git a/coverage/pom.xml b/coverage/pom.xml
@@ -6,7 +6,7 @@
     <parent>
         <groupId>fr.insee.trevas</groupId>
         <artifactId>trevas-parent</artifactId>
-        <version>1.7.0</version>
+        <version>1.8.0</version>
     </parent>
 
     <artifactId>coverage</artifactId>
@@ -22,32 +22,32 @@
         <dependency>
             <groupId>fr.insee.trevas</groupId>
             <artifactId>vtl-engine</artifactId>
-            <version>1.7.0</version>
+            <version>1.8.0</version>
         </dependency>
         <dependency>
             <groupId>fr.insee.trevas</groupId>
             <artifactId>vtl-jackson</artifactId>
-            <version>1.7.0</version>
+            <version>1.8.0</version>
         </dependency>
         <dependency>
             <groupId>fr.insee.trevas</groupId>
             <artifactId>vtl-jdbc</artifactId>
-            <version>1.7.0</version>
+            <version>1.8.0</version>
         </dependency>
         <dependency>
             <groupId>fr.insee.trevas</groupId>
             <artifactId>vtl-model</artifactId>
-            <version>1.7.0</version>
+            <version>1.8.0</version>
         </dependency>
         <dependency>
             <groupId>fr.insee.trevas</groupId>
             <artifactId>vtl-parser</artifactId>
-            <version>1.7.0</version>
+            <version>1.8.0</version>
         </dependency>
         <dependency>
             <groupId>fr.insee.trevas</groupId>
             <artifactId>vtl-spark</artifactId>
-            <version>1.7.0</version>
+            <version>1.8.0</version>
         </dependency>
     </dependencies>
     <build>

diff --git a/docs/yarn.lock b/docs/yarn.lock
@@ -4711,9 +4711,9 @@ elkjs@^0.9.0:
   integrity sha512-f/ZeWvW/BCXbhGEf1Ujp29EASo/lk1FDnETgNKwJrsVvGZhUWCZyg3xLJjAsxfOmt8KjswHmI5EwCQcPMpOYhQ==
 
 elliptic@^6.5.3, elliptic@^6.5.5:
-  version "6.5.7"
-  resolved "https://registry.yarnpkg.com/elliptic/-/elliptic-6.5.7.tgz#8ec4da2cb2939926a1b9a73619d768207e647c8b"
-  integrity sha512-ESVCtTwiA+XhY3wyh24QqRGBoP3rEdDUl3EDUUo9tft074fi19IrdpH7hLCMMP3CIj7jb3W96rn8lt/BqIlt5Q==
+  version "6.6.0"
+  resolved "https://registry.yarnpkg.com/elliptic/-/elliptic-6.6.0.tgz#5919ec723286c1edf28685aa89261d4761afa210"
+  integrity sha512-dpwoQcLc/2WLQvJvLRHKZ+f9FgOdjnq11rurqwekGQygGPsYSK29OMMD2WalatiqQ+XGFDglTNixpPfI+lpaAA==
   dependencies:
     bn.js "^4.11.9"
     brorand "^1.1.0"
@@ -6164,9 +6164,9 @@ http-parser-js@>=0.5.1:
   integrity sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q==
 
 http-proxy-middleware@^2.0.3:
-  version "2.0.6"
-  resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.6.tgz#e1a4dd6979572c7ab5a4e4b55095d1f32a74963f"
-  integrity sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==
+  version "2.0.7"
+  resolved "https://registry.yarnpkg.com/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz#915f236d92ae98ef48278a95dedf17e991936ec6"
+  integrity sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==
   dependencies:
     "@types/http-proxy" "^1.17.8"
     http-proxy "^1.18.1"
@@ -7321,9 +7321,9 @@ merge2@^1.3.0, merge2@^1.4.1:
   integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==
 
 mermaid@^10.4.0:
-  version "10.9.2"
-  resolved "https://registry.yarnpkg.com/mermaid/-/mermaid-10.9.2.tgz#108fe98060e6fba6bc826e5b454674aa2d32b817"
-  integrity sha512-UkZyMSuIYcI1Q0H+2pv/5CiY84sOwQ2XlKoDZMl9Y/MtrLEtxQtyA6LWGkMxnZxj0dJqI+7nw51bYjNnrbdFsQ==
+  version "10.9.3"
+  resolved "https://registry.yarnpkg.com/mermaid/-/mermaid-10.9.3.tgz#90bc6f15c33dbe5d9507fed31592cc0d88fee9f7"
+  integrity sha512-V80X1isSEvAewIL3xhmz/rVmc27CVljcsbWxkxlWJWY/1kQa4XOABqpDl2qQLGKzpKm6WbTfUEKImBlUfFYArw==
   dependencies:
     "@braintree/sanitize-url" "^6.0.1"
     "@types/d3-scale" "^4.0.3"

diff --git a/pom.xml b/pom.xml
@@ -5,7 +5,7 @@
     <groupId>fr.insee.trevas</groupId>
     <artifactId>trevas-parent</artifactId>
     <packaging>pom</packaging>
-    <version>1.7.0</version>
+    <version>1.8.0</version>
     <modules>
         <module>vtl-parser</module>
         <module>vtl-model</module>

diff --git a/vtl-csv/pom.xml b/vtl-csv/pom.xml
@@ -7,13 +7,13 @@
     <parent>
         <groupId>fr.insee.trevas</groupId>
         <artifactId>trevas-parent</artifactId>
-        <version>1.7.0</version>
+        <version>1.8.0</version>
     </parent>
 
     <artifactId>vtl-csv</artifactId>
     <name>VTL CSV</name>
     <description>CSV module for the VTL model</description>
-    <version>1.7.0</version>
+    <version>1.8.0</version>
 
     <properties>
         <sonar.coverage.jacoco.xmlReportPaths>
@@ -25,7 +25,7 @@
         <dependency>
             <groupId>fr.insee.trevas</groupId>
             <artifactId>vtl-model</artifactId>
-            <version>1.7.0</version>
+            <version>1.8.0</version>
         </dependency>
         <dependency>
             <groupId>net.sf.supercsv</groupId>

diff --git a/vtl-engine/pom.xml b/vtl-engine/pom.xml
@@ -7,13 +7,13 @@
     <parent>
         <groupId>fr.insee.trevas</groupId>
         <artifactId>trevas-parent</artifactId>
-        <version>1.7.0</version>
+        <version>1.8.0</version>
     </parent>
 
     <artifactId>vtl-engine</artifactId>
     <name>VTL Engine</name>
     <description>VTL engine framework for Trevas</description>
-    <version>1.7.0</version>
+    <version>1.8.0</version>
 
     <properties>
         <sonar.coverage.jacoco.xmlReportPaths>
@@ -25,13 +25,13 @@
         <dependency>
             <groupId>fr.insee.trevas</groupId>
             <artifactId>vtl-parser</artifactId>
-            <version>1.7.0</version>
+            <version>1.8.0</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>fr.insee.trevas</groupId>
             <artifactId>vtl-model</artifactId>
-            <version>1.7.0</version>
+            <version>1.8.0</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlNativeMethods.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/VtlNativeMethods.java
@@ -47,10 +47,6 @@ public class VtlNativeMethods {
             Fun.<Boolean, Double, Double>toMethod(ConditionalVisitor::ifThenElse),
             Fun.<Boolean, String, String>toMethod(ConditionalVisitor::ifThenElse),
             Fun.<Boolean, Boolean, Boolean>toMethod(ConditionalVisitor::ifThenElse),
-            Fun.<Boolean, Long>toMethod(ConditionalVisitor::caseFn),
-            Fun.<Boolean, Double>toMethod(ConditionalVisitor::caseFn),
-            Fun.<Boolean, String>toMethod(ConditionalVisitor::caseFn),
-            Fun.<Boolean, Boolean>toMethod(ConditionalVisitor::caseFn),
             Fun.<Long, Long>toMethod(ConditionalVisitor::nvl),
             Fun.<Double, Double>toMethod(ConditionalVisitor::nvl),
             Fun.<Double, Long>toMethod(ConditionalVisitor::nvl),

diff --git a/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ConditionalVisitor.java b/vtl-engine/src/main/java/fr/insee/vtl/engine/visitors/expression/ConditionalVisitor.java
@@ -10,14 +10,10 @@
 import fr.insee.vtl.parser.VtlBaseVisitor;
 import fr.insee.vtl.parser.VtlParser;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
+import java.util.*;
 import java.util.stream.Collectors;
 
 import static fr.insee.vtl.engine.VtlScriptEngine.fromContext;
-import static fr.insee.vtl.engine.utils.TypeChecking.assertBoolean;
 import static fr.insee.vtl.engine.utils.TypeChecking.hasSameTypeOrNull;
 
 /**
@@ -68,34 +64,6 @@ public static Boolean ifThenElse(Boolean condition, Boolean thenExpr, Boolean el
         return condition ? thenExpr : elseExpr;
     }
 
-    public static Long caseFn(Boolean condition, Long thenExpr) {
-        if (condition == null) {
-            return null;
-        }
-        return condition ? thenExpr : null;
-    }
-
-    public static Double caseFn(Boolean condition, Double thenExpr) {
-        if (condition == null) {
-            return null;
-        }
-        return condition ? thenExpr : null;
-    }
-
-    public static String caseFn(Boolean condition, String thenExpr) {
-        if (condition == null) {
-            return null;
-        }
-        return condition ? thenExpr : null;
-    }
-
-    public static Boolean caseFn(Boolean condition, Boolean thenExpr) {
-        if (condition == null) {
-            return null;
-        }
-        return condition ? thenExpr : null;
-    }
-
     public static Long nvl(Long value, Long defaultValue) {
         return value == null ? defaultValue : value;
     }
@@ -149,92 +117,59 @@ public ResolvableExpression visitIfExpr(VtlParser.IfExprContext ctx) {
      */
     @Override
     public ResolvableExpression visitCaseExpr(VtlParser.CaseExprContext ctx) {
-        Positioned pos = fromContext(ctx);
-        List<VtlParser.ExprContext> exprs = ctx.expr();
-        List<VtlParser.ExprContext> whenExprs = new ArrayList<>();
-        List<VtlParser.ExprContext> thenExprs = new ArrayList<>();
-        for (int i = 0; i < exprs.size() - 1; i = i + 2) {
-            whenExprs.add(exprs.get(i));
-            thenExprs.add(exprs.get(i + 1));
-        }
-        List<ResolvableExpression> whenExpressions = whenExprs.stream()
-                .map(e -> assertBoolean(exprVisitor.visit(e), e))
-                .collect(Collectors.toList());
-        List<ResolvableExpression> thenExpressions = thenExprs.stream()
-                .map(exprVisitor::visit)
-                .collect(Collectors.toList());
-        ResolvableExpression elseExpression = exprVisitor.visit(exprs.get(exprs.size() - 1));
-        List<ResolvableExpression> forTypeCheck = (new ArrayList<>(thenExpressions));
-        forTypeCheck.add(elseExpression);
-        // TODO: handle better the default element position
-        if (!hasSameTypeOrNull(forTypeCheck)) {
-            try {
-                throw new InvalidTypeException(
-                        forTypeCheck.get(0).getClass(),
-                        Boolean.class,
-                        fromContext(ctx.expr(0))
-                );
-            } catch (InvalidTypeException e) {
-                throw new RuntimeException(e);
+        try {
+            Positioned pos = fromContext(ctx);
+            List<VtlParser.ExprContext> exprs = ctx.expr();
+            List<VtlParser.ExprContext> whenExprs = new ArrayList<>();
+            List<VtlParser.ExprContext> thenExprs = new ArrayList<>();
+            for (int i = 0; i < exprs.size() - 1; i = i + 2) {
+                whenExprs.add(exprs.get(i));
+                thenExprs.add(exprs.get(i + 1));
+            }
+            List<ResolvableExpression> whenExpressions = whenExprs.stream()
+                    .map(exprVisitor::visit)
+                    .collect(Collectors.toList());
+            List<ResolvableExpression> thenExpressions = thenExprs.stream()
+                    .map(exprVisitor::visit)
+                    .collect(Collectors.toList());
+            ResolvableExpression elseExpression = exprVisitor.visit(exprs.get(exprs.size() - 1));
+            List<ResolvableExpression> forTypeCheck = (new ArrayList<>(thenExpressions));
+            forTypeCheck.add(elseExpression);
+            // TODO: handle better the default element position
+            if (!hasSameTypeOrNull(forTypeCheck)) {
+                try {
+                    throw new InvalidTypeException(
+                            forTypeCheck.get(0).getClass(),
+                            Boolean.class,
+                            fromContext(ctx.expr(0))
+                    );
+                } catch (InvalidTypeException e) {
+                    throw new RuntimeException(e);
+                }
             }
-        }
-
-        Class<?> outputType = elseExpression.getType();
 
-        if (outputType.equals(String.class)) {
-            return ResolvableExpression.withType(String.class)
-                    .withPosition(pos)
-                    .using(context -> {
-                        for (int i = 0; i < whenExprs.size(); i++) {
-                            Boolean condition = (Boolean) whenExpressions.get(i).resolve(context);
-                            if (condition) {
-                                return (String) (new CastExpression(pos, thenExpressions.get(i), outputType)).resolve(context);
-                            }
-                        }
-                        return (String) (new CastExpression(pos, elseExpression, outputType)).resolve(context);
-                    });
-        }
-        if (outputType.equals(Double.class)) {
-            return ResolvableExpression.withType(Double.class)
-                    .withPosition(pos)
-                    .using(context -> {
-                        for (int i = 0; i < whenExprs.size(); i++) {
-                            Boolean condition = (Boolean) whenExpressions.get(i).resolve(context);
-                            if (condition) {
-                                return (Double) (new CastExpression(pos, thenExpressions.get(i), outputType)).resolve(context);
-                            }
-                        }
-                        return (Double) (new CastExpression(pos, elseExpression, outputType)).resolve(context);
-                    });
+            Class<?> outputType = elseExpression.getType();
+            return new CastExpression(pos, caseToIfIt(whenExpressions.listIterator(), thenExpressions.listIterator(), elseExpression), outputType);
+        } catch (VtlScriptException e) {
+            throw new VtlRuntimeException(e);
         }
-        if (outputType.equals(Long.class)) {
-            return ResolvableExpression.withType(Long.class)
-                    .withPosition(pos)
-                    .using(context -> {
-                        for (int i = 0; i < whenExprs.size(); i++) {
-                            Boolean condition = (Boolean) whenExpressions.get(i).resolve(context);
-                            if (condition) {
-                                return (Long) (new CastExpression(pos, thenExpressions.get(i), outputType)).resolve(context);
-                            }
-                        }
-                        return (Long) (new CastExpression(pos, elseExpression, outputType)).resolve(context);
-                    });
+    }
+
+    private ResolvableExpression caseToIfIt(ListIterator<ResolvableExpression> whenExpr, ListIterator<ResolvableExpression> thenExpr, ResolvableExpression elseExpression) throws VtlScriptException {
+        if (!whenExpr.hasNext() || !thenExpr.hasNext()) {
+            return elseExpression;
         }
-        if (outputType.equals(Boolean.class)) {
-            return ResolvableExpression.withType(Boolean.class)
-                    .withPosition(pos)
-                    .using(context -> {
-                        for (int i = 0; i < whenExprs.size(); i++) {
-                            Boolean condition = (Boolean) whenExpressions.get(i).resolve(context);
-                            if (condition) {
-                                return (Boolean) (new CastExpression(pos, thenExpressions.get(i), outputType)).resolve(context);
-                            }
-                        }
-                        return (Boolean) (new CastExpression(pos, elseExpression, outputType)).resolve(context);
-                    });
-        } else return null;
+
+        ResolvableExpression nextWhen = whenExpr.next();
+
+        return genericFunctionsVisitor.invokeFunction("ifThenElse", Java8Helpers.listOf(
+                nextWhen,
+                thenExpr.next(),
+                caseToIfIt(whenExpr, thenExpr, elseExpression)
+        ), nextWhen);
     }
 
+
     /**
      * Visits nvl expressions.
      *

diff --git a/.../main/java/fr/insee/vtl/engine/visitors/expression/functions/GenericFunctionsVisitor.java b/.../main/java/fr/insee/vtl/engine/visitors/expression/functions/GenericFunctionsVisitor.java
@@ -7,10 +7,10 @@
 import fr.insee.vtl.engine.expressions.CastExpression;
 import fr.insee.vtl.engine.expressions.ComponentExpression;
 import fr.insee.vtl.engine.expressions.FunctionExpression;
-import fr.insee.vtl.model.utils.Java8Helpers;
 import fr.insee.vtl.engine.visitors.expression.ExpressionVisitor;
 import fr.insee.vtl.model.*;
 import fr.insee.vtl.model.exceptions.VtlScriptException;
+import fr.insee.vtl.model.utils.Java8Helpers;
 import fr.insee.vtl.parser.VtlBaseVisitor;
 import fr.insee.vtl.parser.VtlParser;
 import org.antlr.v4.runtime.Token;
@@ -169,7 +169,8 @@ private DatasetExpression invokeFunctionOnDataset(String funcName, List<Resolvab
                 .collect(Collectors.toMap(e -> "arg" + e.hashCode(), e -> e));
         if (measureNames.size() != 1) {
             throw new VtlRuntimeException(
-                    new InvalidArgumentException("mono-measure datasets don't contain same measures (number or names)", position)
+                    new InvalidArgumentException("Variables in the mono-measure datasets are not named the same: " +
+                            measureNames + " found", position)
             );
         }
         DatasetExpression ds = proc.executeInnerJoin(dsExprs);