NUTCH-2835 Upgrade commons-jexl from 2 --> 3 (#558)

apache · Dec 17, 2020 · 8d8e08b · 8d8e08b
1 parent 4c7d422
commit 8d8e08b
Show file tree

Hide file tree

Showing 8 changed files with 34 additions and 39 deletions.
diff --git a/ivy/ivy.xml b/ivy/ivy.xml
@@ -46,7 +46,7 @@
 		<dependency org="org.apache.httpcomponents" name="httpclient" rev="4.5.6" conf="*->master" />
 		<dependency org="commons-codec" name="commons-codec" rev="1.11" conf="*->default" />
 		<dependency org="org.apache.commons" name="commons-compress" rev="1.18" conf="*->default" />
-		<dependency org="org.apache.commons" name="commons-jexl" rev="2.1.1" />
+		<dependency org="org.apache.commons" name="commons-jexl3" rev="3.1" conf="*->default"/>
 		<dependency org="com.tdunning" name="t-digest" rev="3.2" />
 
 		<!-- Hadoop Dependencies -->

diff --git a/src/java/org/apache/nutch/crawl/CrawlDatum.java b/src/java/org/apache/nutch/crawl/CrawlDatum.java
@@ -25,9 +25,9 @@
 import java.util.Map;
 import java.util.Map.Entry;
 
-import org.apache.commons.jexl2.JexlContext;
-import org.apache.commons.jexl2.Expression;
-import org.apache.commons.jexl2.MapContext;
+import org.apache.commons.jexl3.JexlContext;
+import org.apache.commons.jexl3.JexlExpression;
+import org.apache.commons.jexl3.MapContext;
 import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
@@ -542,7 +542,7 @@ public Object clone() {
     }
   }
 
-  public boolean evaluate(Expression expr, String url) {
+  public boolean evaluate(JexlExpression expr, String url) {
     if (expr != null && url != null) {
       // Create a context and add data
       JexlContext jcontext = new MapContext();

diff --git a/src/java/org/apache/nutch/crawl/CrawlDbReader.java b/src/java/org/apache/nutch/crawl/CrawlDbReader.java
@@ -74,7 +74,7 @@
 import org.apache.nutch.util.SegmentReaderUtil;
 import org.apache.nutch.util.StringUtil;
 import org.apache.nutch.util.TimingUtil;
-import org.apache.commons.jexl2.Expression;
+import org.apache.commons.jexl3.JexlExpression;
 
 import com.fasterxml.jackson.core.JsonGenerationException;
 import com.fasterxml.jackson.core.JsonGenerator;
@@ -864,7 +864,7 @@ public static class CrawlDbDumpMapper
     Matcher matcher = null;
     String status = null;
     Integer retry = null;
-    Expression expr = null;
+    JexlExpression expr = null;
     float sample;
 
     @Override

diff --git a/src/java/org/apache/nutch/crawl/Generator.java b/src/java/org/apache/nutch/crawl/Generator.java
@@ -34,9 +34,9 @@
 import org.apache.hadoop.conf.Configurable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.apache.commons.jexl2.Expression;
-import org.apache.commons.jexl2.JexlContext;
-import org.apache.commons.jexl2.MapContext;
+import org.apache.commons.jexl3.JexlExpression;
+import org.apache.commons.jexl3.JexlContext;
+import org.apache.commons.jexl3.MapContext;
 import org.apache.hadoop.mapreduce.Counter;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
@@ -182,7 +182,7 @@ public static class SelectorMapper
     private float scoreThreshold = 0f;
     private int intervalThreshold = -1;
     private byte restrictStatus = -1;
-    private Expression expr = null;
+    private JexlExpression expr = null;
 
     @Override
     public void setup(
@@ -306,8 +306,8 @@ public static class SelectorReducer extends
     private URLNormalizers normalizers;
     private static boolean normalise;
     private SequenceFile.Reader[] hostdbReaders = null;
-    private Expression maxCountExpr = null;
-    private Expression fetchDelayExpr = null;
+    private JexlExpression maxCountExpr = null;
+    private JexlExpression fetchDelayExpr = null;
 
     public void open() {
       if (conf.get(GENERATOR_HOSTDB) != null) {

diff --git a/src/java/org/apache/nutch/hostdb/ReadHostDb.java b/src/java/org/apache/nutch/hostdb/ReadHostDb.java
@@ -44,10 +44,11 @@
 import org.apache.nutch.util.TimingUtil;
 import org.apache.nutch.util.SegmentReaderUtil;
 
-import org.apache.commons.jexl2.JexlContext;
-import org.apache.commons.jexl2.Expression;
-import org.apache.commons.jexl2.JexlEngine;
-import org.apache.commons.jexl2.MapContext;
+import org.apache.commons.jexl3.JexlBuilder;
+import org.apache.commons.jexl3.JexlContext;
+import org.apache.commons.jexl3.JexlExpression;
+import org.apache.commons.jexl3.JexlEngine;
+import org.apache.commons.jexl3.MapContext;
 
 /**
  * @see <a href='https://commons.apache.org/proper/commons-jexl/reference/syntax.html'>Commons</a>
@@ -67,7 +68,7 @@ static class ReadHostDbMapper extends Mapper<Text, HostDatum, Text, Text> {
     protected boolean dumpHomepages = false;
     protected boolean fieldHeader = true;
     protected Text emptyText = new Text();
-    protected Expression expr = null;
+    protected JexlExpression expr = null;
 
     @Override
     public void setup(Context context) {
@@ -77,11 +78,7 @@ public void setup(Context context) {
       String expr = context.getConfiguration().get(HOSTDB_FILTER_EXPRESSION);
       if (expr != null) {
         // Create or retrieve a JexlEngine
-        JexlEngine jexl = new JexlEngine();
-
-        // Dont't be silent and be strict
-        jexl.setSilent(true);
-        jexl.setStrict(true);
+        JexlEngine jexl = new JexlBuilder().silent(true).strict(true).create();
 
         // Create an expression object
         this.expr = jexl.createExpression(expr);

diff --git a/src/java/org/apache/nutch/util/JexlUtil.java b/src/java/org/apache/nutch/util/JexlUtil.java
@@ -21,8 +21,9 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import org.apache.commons.jexl2.Expression;
-import org.apache.commons.jexl2.JexlEngine;
+import org.apache.commons.jexl3.JexlBuilder;
+import org.apache.commons.jexl3.JexlEngine;
+import org.apache.commons.jexl3.JexlExpression;
 import org.apache.commons.lang.time.DateUtils;
 
 import org.slf4j.Logger;
@@ -46,7 +47,7 @@ public class JexlUtil {
    * @param expr string JEXL expression
    * @return parsed JEXL expression or null in case of parse error
    */
-  public static Expression parseExpression(String expr) {
+  public static JexlExpression parseExpression(String expr) {
     if (expr == null) return null;
 
     try {
@@ -65,10 +66,7 @@ public static Expression parseExpression(String expr) {
         expr = expr.replace(date, Long.toString(time));
       }
 
-      JexlEngine jexl = new JexlEngine();
-
-      jexl.setSilent(true);
-      jexl.setStrict(true);
+      JexlEngine jexl = new JexlBuilder().silent(true).strict(true).create();
 
       return jexl.createExpression(expr);
     } catch (Exception e) {

diff --git a/src/plugin/exchange-jexl/src/java/org/apache/nutch/exchange/jexl/JexlExchange.java b/src/plugin/exchange-jexl/src/java/org/apache/nutch/exchange/jexl/JexlExchange.java
@@ -16,9 +16,9 @@
  */
 package org.apache.nutch.exchange.jexl;
 
-import org.apache.commons.jexl2.Expression;
-import org.apache.commons.jexl2.JexlContext;
-import org.apache.commons.jexl2.MapContext;
+import org.apache.commons.jexl3.JexlExpression;
+import org.apache.commons.jexl3.JexlContext;
+import org.apache.commons.jexl3.MapContext;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.nutch.exchange.Exchange;
 import org.apache.nutch.indexer.NutchDocument;
@@ -32,7 +32,7 @@ public class JexlExchange implements Exchange {
 
   private Configuration conf;
 
-  private Expression expression;
+  private JexlExpression expression;
 
   /**
    * Initializes the internal variables.

diff --git a/src/plugin/index-jexl-filter/src/java/org/apache/nutch/indexer/jexl/JexlIndexingFilter.java b/src/plugin/index-jexl-filter/src/java/org/apache/nutch/indexer/jexl/JexlIndexingFilter.java
@@ -20,9 +20,9 @@
 import java.util.List;
 import java.util.Map.Entry;
 
-import org.apache.commons.jexl2.Expression;
-import org.apache.commons.jexl2.JexlContext;
-import org.apache.commons.jexl2.MapContext;
+import org.apache.commons.jexl3.JexlExpression;
+import org.apache.commons.jexl3.JexlContext;
+import org.apache.commons.jexl3.MapContext;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.Text;
 import org.apache.nutch.crawl.CrawlDatum;
@@ -48,7 +48,7 @@ public class JexlIndexingFilter implements IndexingFilter {
       .getLogger(MethodHandles.lookup().lookupClass());
 
   private Configuration conf;
-  private Expression expr;
+  private JexlExpression expr;
 
   @Override
   public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
@@ -96,7 +96,7 @@ public NutchDocument filter(NutchDocument doc, Parse parse, Text url,
         return doc;
       }
     } catch (Exception e) {
-      LOG.warn("Failed evaluating JEXL {}", expr.getExpression(), e);
+      LOG.warn("Failed evaluating JEXL {}", expr.getSourceText(), e);
     }
     return null;
   }