Merge pull request #1865 from Stdri/issue/1863_mysql_sink_quote_fields

Issue/1863 mysql sink quote fields
telefonicaid · Sep 17, 2020 · 07b54a5 · 07b54a5
2 parents 7cd18f7 + bc592b0
commit 07b54a5
Show file tree

Hide file tree

Showing 5 changed files with 126 additions and 56 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
+.metadata
 target/
 # IntelliJ local files
 *.idea

diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE
@@ -1,3 +1,3 @@
-[cygnus-common][SQLBackendImpl] Add configuration option to persist errors in SQL sinks (#1928) 
+[cygnus-common][SQLBackendImpl] Add configuration option to persist errors in SQL sinks (#1928)
 [cygnus-ngsi-ld] Creation of the new PostGIS sink for persisting NGSI-LD notifications (#1905)
-
+[cygnus-ngsi][MysqlSink] Quoted Sql field names to avoid naming problems. (#1863)
diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSink.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSink.java
@@ -48,6 +48,7 @@
  */
 public class NGSIMySQLSink extends NGSISink {
 
+    private static final String MYSQL_QUOTE_CHAR = "`";
     private static final String DEFAULT_ROW_ATTR_PERSISTENCE = "row";
     private static final String DEFAULT_PASSWORD = "";
     private static final String DEFAULT_PORT = "3306";
@@ -364,7 +365,7 @@ protected NGSIGenericAggregator getAggregator(boolean rowAttrPersistence) {
     private void persistAggregation(NGSIGenericAggregator aggregator)
         throws CygnusPersistenceError, CygnusRuntimeError, CygnusBadContextData {
         String fieldsForCreate = NGSIUtils.getFieldsForCreate(aggregator.getAggregationToPersist());
-        String fieldsForInsert = NGSIUtils.getFieldsForInsert(aggregator.getAggregationToPersist());
+        String fieldsForInsert = NGSIUtils.getFieldsForInsert(aggregator.getAggregationToPersist(), MYSQL_QUOTE_CHAR);
         String valuesForInsert = NGSIUtils.getValuesForInsert(aggregator.getAggregationToPersist(), aggregator.isAttrNativeTypes());
         String dbName = aggregator.getDbName(enableLowercase);
         String tableName = aggregator.getTableName(enableLowercase);

diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java
@@ -22,7 +22,6 @@
 import com.google.gson.JsonObject;
 import com.telefonica.iot.cygnus.log.CygnusLogger;
 
-import java.lang.reflect.Array;
 import java.util.*;
 import java.util.regex.Pattern;
 import org.apache.commons.lang3.tuple.ImmutablePair;
@@ -31,8 +30,6 @@
 import org.json.simple.parser.JSONParser;
 import org.json.simple.parser.ParseException;
 
-import static java.io.File.separator;
-
 /**
  *
  * @author frb
@@ -271,15 +268,26 @@ public static String getFieldsForCreate(LinkedHashMap<String, ArrayList<JsonElem
      * @return the fields (column names) for insert in SQL format.
      */
     public static String getFieldsForInsert(LinkedHashMap<String, ArrayList<JsonElement>> aggregation) {
+        return getFieldsForInsert(aggregation, "");
+    }
+
+    /**
+     * Gets fields for insert.
+     * 
+     * @param aggregation the aggregation
+     * @param quoteChar char to quote field names
+     * @return he fields (column names) for insert in SQL format.
+     */
+    public static String getFieldsForInsert(LinkedHashMap<String, ArrayList<JsonElement>> aggregation, String quoteChar) {
         String fieldsForInsert = "(";
         boolean first = true;
         Iterator<String> it = aggregation.keySet().iterator();
         while (it.hasNext()) {
             if (first) {
-                fieldsForInsert += (String) it.next();
+                fieldsForInsert += quoteChar + (String) it.next() + quoteChar;
                 first = false;
             } else {
-                fieldsForInsert += "," + (String) it.next();
+                fieldsForInsert += "," + quoteChar + (String) it.next() + quoteChar;
             } // if else
         } // while
         return fieldsForInsert + ")";
@@ -398,7 +406,7 @@ public static ArrayList<String> attributeNames(LinkedHashMap<String, ArrayList<J
         attributeNames.remove(NGSIConstants.FIWARE_SERVICE_PATH);
         attributeNames.remove(NGSIConstants.ENTITY_ID);
         attributeNames.remove(NGSIConstants.ENTITY_TYPE);
-        ArrayList <String> cropedList = (ArrayList<String>) attributeNames.clone();
+        ArrayList<String> cropedList = (ArrayList<String>) attributeNames.clone();
         for (String key : cropedList) {
             if (key.contains("_md") || key.contains(NGSIConstants.AUTOGENERATED_ATTR_TYPE)) {
                 attributeNames.remove(key);

diff --git a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/utils/NGSIUtilsTest.java b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/utils/NGSIUtilsTest.java
@@ -18,30 +18,36 @@
 package com.telefonica.iot.cygnus.utils;
 
 import static com.telefonica.iot.cygnus.utils.CommonUtilsForTests.getTestTraceHead;
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.log4j.Level;
 import org.apache.log4j.LogManager;
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
-import static org.junit.Assert.assertEquals;
 import org.junit.Test;
 
+import com.google.gson.JsonElement;
+
 /**
  *
  * @author frb
  */
 public class NGSIUtilsTest {
-    
+
     /**
      * Constructor.
      */
     public NGSIUtilsTest() {
         LogManager.getRootLogger().setLevel(Level.FATAL);
     } // NGSIUtilsTest
-    
+
     /**
-     * [NGSIUtils.getGeometry] -------- When getting a geometry, a CartoDB point is obtained when passing
-     * an attribute of type 'geo:point'.
+     * [NGSIUtils.getGeometry] -------- When getting a geometry, a CartoDB point
+     * is obtained when passing an attribute of type 'geo:point'.
      */
     @Test
     public void testGetGeometryGeopoint() {
@@ -52,25 +58,24 @@ public void testGetGeometryGeopoint() {
         String attrValue = "-3.7167, 40.3833";
         String attrType = "geo:point";
         boolean swapCoordinates = false; // irrelevant for this test
-        ImmutablePair<String, Boolean> geometry = NGSIUtils.getGeometry(
-                attrValue, attrType, attrMetadataStr, swapCoordinates);
+        ImmutablePair<String, Boolean> geometry = NGSIUtils.getGeometry(attrValue, attrType, attrMetadataStr,
+                swapCoordinates);
 
         try {
-            assertEquals("ST_SetSRID(ST_MakePoint(-3.7167::double precision , 40.3833::double precision ), 4326)", geometry.getLeft());
-            System.out.println(getTestTraceHead("[Utils.getLocation]")
-                    + "-  OK  - Geometry '" + geometry.getLeft() + "' obtained for an attribute of type '" + attrType
-                    + "' and value '" + attrValue + "'");
+            assertEquals("ST_SetSRID(ST_MakePoint(-3.7167::double precision , 40.3833::double precision ), 4326)",
+                    geometry.getLeft());
+            System.out.println(getTestTraceHead("[Utils.getLocation]") + "-  OK  - Geometry '" + geometry.getLeft()
+                    + "' obtained for an attribute of type '" + attrType + "' and value '" + attrValue + "'");
         } catch (AssertionError e) {
-            System.out.println(getTestTraceHead("[Utils.getLocation]")
-                    + "- FAIL - Geometry '" + geometry.getLeft() + "' obtained for an attribute of type '" + attrType
-                    + "' and value '" + attrValue + "'");
+            System.out.println(getTestTraceHead("[Utils.getLocation]") + "- FAIL - Geometry '" + geometry.getLeft()
+                    + "' obtained for an attribute of type '" + attrType + "' and value '" + attrValue + "'");
             throw e;
         } // try catch // try catch
     } // testGetGeometryGeopoint
-    
+
     /**
-     * [NGSIUtils.getGeometry] -------- When getting a geometry, a CartoDB point is obtained when passing
-     * an attribute with 'geometry' metadata.
+     * [NGSIUtils.getGeometry] -------- When getting a geometry, a CartoDB point
+     * is obtained when passing an attribute with 'geometry' metadata.
      */
     @Test
     public void testGetGeometryMetadata() {
@@ -87,25 +92,27 @@ public void testGetGeometryMetadata() {
         String attrValue = "-3.7167, 40.3833";
         String attrType = "coordinates"; // irrelevant for this test
         boolean swapCoordinates = false; // irrelevant for this test
-        ImmutablePair<String, Boolean> geometry = NGSIUtils.getGeometry(
-                attrValue, attrType, attrMetadataStr, swapCoordinates);
+        ImmutablePair<String, Boolean> geometry = NGSIUtils.getGeometry(attrValue, attrType, attrMetadataStr,
+                swapCoordinates);
 
         try {
-            assertEquals("ST_SetSRID(ST_MakePoint(-3.7167::double precision , 40.3833::double precision ), 4326)", geometry.getLeft());
-            System.out.println(getTestTraceHead("[Utils.getLocation]")
-                    + "-  OK  - Geometry '" + geometry.getLeft() + "' obtained for an attribute with metadata '"
-                    + attrMetadataStr + "' and value '" + attrValue + "'");
+            assertEquals("ST_SetSRID(ST_MakePoint(-3.7167::double precision , 40.3833::double precision ), 4326)",
+                    geometry.getLeft());
+            System.out.println(getTestTraceHead("[Utils.getLocation]") + "-  OK  - Geometry '" + geometry.getLeft()
+                    + "' obtained for an attribute with metadata '" + attrMetadataStr + "' and value '" + attrValue
+                    + "'");
         } catch (AssertionError e) {
-            System.out.println(getTestTraceHead("[Utils.getLocation]")
-                    + "- FAIL - Geometry '" + geometry.getLeft() + "' obtained for an attribute with metadata '"
-                    + attrMetadataStr + "' and value '" + attrValue + "'");
+            System.out.println(getTestTraceHead("[Utils.getLocation]") + "- FAIL - Geometry '" + geometry.getLeft()
+                    + "' obtained for an attribute with metadata '" + attrMetadataStr + "' and value '" + attrValue
+                    + "'");
             throw e;
         } // try catch // try catch
     } // testGetGeometryMetadata
-    
+
     /**
-     * [NGSIUtils.getGeometry] -------- When getting a geometry, the original attribute is returned when the
-     * attribute type is not geo:point and there is no WGS84 geometry metadata.
+     * [NGSIUtils.getGeometry] -------- When getting a geometry, the original
+     * attribute is returned when the attribute type is not geo:point and there
+     * is no WGS84 geometry metadata.
      */
     @Test
     public void testGetGeometryNoGeolocation() {
@@ -116,23 +123,23 @@ public void testGetGeometryNoGeolocation() {
         String attrValue = "-3.7167, 40.3833";
         String attrType = "coordinates";
         boolean swapCoordinates = false; // irrelevant for this test
-        ImmutablePair<String, Boolean> geometry = NGSIUtils.getGeometry(
-                attrValue, attrType, attrMetadataStr, swapCoordinates);
+        ImmutablePair<String, Boolean> geometry = NGSIUtils.getGeometry(attrValue, attrType, attrMetadataStr,
+                swapCoordinates);
 
         try {
             assertEquals(attrValue, geometry.getLeft());
-            System.out.println(getTestTraceHead("[Utils.getLocation]")
-                    + "-  OK  - Geometry '" + geometry.getLeft() + "' obtained for a not geolocated attribute");
+            System.out.println(getTestTraceHead("[Utils.getLocation]") + "-  OK  - Geometry '" + geometry.getLeft()
+                    + "' obtained for a not geolocated attribute");
         } catch (AssertionError e) {
-            System.out.println(getTestTraceHead("[Utils.getLocation]")
-                    + "- FAIL - Geometry '" + geometry.getLeft() + "' obtained for a not geolocated attribute");
+            System.out.println(getTestTraceHead("[Utils.getLocation]") + "- FAIL - Geometry '" + geometry.getLeft()
+                    + "' obtained for a not geolocated attribute");
             throw e;
         } // try catch // try catch
     } // testGetGeometryNoGeolocation
-    
+
     /**
-     * [NGSIUtils.getGeometry] -------- When getting a geometry, a CartoDB geometry is obtained when passing
-     * an attribute of type 'geo:json'.
+     * [NGSIUtils.getGeometry] -------- When getting a geometry, a CartoDB
+     * geometry is obtained when passing an attribute of type 'geo:json'.
      */
     @Test
     public void testGetGeometryGeojson() {
@@ -143,20 +150,73 @@ public void testGetGeometryGeojson() {
         String attrValue = "{\"coordinates\": [-3.7167, 40.3833], \"type\": \"Point\"}";
         String attrType = "geo:json";
         boolean swapCoordinates = false; // irrelevant for this test
-        ImmutablePair<String, Boolean> geometry = NGSIUtils.getGeometry(
-                attrValue, attrType, attrMetadataStr, swapCoordinates);
+        ImmutablePair<String, Boolean> geometry = NGSIUtils.getGeometry(attrValue, attrType, attrMetadataStr,
+                swapCoordinates);
 
         try {
-            assertEquals("ST_GeomFromGeoJSON('{\"coordinates\": [-3.7167, 40.3833], \"type\": \"Point\"}')", geometry.getLeft());
-            System.out.println(getTestTraceHead("[Utils.getLocation]")
-                    + "-  OK  - Geometry '" + geometry.getLeft() + "' obtained for an attribute of type '" + attrType
-                    + "' and value '" + attrValue + "'");
+            assertEquals("ST_GeomFromGeoJSON('{\"coordinates\": [-3.7167, 40.3833], \"type\": \"Point\"}')",
+                    geometry.getLeft());
+            System.out.println(getTestTraceHead("[Utils.getLocation]") + "-  OK  - Geometry '" + geometry.getLeft()
+                    + "' obtained for an attribute of type '" + attrType + "' and value '" + attrValue + "'");
         } catch (AssertionError e) {
-            System.out.println(getTestTraceHead("[Utils.getLocation]")
-                    + "- FAIL - Geometry '" + geometry.getLeft() + "' obtained for an attribute of type '" + attrType
-                    + "' and value '" + attrValue + "'");
+            System.out.println(getTestTraceHead("[Utils.getLocation]") + "- FAIL - Geometry '" + geometry.getLeft()
+                    + "' obtained for an attribute of type '" + attrType + "' and value '" + attrValue + "'");
             throw e;
         } // try catch // try catch
     } // testGetGeometryGeojson
 
+    /**
+     * [NGSIUtils.getFieldsForInsert] -------- Get fields to insert with quotes
+     */
+    @Test
+    public void testGetFieldForInsertWithQuotes() {
+        LinkedHashMap<String, ArrayList<JsonElement>> aggregation = getValueFields();
+        String resultCompare = "(`recvTime`,`fiwareServicePath`,`entityId`,`entityType`,`load`,`load_md`)";
+        String result = NGSIUtils.getFieldsForInsert(aggregation, "`");
+        try {
+            assertEquals(resultCompare, result);
+            System.out.println(String.format("%s - OK  - getFieldForInsert '%s' is the same as '%s'",
+                    getTestTraceHead("[NGSIUtils.getFieldsForInsert]"), result, resultCompare));
+        } catch (AssertionError e) {
+            System.err.println(String.format("%s - FAIL  - getFieldForInsert '%s' is different from '%s'",
+                    getTestTraceHead("[NGSIUtils.getFieldsForInsert]"), result, resultCompare));
+            throw e;
+        } // try catch
+    }// testGetFieldForInsertWithQuotes
+
+    /**
+     * [NGSIUtils.getFieldsForInsert] -------- Get fields to insert without
+     * quotes
+     */
+    @Test
+    public void testGetFieldForInsertWithoutQuotes() {
+        LinkedHashMap<String, ArrayList<JsonElement>> aggregation = getValueFields();
+        String resultCompare = "(recvTime,fiwareServicePath,entityId,entityType,load,load_md)";
+        String result = NGSIUtils.getFieldsForInsert(aggregation);
+        try {
+            assertEquals(resultCompare, result);
+            System.out.println(String.format("%s - OK  - getFieldForInsert '%s' is the same as '%s'",
+                    getTestTraceHead("[NGSIUtils.getFieldsForInsert]"), result, resultCompare));
+        } catch (AssertionError e) {
+            System.err.println(String.format("%s - FAIL  - getFieldForInsert '%s' is different from '%s'",
+                    getTestTraceHead("[NGSIUtils.getFieldsForInsert]"), result, resultCompare));
+            throw e;
+        } // try catch
+
+    }// testGetFieldForInsertWithoutQuotes
+
+    /**
+     * It's a mock field
+     */
+    private LinkedHashMap<String, ArrayList<JsonElement>> getValueFields() {
+        LinkedHashMap<String, ArrayList<JsonElement>> aggregation = new LinkedHashMap<String, ArrayList<JsonElement>>();
+        aggregation.put("recvTime", null);
+        aggregation.put("fiwareServicePath", null);
+        aggregation.put("entityId", null);
+        aggregation.put("entityType", null);
+        aggregation.put("load", null);
+        aggregation.put("load_md", null);
+        return aggregation;
+    }// getValueFields
+
 } // NGSIUtilsTest