diff --git a/.gitignore b/.gitignore index e4d9a322a..a467107b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.metadata target/ # IntelliJ local files *.idea diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 1569f8b6c..006887f46 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,3 @@ -[cygnus-common][SQLBackendImpl] Add configuration option to persist errors in SQL sinks (#1928) +[cygnus-common][SQLBackendImpl] Add configuration option to persist errors in SQL sinks (#1928) [cygnus-ngsi-ld] Creation of the new PostGIS sink for persisting NGSI-LD notifications (#1905) - +[cygnus-ngsi][MysqlSink] Quoted Sql field names to avoid naming problems. (#1863) diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSink.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSink.java index 7636c9708..fb082bd14 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSink.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSink.java @@ -48,6 +48,7 @@ */ public class NGSIMySQLSink extends NGSISink { + private static final String MYSQL_QUOTE_CHAR = "`"; private static final String DEFAULT_ROW_ATTR_PERSISTENCE = "row"; private static final String DEFAULT_PASSWORD = ""; private static final String DEFAULT_PORT = "3306"; @@ -364,7 +365,7 @@ protected NGSIGenericAggregator getAggregator(boolean rowAttrPersistence) { private void persistAggregation(NGSIGenericAggregator aggregator) throws CygnusPersistenceError, CygnusRuntimeError, CygnusBadContextData { String fieldsForCreate = NGSIUtils.getFieldsForCreate(aggregator.getAggregationToPersist()); - String fieldsForInsert = NGSIUtils.getFieldsForInsert(aggregator.getAggregationToPersist()); + String fieldsForInsert = NGSIUtils.getFieldsForInsert(aggregator.getAggregationToPersist(), MYSQL_QUOTE_CHAR); String valuesForInsert = NGSIUtils.getValuesForInsert(aggregator.getAggregationToPersist(), aggregator.isAttrNativeTypes()); String dbName = aggregator.getDbName(enableLowercase); String tableName = aggregator.getTableName(enableLowercase); diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java index c7acdea76..1ac4249dd 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/utils/NGSIUtils.java @@ -22,7 +22,6 @@ import com.google.gson.JsonObject; import com.telefonica.iot.cygnus.log.CygnusLogger; -import java.lang.reflect.Array; import java.util.*; import java.util.regex.Pattern; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -31,8 +30,6 @@ import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; -import static java.io.File.separator; - /** * * @author frb @@ -271,15 +268,26 @@ public static String getFieldsForCreate(LinkedHashMap> aggregation) { + return getFieldsForInsert(aggregation, ""); + } + + /** + * Gets fields for insert. + * + * @param aggregation the aggregation + * @param quoteChar char to quote field names + * @return he fields (column names) for insert in SQL format. + */ + public static String getFieldsForInsert(LinkedHashMap> aggregation, String quoteChar) { String fieldsForInsert = "("; boolean first = true; Iterator it = aggregation.keySet().iterator(); while (it.hasNext()) { if (first) { - fieldsForInsert += (String) it.next(); + fieldsForInsert += quoteChar + (String) it.next() + quoteChar; first = false; } else { - fieldsForInsert += "," + (String) it.next(); + fieldsForInsert += "," + quoteChar + (String) it.next() + quoteChar; } // if else } // while return fieldsForInsert + ")"; @@ -398,7 +406,7 @@ public static ArrayList attributeNames(LinkedHashMap cropedList = (ArrayList) attributeNames.clone(); + ArrayList cropedList = (ArrayList) attributeNames.clone(); for (String key : cropedList) { if (key.contains("_md") || key.contains(NGSIConstants.AUTOGENERATED_ATTR_TYPE)) { attributeNames.remove(key); diff --git a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/utils/NGSIUtilsTest.java b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/utils/NGSIUtilsTest.java index 596c3e455..576fc102f 100644 --- a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/utils/NGSIUtilsTest.java +++ b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/utils/NGSIUtilsTest.java @@ -18,30 +18,36 @@ package com.telefonica.iot.cygnus.utils; import static com.telefonica.iot.cygnus.utils.CommonUtilsForTests.getTestTraceHead; +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.LinkedHashMap; + import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.json.simple.JSONArray; import org.json.simple.JSONObject; -import static org.junit.Assert.assertEquals; import org.junit.Test; +import com.google.gson.JsonElement; + /** * * @author frb */ public class NGSIUtilsTest { - + /** * Constructor. */ public NGSIUtilsTest() { LogManager.getRootLogger().setLevel(Level.FATAL); } // NGSIUtilsTest - + /** - * [NGSIUtils.getGeometry] -------- When getting a geometry, a CartoDB point is obtained when passing - * an attribute of type 'geo:point'. + * [NGSIUtils.getGeometry] -------- When getting a geometry, a CartoDB point + * is obtained when passing an attribute of type 'geo:point'. */ @Test public void testGetGeometryGeopoint() { @@ -52,25 +58,24 @@ public void testGetGeometryGeopoint() { String attrValue = "-3.7167, 40.3833"; String attrType = "geo:point"; boolean swapCoordinates = false; // irrelevant for this test - ImmutablePair geometry = NGSIUtils.getGeometry( - attrValue, attrType, attrMetadataStr, swapCoordinates); + ImmutablePair geometry = NGSIUtils.getGeometry(attrValue, attrType, attrMetadataStr, + swapCoordinates); try { - assertEquals("ST_SetSRID(ST_MakePoint(-3.7167::double precision , 40.3833::double precision ), 4326)", geometry.getLeft()); - System.out.println(getTestTraceHead("[Utils.getLocation]") - + "- OK - Geometry '" + geometry.getLeft() + "' obtained for an attribute of type '" + attrType - + "' and value '" + attrValue + "'"); + assertEquals("ST_SetSRID(ST_MakePoint(-3.7167::double precision , 40.3833::double precision ), 4326)", + geometry.getLeft()); + System.out.println(getTestTraceHead("[Utils.getLocation]") + "- OK - Geometry '" + geometry.getLeft() + + "' obtained for an attribute of type '" + attrType + "' and value '" + attrValue + "'"); } catch (AssertionError e) { - System.out.println(getTestTraceHead("[Utils.getLocation]") - + "- FAIL - Geometry '" + geometry.getLeft() + "' obtained for an attribute of type '" + attrType - + "' and value '" + attrValue + "'"); + System.out.println(getTestTraceHead("[Utils.getLocation]") + "- FAIL - Geometry '" + geometry.getLeft() + + "' obtained for an attribute of type '" + attrType + "' and value '" + attrValue + "'"); throw e; } // try catch // try catch } // testGetGeometryGeopoint - + /** - * [NGSIUtils.getGeometry] -------- When getting a geometry, a CartoDB point is obtained when passing - * an attribute with 'geometry' metadata. + * [NGSIUtils.getGeometry] -------- When getting a geometry, a CartoDB point + * is obtained when passing an attribute with 'geometry' metadata. */ @Test public void testGetGeometryMetadata() { @@ -87,25 +92,27 @@ public void testGetGeometryMetadata() { String attrValue = "-3.7167, 40.3833"; String attrType = "coordinates"; // irrelevant for this test boolean swapCoordinates = false; // irrelevant for this test - ImmutablePair geometry = NGSIUtils.getGeometry( - attrValue, attrType, attrMetadataStr, swapCoordinates); + ImmutablePair geometry = NGSIUtils.getGeometry(attrValue, attrType, attrMetadataStr, + swapCoordinates); try { - assertEquals("ST_SetSRID(ST_MakePoint(-3.7167::double precision , 40.3833::double precision ), 4326)", geometry.getLeft()); - System.out.println(getTestTraceHead("[Utils.getLocation]") - + "- OK - Geometry '" + geometry.getLeft() + "' obtained for an attribute with metadata '" - + attrMetadataStr + "' and value '" + attrValue + "'"); + assertEquals("ST_SetSRID(ST_MakePoint(-3.7167::double precision , 40.3833::double precision ), 4326)", + geometry.getLeft()); + System.out.println(getTestTraceHead("[Utils.getLocation]") + "- OK - Geometry '" + geometry.getLeft() + + "' obtained for an attribute with metadata '" + attrMetadataStr + "' and value '" + attrValue + + "'"); } catch (AssertionError e) { - System.out.println(getTestTraceHead("[Utils.getLocation]") - + "- FAIL - Geometry '" + geometry.getLeft() + "' obtained for an attribute with metadata '" - + attrMetadataStr + "' and value '" + attrValue + "'"); + System.out.println(getTestTraceHead("[Utils.getLocation]") + "- FAIL - Geometry '" + geometry.getLeft() + + "' obtained for an attribute with metadata '" + attrMetadataStr + "' and value '" + attrValue + + "'"); throw e; } // try catch // try catch } // testGetGeometryMetadata - + /** - * [NGSIUtils.getGeometry] -------- When getting a geometry, the original attribute is returned when the - * attribute type is not geo:point and there is no WGS84 geometry metadata. + * [NGSIUtils.getGeometry] -------- When getting a geometry, the original + * attribute is returned when the attribute type is not geo:point and there + * is no WGS84 geometry metadata. */ @Test public void testGetGeometryNoGeolocation() { @@ -116,23 +123,23 @@ public void testGetGeometryNoGeolocation() { String attrValue = "-3.7167, 40.3833"; String attrType = "coordinates"; boolean swapCoordinates = false; // irrelevant for this test - ImmutablePair geometry = NGSIUtils.getGeometry( - attrValue, attrType, attrMetadataStr, swapCoordinates); + ImmutablePair geometry = NGSIUtils.getGeometry(attrValue, attrType, attrMetadataStr, + swapCoordinates); try { assertEquals(attrValue, geometry.getLeft()); - System.out.println(getTestTraceHead("[Utils.getLocation]") - + "- OK - Geometry '" + geometry.getLeft() + "' obtained for a not geolocated attribute"); + System.out.println(getTestTraceHead("[Utils.getLocation]") + "- OK - Geometry '" + geometry.getLeft() + + "' obtained for a not geolocated attribute"); } catch (AssertionError e) { - System.out.println(getTestTraceHead("[Utils.getLocation]") - + "- FAIL - Geometry '" + geometry.getLeft() + "' obtained for a not geolocated attribute"); + System.out.println(getTestTraceHead("[Utils.getLocation]") + "- FAIL - Geometry '" + geometry.getLeft() + + "' obtained for a not geolocated attribute"); throw e; } // try catch // try catch } // testGetGeometryNoGeolocation - + /** - * [NGSIUtils.getGeometry] -------- When getting a geometry, a CartoDB geometry is obtained when passing - * an attribute of type 'geo:json'. + * [NGSIUtils.getGeometry] -------- When getting a geometry, a CartoDB + * geometry is obtained when passing an attribute of type 'geo:json'. */ @Test public void testGetGeometryGeojson() { @@ -143,20 +150,73 @@ public void testGetGeometryGeojson() { String attrValue = "{\"coordinates\": [-3.7167, 40.3833], \"type\": \"Point\"}"; String attrType = "geo:json"; boolean swapCoordinates = false; // irrelevant for this test - ImmutablePair geometry = NGSIUtils.getGeometry( - attrValue, attrType, attrMetadataStr, swapCoordinates); + ImmutablePair geometry = NGSIUtils.getGeometry(attrValue, attrType, attrMetadataStr, + swapCoordinates); try { - assertEquals("ST_GeomFromGeoJSON('{\"coordinates\": [-3.7167, 40.3833], \"type\": \"Point\"}')", geometry.getLeft()); - System.out.println(getTestTraceHead("[Utils.getLocation]") - + "- OK - Geometry '" + geometry.getLeft() + "' obtained for an attribute of type '" + attrType - + "' and value '" + attrValue + "'"); + assertEquals("ST_GeomFromGeoJSON('{\"coordinates\": [-3.7167, 40.3833], \"type\": \"Point\"}')", + geometry.getLeft()); + System.out.println(getTestTraceHead("[Utils.getLocation]") + "- OK - Geometry '" + geometry.getLeft() + + "' obtained for an attribute of type '" + attrType + "' and value '" + attrValue + "'"); } catch (AssertionError e) { - System.out.println(getTestTraceHead("[Utils.getLocation]") - + "- FAIL - Geometry '" + geometry.getLeft() + "' obtained for an attribute of type '" + attrType - + "' and value '" + attrValue + "'"); + System.out.println(getTestTraceHead("[Utils.getLocation]") + "- FAIL - Geometry '" + geometry.getLeft() + + "' obtained for an attribute of type '" + attrType + "' and value '" + attrValue + "'"); throw e; } // try catch // try catch } // testGetGeometryGeojson + /** + * [NGSIUtils.getFieldsForInsert] -------- Get fields to insert with quotes + */ + @Test + public void testGetFieldForInsertWithQuotes() { + LinkedHashMap> aggregation = getValueFields(); + String resultCompare = "(`recvTime`,`fiwareServicePath`,`entityId`,`entityType`,`load`,`load_md`)"; + String result = NGSIUtils.getFieldsForInsert(aggregation, "`"); + try { + assertEquals(resultCompare, result); + System.out.println(String.format("%s - OK - getFieldForInsert '%s' is the same as '%s'", + getTestTraceHead("[NGSIUtils.getFieldsForInsert]"), result, resultCompare)); + } catch (AssertionError e) { + System.err.println(String.format("%s - FAIL - getFieldForInsert '%s' is different from '%s'", + getTestTraceHead("[NGSIUtils.getFieldsForInsert]"), result, resultCompare)); + throw e; + } // try catch + }// testGetFieldForInsertWithQuotes + + /** + * [NGSIUtils.getFieldsForInsert] -------- Get fields to insert without + * quotes + */ + @Test + public void testGetFieldForInsertWithoutQuotes() { + LinkedHashMap> aggregation = getValueFields(); + String resultCompare = "(recvTime,fiwareServicePath,entityId,entityType,load,load_md)"; + String result = NGSIUtils.getFieldsForInsert(aggregation); + try { + assertEquals(resultCompare, result); + System.out.println(String.format("%s - OK - getFieldForInsert '%s' is the same as '%s'", + getTestTraceHead("[NGSIUtils.getFieldsForInsert]"), result, resultCompare)); + } catch (AssertionError e) { + System.err.println(String.format("%s - FAIL - getFieldForInsert '%s' is different from '%s'", + getTestTraceHead("[NGSIUtils.getFieldsForInsert]"), result, resultCompare)); + throw e; + } // try catch + + }// testGetFieldForInsertWithoutQuotes + + /** + * It's a mock field + */ + private LinkedHashMap> getValueFields() { + LinkedHashMap> aggregation = new LinkedHashMap>(); + aggregation.put("recvTime", null); + aggregation.put("fiwareServicePath", null); + aggregation.put("entityId", null); + aggregation.put("entityType", null); + aggregation.put("load", null); + aggregation.put("load_md", null); + return aggregation; + }// getValueFields + } // NGSIUtilsTest