diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 1bc0d24be..1193e6623 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,3 @@ +- [cygnus-commons][NGSIHandler] Check TextUnrestricted type to escape character ' as '' (#2125) - [cygnus-ngsi][KafkaSink] Upgrade libthrift dependency from 0.12.0 to 0.14.1 due to github vulnerability report - [cygnus-commons] Upgrade postgresql dependency from 42.2.22 to 42.2.25 due to github vulnerability report diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java index 924aebd63..079bea030 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java @@ -757,6 +757,27 @@ public void setEnableNameMappings(boolean enableNameMappings) { this.enableNameMappings = enableNameMappings; } //setEnableNameMappings + /** + * getEscapedString + * + * @param JsonElement an UnrestrictedString + * @param String a quotationMark to escape + * @return the escaped string + */ + public String getEscapedString(JsonElement value, String quotationMark) { + String escaped = value.toString(); + switch (quotationMark) { + case "'": + escaped = escaped.replaceAll("'", "''"); + break; + case "\"": + // Currently not used but maybe in the future could be useful + escaped = escaped.replaceAll("\"", "\"\""); + break; + } + return escaped; + } + /** * Aggregate declaration for child classes. * diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java index 62c1d7f4f..58fcae636 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java @@ -145,6 +145,8 @@ public void aggregate(NGSIEvent event) { } catch (Exception e) { LOGGER.error("[" + getName() + "] Processing context attribute (name=" + attrValue.toString()); } + } else if (attrType.equals("TextUnrestricted")) { + attrValue = jsonParser.parse(getEscapedString(attrValue, "'")); } // Check if the attribute already exists in the form of 2 columns (one for metadata); if not existing, // add an empty value for all previous rows diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java index 17ac9b80f..ca606ebc6 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java @@ -87,6 +87,9 @@ public void aggregate(NGSIEvent event) { JsonArray jsonAttrMetadata = (JsonArray) jsonParser.parse(attrMetadata); LOGGER.debug("[" + getName() + "] Processing context attribute (name=" + attrName + ", type=" + attrType + ")"); + if (attrType.equals("TextUnrestricted")) { + attrValue = jsonParser.parse(getEscapedString(attrValue, "'")); + } // aggregate the attribute information aggregation.get(NGSIConstants.RECV_TIME_TS).add(new JsonPrimitive(Long.toString(recvTimeTs))); aggregation.get(NGSIConstants.RECV_TIME).add(new JsonPrimitive(recvTime)); diff --git a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java index 3f9d66402..7ccf48749 100644 --- a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java +++ b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java @@ -1287,8 +1287,8 @@ private NotifyContextRequest.ContextElement createContextElement() { contextAttribute1.setContextMetadata(metadata); NotifyContextRequest.ContextAttribute contextAttribute2 = new NotifyContextRequest.ContextAttribute(); contextAttribute2.setName("someName2"); - contextAttribute2.setType("someType2"); - contextAttribute2.setContextValue(new JsonPrimitive("someValue2")); + contextAttribute2.setType("TextUnrestricted"); + contextAttribute2.setContextValue(new JsonPrimitive("someValue'2")); contextAttribute2.setContextMetadata(null); ArrayList attributes = new ArrayList<>(); attributes.add(contextAttribute1); @@ -1352,7 +1352,7 @@ public void testNativeTypeColumnBatch() throws CygnusBadConfiguration, CygnusRun for (NGSIEvent event : events) { aggregator.aggregate(event); } - String correctBatch = "('2016-04-20 07:19:55.801','somePath','someId','someType',2,'[]',TRUE,'[]','2016-09-21T01:23:00.00Z','[]','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]','{\"String\": \"string\"}','[]','foo','[]','','[]',NULL,NULL,NULL,NULL),('2016-04-20 07:19:55.801','somePath','someId','someType',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]','someValue2','[]')"; + String correctBatch = "('2016-04-20 07:19:55.801','somePath','someId','someType',2,'[]',TRUE,'[]','2016-09-21T01:23:00.00Z','[]','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]','{\"String\": \"string\"}','[]','foo','[]','','[]',NULL,NULL,NULL,NULL),('2016-04-20 07:19:55.801','somePath','someId','someType',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]','someValue''2','[]')"; String valuesForInsert = SQLQueryUtils.getValuesForInsert(aggregator.getAggregationToPersist(), aggregator.isAttrNativeTypes()); if (valuesForInsert.equals(correctBatch)) { System.out.println(getTestTraceHead("[NGSIMySQKSink.testNativeTypesColumnBatch]") @@ -1397,7 +1397,7 @@ public void testNativeTypeRowBatch() throws CygnusBadConfiguration, CygnusRuntim for (NGSIEvent event : events) { aggregator.aggregate(event); } // for - String correctBatch = "('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someNumber','number','2','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','somneBoolean','Boolean','true','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someDate','DateTime','2016-09-21T01:23:00.00Z','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someGeoJson','geo:json','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someJson','json','{\"String\": \"string\"}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString','string','foo','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString2','string','','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName1','someType1','-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName2','someType2','someValue2','[]')"; + String correctBatch = "('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someNumber','number','2','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','somneBoolean','Boolean','true','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someDate','DateTime','2016-09-21T01:23:00.00Z','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someGeoJson','geo:json','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someJson','json','{\"String\": \"string\"}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString','string','foo','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString2','string','','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName1','someType1','-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName2','TextUnrestricted','someValue''2','[]')"; String valuesForInsert = SQLQueryUtils.getValuesForInsert(aggregator.getAggregationToPersist(), aggregator.isAttrNativeTypes()); if (valuesForInsert.equals(correctBatch)) { System.out.println(getTestTraceHead("[NGSIMySQKSink.testNativeTypesRowBatch]") diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md index 16b8d7d18..bf33f20da 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md @@ -7,6 +7,7 @@ Content: * [Administration guide](#section2) * [Configuration](#section2.1) * [Accepted character set](#section2.2) + * [Quote escaping for `TextUnrestricted` attributes](#section2.3) * [Programmers guide](#section3) * [`NGSIRestHandler` class](#section3.1) @@ -43,26 +44,26 @@ Let's assume the following not-intercepted event regarding a received notificati ``` notification={ headers={ - fiware-service=hotel1, - fiware-servicepath=/other,/suites, - correlation-id=1234567890-0000-1234567890 + fiware-service=hotel1, + fiware-servicepath=/other,/suites, + correlation-id=1234567890-0000-1234567890 }, body={ { - entityId=suite.12, - entityType=room, - attributes=[ - ... - ] - }, - { - entityId=other.9, - entityType=room, - attributes=[ - ... - ] - } - } + entityId=suite.12, + entityType=room, + attributes=[ + ... + ] + }, + { + entityId=other.9, + entityType=room, + attributes=[ + ... + ] + } + } } ``` @@ -72,40 +73,40 @@ As can be seen, two entities (`suite.12` and `other.9`) of the same type (`room` ``` ngsi-event-1={ headers={ - fiware-service=hotel, - fiware-servicepath=/suites, - transaction-id=1234567890-0000-1234567890, - correlation-id=1234567890-0000-1234567890, - timestamp=1234567890, - mapped-fiware-service=hotel - mapped-fiware-service-path=/suites - }, + fiware-service=hotel, + fiware-servicepath=/suites, + transaction-id=1234567890-0000-1234567890, + correlation-id=1234567890-0000-1234567890, + timestamp=1234567890, + mapped-fiware-service=hotel + mapped-fiware-service-path=/suites + }, original-context-element={ - entityId=suite.12, - entityType=room, - attributes=[ - ... - ] - } + entityId=suite.12, + entityType=room, + attributes=[ + ... + ] + } } ngsi-event-2={ headers={ - fiware-service=hotel, - fiware-servicepath=/other, - transaction-id=1234567890-0000-1234567890, - correlation-id=1234567890-0000-1234567890, - timestamp=1234567890, - mapped-fiware-service=hotel - mapped-fiware-service-path=/other + fiware-service=hotel, + fiware-servicepath=/other, + transaction-id=1234567890-0000-1234567890, + correlation-id=1234567890-0000-1234567890, + timestamp=1234567890, + mapped-fiware-service=hotel + mapped-fiware-service-path=/other }, original-context-element={ - entityId=other.9, - entityType=room, - attributes=[ - ... - ] - } + entityId=other.9, + entityType=room, + attributes=[ + ... + ] + } } ``` @@ -139,6 +140,13 @@ It is expected UTF-8 character set is maintained by all the Flume elements in th [Top](#top) +### Quote escaping for `TextUnrestricted` attributes +Cygnus escapes from `'` to `''` in attributes of type `TextUnrestricted` with the aim of avoiding injection attacks. + +Note that other attributes (i.e. with type different to `TextUnrestricted`) don't need such escaping as single quote (`'`) is a [forbidden character in Context Broker](https://fiware-orion.readthedocs.io/en/master/user/forbidden_characters/index.html) so that value will never arrives to Cygnus in notifications. + +[Top](#top) + ## Programmers guide ### `NGSIRestHandler` class TBD