Task/handle textunrestricted attr

CHANGES_NEXT_RELEASE

@@ -1,2 +1,3 @@
+- [cygnus-commons][NGSIHandler] Check TextUnrestricted type to escape character ' as '' (#2125)
 - [cygnus-ngsi][KafkaSink] Upgrade libthrift dependency from 0.12.0 to 0.14.1 due to github vulnerability report
 - [cygnus-commons] Upgrade postgresql dependency from 42.2.22 to 42.2.25 due to github vulnerability report

cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java

@@ -757,6 +757,27 @@ public void setEnableNameMappings(boolean enableNameMappings) {
         this.enableNameMappings = enableNameMappings;
     } //setEnableNameMappings
 
+    /**
+     * getEscapedString
+     *
+     * @param JsonElement an UnrestrictedString
+     * @param String a quotationMark to escape
+     * @return the escaped string
+     */
+    public String getEscapedString(JsonElement value, String quotationMark) {
+        String escaped = value.toString();
+        switch (quotationMark) {
+        case "'":
+            escaped = escaped.replaceAll("'", "''");
+            break;
+        case "\"":
+            // Currently not used but maybe in the future could be useful
+            escaped = escaped.replaceAll("\"", "\"\"");
+            break;
+        }
+        return escaped;
+    }
+
     /**
      * Aggregate declaration for child classes.
      *

cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java

@@ -145,6 +145,8 @@ public void aggregate(NGSIEvent event) {
                 } catch (Exception e) {
                     LOGGER.error("[" + getName() + "] Processing context attribute (name=" + attrValue.toString());
                 }
+            } else if (attrType.equals("TextUnrestricted")) {
+                attrValue = jsonParser.parse(getEscapedString(attrValue, "'"));
             }
             // Check if the attribute already exists in the form of 2 columns (one for metadata); if not existing,
             // add an empty value for all previous rows

cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java

@@ -87,6 +87,9 @@ public void aggregate(NGSIEvent event) {
             JsonArray jsonAttrMetadata = (JsonArray) jsonParser.parse(attrMetadata);
             LOGGER.debug("[" + getName() + "] Processing context attribute (name=" + attrName + ", type="
                     + attrType + ")");
+            if (attrType.equals("TextUnrestricted")) {
+                attrValue = jsonParser.parse(getEscapedString(attrValue, "'"));
+            }
             // aggregate the attribute information
             aggregation.get(NGSIConstants.RECV_TIME_TS).add(new JsonPrimitive(Long.toString(recvTimeTs)));
             aggregation.get(NGSIConstants.RECV_TIME).add(new JsonPrimitive(recvTime));

cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java

@@ -1287,8 +1287,8 @@ private NotifyContextRequest.ContextElement createContextElement() {
         contextAttribute1.setContextMetadata(metadata);
         NotifyContextRequest.ContextAttribute contextAttribute2 = new NotifyContextRequest.ContextAttribute();
         contextAttribute2.setName("someName2");
-        contextAttribute2.setType("someType2");
-        contextAttribute2.setContextValue(new JsonPrimitive("someValue2"));
+        contextAttribute2.setType("TextUnrestricted");
+        contextAttribute2.setContextValue(new JsonPrimitive("someValue'2"));
         contextAttribute2.setContextMetadata(null);
         ArrayList<NotifyContextRequest.ContextAttribute> attributes = new ArrayList<>();
         attributes.add(contextAttribute1);
@@ -1352,7 +1352,7 @@ public void testNativeTypeColumnBatch() throws CygnusBadConfiguration, CygnusRun
                 for (NGSIEvent event : events) {
                     aggregator.aggregate(event);
                 }
-                String correctBatch = "('2016-04-20 07:19:55.801','somePath','someId','someType',2,'[]',TRUE,'[]','2016-09-21T01:23:00.00Z','[]','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]','{\"String\": \"string\"}','[]','foo','[]','','[]',NULL,NULL,NULL,NULL),('2016-04-20 07:19:55.801','somePath','someId','someType',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]','someValue2','[]')";
+                String correctBatch = "('2016-04-20 07:19:55.801','somePath','someId','someType',2,'[]',TRUE,'[]','2016-09-21T01:23:00.00Z','[]','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]','{\"String\": \"string\"}','[]','foo','[]','','[]',NULL,NULL,NULL,NULL),('2016-04-20 07:19:55.801','somePath','someId','someType',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]','someValue''2','[]')";
                 String valuesForInsert = SQLQueryUtils.getValuesForInsert(aggregator.getAggregationToPersist(), aggregator.isAttrNativeTypes());
                 if (valuesForInsert.equals(correctBatch)) {
                     System.out.println(getTestTraceHead("[NGSIMySQKSink.testNativeTypesColumnBatch]")
@@ -1397,7 +1397,7 @@ public void testNativeTypeRowBatch() throws CygnusBadConfiguration, CygnusRuntim
                 for (NGSIEvent event : events) {
                     aggregator.aggregate(event);
                 } // for
-                String correctBatch = "('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someNumber','number','2','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','somneBoolean','Boolean','true','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someDate','DateTime','2016-09-21T01:23:00.00Z','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someGeoJson','geo:json','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someJson','json','{\"String\": \"string\"}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString','string','foo','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString2','string','','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName1','someType1','-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName2','someType2','someValue2','[]')";
+                String correctBatch = "('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someNumber','number','2','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','somneBoolean','Boolean','true','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someDate','DateTime','2016-09-21T01:23:00.00Z','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someGeoJson','geo:json','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someJson','json','{\"String\": \"string\"}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString','string','foo','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString2','string','','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName1','someType1','-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName2','TextUnrestricted','someValue''2','[]')";
                 String valuesForInsert = SQLQueryUtils.getValuesForInsert(aggregator.getAggregationToPersist(), aggregator.isAttrNativeTypes());
                 if (valuesForInsert.equals(correctBatch)) {
                     System.out.println(getTestTraceHead("[NGSIMySQKSink.testNativeTypesRowBatch]")

doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md

@@ -7,6 +7,7 @@ Content:
 * [Administration guide](#section2)
     * [Configuration](#section2.1)
     * [Accepted character set](#section2.2)
+    * [Quote escaping for `TextUnrestricted` attributes](#section2.3)
 * [Programmers guide](#section3)
     * [`NGSIRestHandler` class](#section3.1)
 
@@ -43,26 +44,26 @@ Let's assume the following not-intercepted event regarding a received notificati
 ```
 notification={
    headers={
-	   fiware-service=hotel1,
-	   fiware-servicepath=/other,/suites,
-	   correlation-id=1234567890-0000-1234567890
+       fiware-service=hotel1,
+       fiware-servicepath=/other,/suites,
+       correlation-id=1234567890-0000-1234567890
    },
    body={
       {
-	      entityId=suite.12,
-	      entityType=room,
-	      attributes=[
-	         ...
-	      ]
-	   },
-	   {
-	      entityId=other.9,
-	      entityType=room,
-	      attributes=[
-	         ...
-	      ]
-	   }
-	}
+          entityId=suite.12,
+          entityType=room,
+          attributes=[
+             ...
+          ]
+       },
+       {
+          entityId=other.9,
+          entityType=room,
+          attributes=[
+             ...
+          ]
+       }
+    }
 }
 ```
 
@@ -72,40 +73,40 @@ As can be seen, two entities (`suite.12` and `other.9`) of the same type (`room`
 ```
 ngsi-event-1={
    headers={
-	   fiware-service=hotel,
-	   fiware-servicepath=/suites,
-	   transaction-id=1234567890-0000-1234567890,
-	   correlation-id=1234567890-0000-1234567890,
-	   timestamp=1234567890,
-	   mapped-fiware-service=hotel
-	   mapped-fiware-service-path=/suites
-	},
+       fiware-service=hotel,
+       fiware-servicepath=/suites,
+       transaction-id=1234567890-0000-1234567890,
+       correlation-id=1234567890-0000-1234567890,
+       timestamp=1234567890,
+       mapped-fiware-service=hotel
+       mapped-fiware-service-path=/suites
+    },
    original-context-element={
-	   entityId=suite.12,
-	   entityType=room,
-	   attributes=[
-	      ...
-	   ]
-	}
+       entityId=suite.12,
+       entityType=room,
+       attributes=[
+          ...
+       ]
+    }
 }
 
 ngsi-event-2={
    headers={
-	   fiware-service=hotel,
-	   fiware-servicepath=/other,
-	   transaction-id=1234567890-0000-1234567890,
-	   correlation-id=1234567890-0000-1234567890,
-	   timestamp=1234567890,
-	   mapped-fiware-service=hotel
-	   mapped-fiware-service-path=/other
+       fiware-service=hotel,
+       fiware-servicepath=/other,
+       transaction-id=1234567890-0000-1234567890,
+       correlation-id=1234567890-0000-1234567890,
+       timestamp=1234567890,
+       mapped-fiware-service=hotel
+       mapped-fiware-service-path=/other
    },
    original-context-element={
-	   entityId=other.9,
-	   entityType=room,
-	   attributes=[
-	      ...
-	   ]
-	}
+       entityId=other.9,
+       entityType=room,
+       attributes=[
+          ...
+       ]
+    }
 }
 ```
 
@@ -139,6 +140,13 @@ It is expected UTF-8 character set is maintained by all the Flume elements in th
 
 [Top](#top)
 
+### <a name="section2.3"></a>Quote escaping for `TextUnrestricted` attributes
+Cygnus escapes from `'` to `''` in attributes of type `TextUnrestricted` with the aim of avoiding injection attacks.
+
+Note that other attributes (i.e. with type different to `TextUnrestricted`) don't need such escaping as single quote (`'`) is a [forbidden character in Context Broker](https://fiware-orion.readthedocs.io/en/master/user/forbidden_characters/index.html) so that value will never arrives to Cygnus in notifications.
+
+[Top](#top)
+
 ## <a name="section3"></a>Programmers guide
 ### <a name="section3.1"></a>`NGSIRestHandler` class
 TBD