From 585a3098d5e0fa1bcd411090f327d57be53094ff Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 11:28:51 +0100 Subject: [PATCH 01/22] handle TextUnrestricted attr value --- .../iot/cygnus/aggregation/NGSIGenericColumnAggregator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java index 62c1d7f4f..60d2df17c 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java @@ -145,6 +145,8 @@ public void aggregate(NGSIEvent event) { } catch (Exception e) { LOGGER.error("[" + getName() + "] Processing context attribute (name=" + attrValue.toString()); } + } else if (attrType.equals("TextUnrestricted")) { + attrValue = attrValue.replaceAll("'", "''"); } // Check if the attribute already exists in the form of 2 columns (one for metadata); if not existing, // add an empty value for all previous rows From 338ab827d6510c05e8f90ec72308fcda8d04eb51 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 11:30:46 +0100 Subject: [PATCH 02/22] update test --- .../telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java index 3f9d66402..382d97f1b 100644 --- a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java +++ b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java @@ -1287,8 +1287,8 @@ private NotifyContextRequest.ContextElement createContextElement() { contextAttribute1.setContextMetadata(metadata); NotifyContextRequest.ContextAttribute contextAttribute2 = new NotifyContextRequest.ContextAttribute(); contextAttribute2.setName("someName2"); - contextAttribute2.setType("someType2"); - contextAttribute2.setContextValue(new JsonPrimitive("someValue2")); + contextAttribute2.setType("TextUnrestricted"); + contextAttribute2.setContextValue(new JsonPrimitive("someValue'2")); contextAttribute2.setContextMetadata(null); ArrayList attributes = new ArrayList<>(); attributes.add(contextAttribute1); @@ -1352,7 +1352,7 @@ public void testNativeTypeColumnBatch() throws CygnusBadConfiguration, CygnusRun for (NGSIEvent event : events) { aggregator.aggregate(event); } - String correctBatch = "('2016-04-20 07:19:55.801','somePath','someId','someType',2,'[]',TRUE,'[]','2016-09-21T01:23:00.00Z','[]','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]','{\"String\": \"string\"}','[]','foo','[]','','[]',NULL,NULL,NULL,NULL),('2016-04-20 07:19:55.801','somePath','someId','someType',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]','someValue2','[]')"; + String correctBatch = "('2016-04-20 07:19:55.801','somePath','someId','someType',2,'[]',TRUE,'[]','2016-09-21T01:23:00.00Z','[]','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]','{\"String\": \"string\"}','[]','foo','[]','','[]',NULL,NULL,NULL,NULL),('2016-04-20 07:19:55.801','somePath','someId','someType',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]','someValue''2','[]')"; String valuesForInsert = SQLQueryUtils.getValuesForInsert(aggregator.getAggregationToPersist(), aggregator.isAttrNativeTypes()); if (valuesForInsert.equals(correctBatch)) { System.out.println(getTestTraceHead("[NGSIMySQKSink.testNativeTypesColumnBatch]") @@ -1397,7 +1397,7 @@ public void testNativeTypeRowBatch() throws CygnusBadConfiguration, CygnusRuntim for (NGSIEvent event : events) { aggregator.aggregate(event); } // for - String correctBatch = "('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someNumber','number','2','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','somneBoolean','Boolean','true','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someDate','DateTime','2016-09-21T01:23:00.00Z','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someGeoJson','geo:json','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someJson','json','{\"String\": \"string\"}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString','string','foo','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString2','string','','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName1','someType1','-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName2','someType2','someValue2','[]')"; + String correctBatch = "('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someNumber','number','2','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','somneBoolean','Boolean','true','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someDate','DateTime','2016-09-21T01:23:00.00Z','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someGeoJson','geo:json','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someJson','json','{\"String\": \"string\"}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString','string','foo','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString2','string','','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName1','someType1','-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName2','someType2','someValue''2','[]')"; String valuesForInsert = SQLQueryUtils.getValuesForInsert(aggregator.getAggregationToPersist(), aggregator.isAttrNativeTypes()); if (valuesForInsert.equals(correctBatch)) { System.out.println(getTestTraceHead("[NGSIMySQKSink.testNativeTypesRowBatch]") From b7e5017a47d1c94893e937dbf746ed0087baed6e Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 11:34:24 +0100 Subject: [PATCH 03/22] update test --- .../java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java index 382d97f1b..7ccf48749 100644 --- a/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java +++ b/cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java @@ -1397,7 +1397,7 @@ public void testNativeTypeRowBatch() throws CygnusBadConfiguration, CygnusRuntim for (NGSIEvent event : events) { aggregator.aggregate(event); } // for - String correctBatch = "('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someNumber','number','2','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','somneBoolean','Boolean','true','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someDate','DateTime','2016-09-21T01:23:00.00Z','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someGeoJson','geo:json','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someJson','json','{\"String\": \"string\"}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString','string','foo','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString2','string','','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName1','someType1','-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName2','someType2','someValue''2','[]')"; + String correctBatch = "('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someNumber','number','2','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','somneBoolean','Boolean','true','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someDate','DateTime','2016-09-21T01:23:00.00Z','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someGeoJson','geo:json','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someJson','json','{\"String\": \"string\"}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString','string','foo','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString2','string','','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName1','someType1','-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName2','TextUnrestricted','someValue''2','[]')"; String valuesForInsert = SQLQueryUtils.getValuesForInsert(aggregator.getAggregationToPersist(), aggregator.isAttrNativeTypes()); if (valuesForInsert.equals(correctBatch)) { System.out.println(getTestTraceHead("[NGSIMySQKSink.testNativeTypesRowBatch]") From 27705a146b0677935fba859bea17a2b7012a490c Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 11:39:04 +0100 Subject: [PATCH 04/22] get String when textUnrestricted --- .../iot/cygnus/aggregation/NGSIGenericColumnAggregator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java index 60d2df17c..2d1e94448 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java @@ -146,7 +146,7 @@ public void aggregate(NGSIEvent event) { LOGGER.error("[" + getName() + "] Processing context attribute (name=" + attrValue.toString()); } } else if (attrType.equals("TextUnrestricted")) { - attrValue = attrValue.replaceAll("'", "''"); + attrValue = attrValue.toString().replaceAll("'", "''"); } // Check if the attribute already exists in the form of 2 columns (one for metadata); if not existing, // add an empty value for all previous rows From 45f46d40ee4a36b190da3af662a052fcd373f6e4 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 11:44:20 +0100 Subject: [PATCH 05/22] get String when textUnrestricted --- .../iot/cygnus/aggregation/NGSIGenericColumnAggregator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java index 2d1e94448..962615a09 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java @@ -146,7 +146,7 @@ public void aggregate(NGSIEvent event) { LOGGER.error("[" + getName() + "] Processing context attribute (name=" + attrValue.toString()); } } else if (attrType.equals("TextUnrestricted")) { - attrValue = attrValue.toString().replaceAll("'", "''"); + attrValue = jsonParser.parse(attrValue.toString().replaceAll("'", "''")); } // Check if the attribute already exists in the form of 2 columns (one for metadata); if not existing, // add an empty value for all previous rows From 4d8cb9ecbe76ba788c1d0538b59a15e85bd4397b Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 11:58:31 +0100 Subject: [PATCH 06/22] apply for row aggregator --- .../iot/cygnus/aggregation/NGSIGenericRowAggregator.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java index 17ac9b80f..f85eec96d 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java @@ -87,6 +87,9 @@ public void aggregate(NGSIEvent event) { JsonArray jsonAttrMetadata = (JsonArray) jsonParser.parse(attrMetadata); LOGGER.debug("[" + getName() + "] Processing context attribute (name=" + attrName + ", type=" + attrType + ")"); + if (attrType.equals("TextUnrestricted")) { + attrValue = jsonParser.parse(attrValue.toString().replaceAll("'", "''")); + } // aggregate the attribute information aggregation.get(NGSIConstants.RECV_TIME_TS).add(new JsonPrimitive(Long.toString(recvTimeTs))); aggregation.get(NGSIConstants.RECV_TIME).add(new JsonPrimitive(recvTime)); From 2f1c3047fb769c859140dcbc434a858558e038d4 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 12:30:59 +0100 Subject: [PATCH 07/22] use getEscapedString --- .../aggregation/NGSIGenericAggregator.java | 21 +++++++++++++++++++ .../NGSIGenericColumnAggregator.java | 2 +- .../aggregation/NGSIGenericRowAggregator.java | 2 +- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java index 924aebd63..6dc22b6a0 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java @@ -757,6 +757,27 @@ public void setEnableNameMappings(boolean enableNameMappings) { this.enableNameMappings = enableNameMappings; } //setEnableNameMappings + /** + * getEscapedString + * + * @param JsonElement an UnrestrictedString + * @param String a quotationMark to escape + * @return the escaped string + */ + public String getEscapedString(JsonElement value, String quotationMark) { + String escaped = value.toString(); + switch (quotationMark) { + case "'": + escaped = escaped.replaceAll("'", "''"); + break; + case '"': + escaped = escaped.replaceAll('"', '""'); + break; + } + escaped = quotationMark + escaped + quotationMark; + return escaped; + } + /** * Aggregate declaration for child classes. * diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java index 962615a09..58fcae636 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java @@ -146,7 +146,7 @@ public void aggregate(NGSIEvent event) { LOGGER.error("[" + getName() + "] Processing context attribute (name=" + attrValue.toString()); } } else if (attrType.equals("TextUnrestricted")) { - attrValue = jsonParser.parse(attrValue.toString().replaceAll("'", "''")); + attrValue = jsonParser.parse(getEscapedString(attrValue, "'")); } // Check if the attribute already exists in the form of 2 columns (one for metadata); if not existing, // add an empty value for all previous rows diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java index f85eec96d..ca606ebc6 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java @@ -88,7 +88,7 @@ public void aggregate(NGSIEvent event) { LOGGER.debug("[" + getName() + "] Processing context attribute (name=" + attrName + ", type=" + attrType + ")"); if (attrType.equals("TextUnrestricted")) { - attrValue = jsonParser.parse(attrValue.toString().replaceAll("'", "''")); + attrValue = jsonParser.parse(getEscapedString(attrValue, "'")); } // aggregate the attribute information aggregation.get(NGSIConstants.RECV_TIME_TS).add(new JsonPrimitive(Long.toString(recvTimeTs))); From ebaabbe1fcec4b41ba8b961c491f805cdc487c06 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 12:38:14 +0100 Subject: [PATCH 08/22] update cnr --- CHANGES_NEXT_RELEASE | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 1bc0d24be..0e1c3971c 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,2 +1,3 @@ +- [cygnus-commons] Check TextUnrestricted type to escape character ' as '' (#2125) - [cygnus-ngsi][KafkaSink] Upgrade libthrift dependency from 0.12.0 to 0.14.1 due to github vulnerability report - [cygnus-commons] Upgrade postgresql dependency from 42.2.22 to 42.2.25 due to github vulnerability report From b8eb08bd6aa5fe737c0faabb4ce2f5ac05d362f5 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 12:42:42 +0100 Subject: [PATCH 09/22] fix escape --- .../iot/cygnus/aggregation/NGSIGenericAggregator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java index 6dc22b6a0..e8899f4db 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java @@ -771,7 +771,7 @@ public String getEscapedString(JsonElement value, String quotationMark) { escaped = escaped.replaceAll("'", "''"); break; case '"': - escaped = escaped.replaceAll('"', '""'); + escaped = escaped.replaceAll('"', '\"'); break; } escaped = quotationMark + escaped + quotationMark; From df6e57cd657e58e6b2fab2f6125aabaa2a28cbc3 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 12:45:14 +0100 Subject: [PATCH 10/22] fix escape --- .../iot/cygnus/aggregation/NGSIGenericAggregator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java index e8899f4db..84a1f9c17 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java @@ -770,9 +770,9 @@ public String getEscapedString(JsonElement value, String quotationMark) { case "'": escaped = escaped.replaceAll("'", "''"); break; - case '"': - escaped = escaped.replaceAll('"', '\"'); - break; + // case '"': + // escaped = escaped.replaceAll('"', '\"'); + // break; } escaped = quotationMark + escaped + quotationMark; return escaped; From 917a345744815bee0112939b887f873784c5bf3a Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 12:52:48 +0100 Subject: [PATCH 11/22] do not concat quotationMark --- .../telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java index 84a1f9c17..e85818d52 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java @@ -774,7 +774,6 @@ public String getEscapedString(JsonElement value, String quotationMark) { // escaped = escaped.replaceAll('"', '\"'); // break; } - escaped = quotationMark + escaped + quotationMark; return escaped; } From 34cbf66046173310315a9e78d980dff299114f0c Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 12:58:25 +0100 Subject: [PATCH 12/22] add other quotationMark case --- .../iot/cygnus/aggregation/NGSIGenericAggregator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java index e85818d52..9ea70f7ac 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java @@ -770,9 +770,9 @@ public String getEscapedString(JsonElement value, String quotationMark) { case "'": escaped = escaped.replaceAll("'", "''"); break; - // case '"': - // escaped = escaped.replaceAll('"', '\"'); - // break; + case "\"": + escaped = escaped.replaceAll("\"", "\"\""); + break; } return escaped; } From 78de604222f5e6468e84102117a216654b6ee7f4 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 13:29:06 +0100 Subject: [PATCH 13/22] Create unrestricted_type.md --- doc/cygnus-ngsi/user_and_programmer_guide/unrestricted_type.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/cygnus-ngsi/user_and_programmer_guide/unrestricted_type.md diff --git a/doc/cygnus-ngsi/user_and_programmer_guide/unrestricted_type.md b/doc/cygnus-ngsi/user_and_programmer_guide/unrestricted_type.md new file mode 100644 index 000000000..4aedb284d --- /dev/null +++ b/doc/cygnus-ngsi/user_and_programmer_guide/unrestricted_type.md @@ -0,0 +1 @@ +An Attribute value of a TextUnrestricted type from an ContextBroker event is escaped from character ' to '' From f4bdb5e0a8dd52f79fd43f8bcd958deb9d9f3cb5 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 13:30:07 +0100 Subject: [PATCH 14/22] Update README.md --- doc/cygnus-ngsi/user_and_programmer_guide/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/cygnus-ngsi/user_and_programmer_guide/README.md b/doc/cygnus-ngsi/user_and_programmer_guide/README.md index 90525662a..75da1b352 100644 --- a/doc/cygnus-ngsi/user_and_programmer_guide/README.md +++ b/doc/cygnus-ngsi/user_and_programmer_guide/README.md @@ -3,6 +3,7 @@ * [Introduction](./introduction.md) * User guide: * [Connecting Orion Context Broker and Cygnus](./connecting_orion.md) + * [ContextBroker Unrestricted type is escaped](./unrestricted_type.md) * Programmer guide: * [Adding a new sink](./adding_new_sink.md) * [Debugging](./debugging.md) From 04c5075f33a0321f9112113b975f6e46abc304a8 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 13:39:51 +0100 Subject: [PATCH 15/22] update doc --- .../ngsi_rest_handler.md | 93 ++++++++++--------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md index 16b8d7d18..b19babb42 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md @@ -43,26 +43,26 @@ Let's assume the following not-intercepted event regarding a received notificati ``` notification={ headers={ - fiware-service=hotel1, - fiware-servicepath=/other,/suites, - correlation-id=1234567890-0000-1234567890 + fiware-service=hotel1, + fiware-servicepath=/other,/suites, + correlation-id=1234567890-0000-1234567890 }, body={ { - entityId=suite.12, - entityType=room, - attributes=[ - ... - ] - }, - { - entityId=other.9, - entityType=room, - attributes=[ - ... - ] - } - } + entityId=suite.12, + entityType=room, + attributes=[ + ... + ] + }, + { + entityId=other.9, + entityType=room, + attributes=[ + ... + ] + } + } } ``` @@ -72,40 +72,40 @@ As can be seen, two entities (`suite.12` and `other.9`) of the same type (`room` ``` ngsi-event-1={ headers={ - fiware-service=hotel, - fiware-servicepath=/suites, - transaction-id=1234567890-0000-1234567890, - correlation-id=1234567890-0000-1234567890, - timestamp=1234567890, - mapped-fiware-service=hotel - mapped-fiware-service-path=/suites - }, + fiware-service=hotel, + fiware-servicepath=/suites, + transaction-id=1234567890-0000-1234567890, + correlation-id=1234567890-0000-1234567890, + timestamp=1234567890, + mapped-fiware-service=hotel + mapped-fiware-service-path=/suites + }, original-context-element={ - entityId=suite.12, - entityType=room, - attributes=[ - ... - ] - } + entityId=suite.12, + entityType=room, + attributes=[ + ... + ] + } } ngsi-event-2={ headers={ - fiware-service=hotel, - fiware-servicepath=/other, - transaction-id=1234567890-0000-1234567890, - correlation-id=1234567890-0000-1234567890, - timestamp=1234567890, - mapped-fiware-service=hotel - mapped-fiware-service-path=/other + fiware-service=hotel, + fiware-servicepath=/other, + transaction-id=1234567890-0000-1234567890, + correlation-id=1234567890-0000-1234567890, + timestamp=1234567890, + mapped-fiware-service=hotel + mapped-fiware-service-path=/other }, original-context-element={ - entityId=other.9, - entityType=room, - attributes=[ - ... - ] - } + entityId=other.9, + entityType=room, + attributes=[ + ... + ] + } } ``` @@ -139,6 +139,11 @@ It is expected UTF-8 character set is maintained by all the Flume elements in th [Top](#top) +### Non valid characters +An Attribute value of a TextUnrestricted type from an ContextBroker event is escaped from character ' to ''. + +[Top](#top) + ## Programmers guide ### `NGSIRestHandler` class TBD From 246f328f9d1b301fb7b87e90c7578e28ce468709 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 13:51:19 +0100 Subject: [PATCH 16/22] Update doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- .../flume_extensions_catalogue/ngsi_rest_handler.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md index b19babb42..6d53d7259 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md @@ -139,8 +139,10 @@ It is expected UTF-8 character set is maintained by all the Flume elements in th [Top](#top) -### Non valid characters -An Attribute value of a TextUnrestricted type from an ContextBroker event is escaped from character ' to ''. +### Quote escaping for `TextUnrestricted` attributes +Cygnus escapes from `'` to `''` in attributes of type `TextUnrestricted` with the aim of avoiding injection attacks. + +Note that other attributes (i.e. with type different to `TextUnrestricted` don't need such escaping as single quote (`'`) is a [forbidden character in Context Broker](https://fiware-orion.readthedocs.io/en/master/user/forbidden_characters/index.html) so that value will never arrives to Cygnus in notifications. [Top](#top) From ba66dec4ed4ff068dd851df045528eefcb3f4f92 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 13:52:11 +0100 Subject: [PATCH 17/22] Delete unrestricted_type.md --- doc/cygnus-ngsi/user_and_programmer_guide/unrestricted_type.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 doc/cygnus-ngsi/user_and_programmer_guide/unrestricted_type.md diff --git a/doc/cygnus-ngsi/user_and_programmer_guide/unrestricted_type.md b/doc/cygnus-ngsi/user_and_programmer_guide/unrestricted_type.md deleted file mode 100644 index 4aedb284d..000000000 --- a/doc/cygnus-ngsi/user_and_programmer_guide/unrestricted_type.md +++ /dev/null @@ -1 +0,0 @@ -An Attribute value of a TextUnrestricted type from an ContextBroker event is escaped from character ' to '' From 81554eba0f3bcae65062574a3641a91d27120a8f Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 13:52:31 +0100 Subject: [PATCH 18/22] Update README.md --- doc/cygnus-ngsi/user_and_programmer_guide/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/cygnus-ngsi/user_and_programmer_guide/README.md b/doc/cygnus-ngsi/user_and_programmer_guide/README.md index 75da1b352..90525662a 100644 --- a/doc/cygnus-ngsi/user_and_programmer_guide/README.md +++ b/doc/cygnus-ngsi/user_and_programmer_guide/README.md @@ -3,7 +3,6 @@ * [Introduction](./introduction.md) * User guide: * [Connecting Orion Context Broker and Cygnus](./connecting_orion.md) - * [ContextBroker Unrestricted type is escaped](./unrestricted_type.md) * Programmer guide: * [Adding a new sink](./adding_new_sink.md) * [Debugging](./debugging.md) From a997c26ea99ea0b19f7f0b831aaaa3c9d03135ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferm=C3=ADn=20Gal=C3=A1n=20M=C3=A1rquez?= Date: Thu, 3 Feb 2022 14:07:26 +0100 Subject: [PATCH 19/22] ADD entry to ToC in .md --- doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md index 6d53d7259..934870a40 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md @@ -7,6 +7,7 @@ Content: * [Administration guide](#section2) * [Configuration](#section2.1) * [Accepted character set](#section2.2) + * [Quote escaping for `TextUnrestricted` attributes](#section2.3) * [Programmers guide](#section3) * [`NGSIRestHandler` class](#section3.1) From e0a8b63466168ffb1d93f4b18c6172eea64d436d Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 14:42:49 +0100 Subject: [PATCH 20/22] Update CHANGES_NEXT_RELEASE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- CHANGES_NEXT_RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE index 0e1c3971c..1193e6623 100644 --- a/CHANGES_NEXT_RELEASE +++ b/CHANGES_NEXT_RELEASE @@ -1,3 +1,3 @@ -- [cygnus-commons] Check TextUnrestricted type to escape character ' as '' (#2125) +- [cygnus-commons][NGSIHandler] Check TextUnrestricted type to escape character ' as '' (#2125) - [cygnus-ngsi][KafkaSink] Upgrade libthrift dependency from 0.12.0 to 0.14.1 due to github vulnerability report - [cygnus-commons] Upgrade postgresql dependency from 42.2.22 to 42.2.25 due to github vulnerability report From 626fd78dcd2b6d8a08f52f620e1fcb8f2fc8ab03 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 14:43:01 +0100 Subject: [PATCH 21/22] Update doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Fermín Galán Márquez --- doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md index 934870a40..bf33f20da 100644 --- a/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md +++ b/doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md @@ -143,7 +143,7 @@ It is expected UTF-8 character set is maintained by all the Flume elements in th ### Quote escaping for `TextUnrestricted` attributes Cygnus escapes from `'` to `''` in attributes of type `TextUnrestricted` with the aim of avoiding injection attacks. -Note that other attributes (i.e. with type different to `TextUnrestricted` don't need such escaping as single quote (`'`) is a [forbidden character in Context Broker](https://fiware-orion.readthedocs.io/en/master/user/forbidden_characters/index.html) so that value will never arrives to Cygnus in notifications. +Note that other attributes (i.e. with type different to `TextUnrestricted`) don't need such escaping as single quote (`'`) is a [forbidden character in Context Broker](https://fiware-orion.readthedocs.io/en/master/user/forbidden_characters/index.html) so that value will never arrives to Cygnus in notifications. [Top](#top) From 08534b0ac8ffeb8a516db54c421ead358f0d3225 Mon Sep 17 00:00:00 2001 From: Alvaro Vega Date: Thu, 3 Feb 2022 14:51:36 +0100 Subject: [PATCH 22/22] Update NGSIGenericAggregator.java --- .../telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java index 9ea70f7ac..079bea030 100644 --- a/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java +++ b/cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java @@ -771,6 +771,7 @@ public String getEscapedString(JsonElement value, String quotationMark) { escaped = escaped.replaceAll("'", "''"); break; case "\"": + // Currently not used but maybe in the future could be useful escaped = escaped.replaceAll("\"", "\"\""); break; }