Skip to content

Commit

Permalink
Merge pull request #2128 from telefonicaid/task/handle_textunrestrict…
Browse files Browse the repository at this point in the history 
…ed_attr2

Task/handle textunrestricted attr
  • Loading branch information
@fgalan
fgalan authored Feb 3, 2022
2 parents f686eec + 08534b0 commit 728889a
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 48 deletions.
1 change: 1 addition & 0 deletions CHANGES_NEXT_RELEASE
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
- [cygnus-commons][NGSIHandler] Check TextUnrestricted type to escape character ' as '' (#2125)
- [cygnus-ngsi][KafkaSink] Upgrade libthrift dependency from 0.12.0 to 0.14.1 due to github vulnerability report
- [cygnus-commons] Upgrade postgresql dependency from 42.2.22 to 42.2.25 due to github vulnerability report
21 changes: 21 additions & 0 deletions cygnus-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericAggregator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -757,6 +757,27 @@ public void setEnableNameMappings(boolean enableNameMappings) {
this.enableNameMappings = enableNameMappings;
} //setEnableNameMappings

/**
* getEscapedString
*
* @param JsonElement an UnrestrictedString
* @param String a quotationMark to escape
* @return the escaped string
*/
public String getEscapedString(JsonElement value, String quotationMark) {
String escaped = value.toString();
switch (quotationMark) {
case "'":
escaped = escaped.replaceAll("'", "''");
break;
case "\"":
// Currently not used but maybe in the future could be useful
escaped = escaped.replaceAll("\"", "\"\"");
break;
}
return escaped;
}

/**
* Aggregate declaration for child classes.
*
Expand Down
2 changes: 2 additions & 0 deletions ...ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericColumnAggregator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,8 @@ public void aggregate(NGSIEvent event) {
} catch (Exception e) {
LOGGER.error("[" + getName() + "] Processing context attribute (name=" + attrValue.toString());
}
} else if (attrType.equals("TextUnrestricted")) {
attrValue = jsonParser.parse(getEscapedString(attrValue, "'"));
}
// Check if the attribute already exists in the form of 2 columns (one for metadata); if not existing,
// add an empty value for all previous rows
Expand Down
3 changes: 3 additions & 0 deletions ...us-ngsi/src/main/java/com/telefonica/iot/cygnus/aggregation/NGSIGenericRowAggregator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ public void aggregate(NGSIEvent event) {
JsonArray jsonAttrMetadata = (JsonArray) jsonParser.parse(attrMetadata);
LOGGER.debug("[" + getName() + "] Processing context attribute (name=" + attrName + ", type="
+ attrType + ")");
if (attrType.equals("TextUnrestricted")) {
attrValue = jsonParser.parse(getEscapedString(attrValue, "'"));
}
// aggregate the attribute information
aggregation.get(NGSIConstants.RECV_TIME_TS).add(new JsonPrimitive(Long.toString(recvTimeTs)));
aggregation.get(NGSIConstants.RECV_TIME).add(new JsonPrimitive(recvTime));
Expand Down
8 changes: 4 additions & 4 deletions cygnus-ngsi/src/test/java/com/telefonica/iot/cygnus/sinks/NGSIMySQLSinkTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1287,8 +1287,8 @@ private NotifyContextRequest.ContextElement createContextElement() {
contextAttribute1.setContextMetadata(metadata);
NotifyContextRequest.ContextAttribute contextAttribute2 = new NotifyContextRequest.ContextAttribute();
contextAttribute2.setName("someName2");
contextAttribute2.setType("someType2");
contextAttribute2.setContextValue(new JsonPrimitive("someValue2"));
contextAttribute2.setType("TextUnrestricted");
contextAttribute2.setContextValue(new JsonPrimitive("someValue'2"));
contextAttribute2.setContextMetadata(null);
ArrayList<NotifyContextRequest.ContextAttribute> attributes = new ArrayList<>();
attributes.add(contextAttribute1);
Expand Down Expand Up @@ -1352,7 +1352,7 @@ public void testNativeTypeColumnBatch() throws CygnusBadConfiguration, CygnusRun
for (NGSIEvent event : events) {
aggregator.aggregate(event);
}
String correctBatch = "('2016-04-20 07:19:55.801','somePath','someId','someType',2,'[]',TRUE,'[]','2016-09-21T01:23:00.00Z','[]','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]','{\"String\": \"string\"}','[]','foo','[]','','[]',NULL,NULL,NULL,NULL),('2016-04-20 07:19:55.801','somePath','someId','someType',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]','someValue2','[]')";
String correctBatch = "('2016-04-20 07:19:55.801','somePath','someId','someType',2,'[]',TRUE,'[]','2016-09-21T01:23:00.00Z','[]','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]','{\"String\": \"string\"}','[]','foo','[]','','[]',NULL,NULL,NULL,NULL),('2016-04-20 07:19:55.801','somePath','someId','someType',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]','someValue''2','[]')";
String valuesForInsert = SQLQueryUtils.getValuesForInsert(aggregator.getAggregationToPersist(), aggregator.isAttrNativeTypes());
if (valuesForInsert.equals(correctBatch)) {
System.out.println(getTestTraceHead("[NGSIMySQKSink.testNativeTypesColumnBatch]")
Expand Down Expand Up @@ -1397,7 +1397,7 @@ public void testNativeTypeRowBatch() throws CygnusBadConfiguration, CygnusRuntim
for (NGSIEvent event : events) {
aggregator.aggregate(event);
} // for
String correctBatch = "('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someNumber','number','2','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','somneBoolean','Boolean','true','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someDate','DateTime','2016-09-21T01:23:00.00Z','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someGeoJson','geo:json','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someJson','json','{\"String\": \"string\"}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString','string','foo','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString2','string','','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName1','someType1','-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName2','someType2','someValue2','[]')";
String correctBatch = "('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someNumber','number','2','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','somneBoolean','Boolean','true','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someDate','DateTime','2016-09-21T01:23:00.00Z','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someGeoJson','geo:json','{\"type\": \"Point\",\"coordinates\": [-0.036177,39.986159]}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someJson','json','{\"String\": \"string\"}','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString','string','foo','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someString2','string','','[]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName1','someType1','-3.7167, 40.3833','[{\"name\":\"location\",\"type\":\"string\",\"value\":\"WGS84\"}]'),('1461136795801','2016-04-20 07:19:55.801','somePath','someId','someType','someName2','TextUnrestricted','someValue''2','[]')";
String valuesForInsert = SQLQueryUtils.getValuesForInsert(aggregator.getAggregationToPersist(), aggregator.isAttrNativeTypes());
if (valuesForInsert.equals(correctBatch)) {
System.out.println(getTestTraceHead("[NGSIMySQKSink.testNativeTypesRowBatch]")
Expand Down
96 changes: 52 additions & 44 deletions doc/cygnus-ngsi/flume_extensions_catalogue/ngsi_rest_handler.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ Content:
* [Administration guide](#section2)
* [Configuration](#section2.1)
* [Accepted character set](#section2.2)
* [Quote escaping for `TextUnrestricted` attributes](#section2.3)
* [Programmers guide](#section3)
* [`NGSIRestHandler` class](#section3.1)

Expand Down Expand Up @@ -43,26 +44,26 @@ Let's assume the following not-intercepted event regarding a received notificati
```
notification={
headers={
fiware-service=hotel1,
fiware-servicepath=/other,/suites,
correlation-id=1234567890-0000-1234567890
fiware-service=hotel1,
fiware-servicepath=/other,/suites,
correlation-id=1234567890-0000-1234567890
},
body={
{
entityId=suite.12,
entityType=room,
attributes=[
...
]
},
{
entityId=other.9,
entityType=room,
attributes=[
...
]
}
}
entityId=suite.12,
entityType=room,
attributes=[
...
]
},
{
entityId=other.9,
entityType=room,
attributes=[
...
]
}
}
}
```

Expand All @@ -72,40 +73,40 @@ As can be seen, two entities (`suite.12` and `other.9`) of the same type (`room`
```
ngsi-event-1={
headers={
fiware-service=hotel,
fiware-servicepath=/suites,
transaction-id=1234567890-0000-1234567890,
correlation-id=1234567890-0000-1234567890,
timestamp=1234567890,
mapped-fiware-service=hotel
mapped-fiware-service-path=/suites
},
fiware-service=hotel,
fiware-servicepath=/suites,
transaction-id=1234567890-0000-1234567890,
correlation-id=1234567890-0000-1234567890,
timestamp=1234567890,
mapped-fiware-service=hotel
mapped-fiware-service-path=/suites
},
original-context-element={
entityId=suite.12,
entityType=room,
attributes=[
...
]
}
entityId=suite.12,
entityType=room,
attributes=[
...
]
}
}
ngsi-event-2={
headers={
fiware-service=hotel,
fiware-servicepath=/other,
transaction-id=1234567890-0000-1234567890,
correlation-id=1234567890-0000-1234567890,
timestamp=1234567890,
mapped-fiware-service=hotel
mapped-fiware-service-path=/other
fiware-service=hotel,
fiware-servicepath=/other,
transaction-id=1234567890-0000-1234567890,
correlation-id=1234567890-0000-1234567890,
timestamp=1234567890,
mapped-fiware-service=hotel
mapped-fiware-service-path=/other
},
original-context-element={
entityId=other.9,
entityType=room,
attributes=[
...
]
}
entityId=other.9,
entityType=room,
attributes=[
...
]
}
}
```

Expand Down Expand Up @@ -139,6 +140,13 @@ It is expected UTF-8 character set is maintained by all the Flume elements in th

[Top](#top)

### <a name="section2.3"></a>Quote escaping for `TextUnrestricted` attributes
Cygnus escapes from `'` to `''` in attributes of type `TextUnrestricted` with the aim of avoiding injection attacks.

Note that other attributes (i.e. with type different to `TextUnrestricted`) don't need such escaping as single quote (`'`) is a [forbidden character in Context Broker](https://fiware-orion.readthedocs.io/en/master/user/forbidden_characters/index.html) so that value will never arrives to Cygnus in notifications.

[Top](#top)

## <a name="section3"></a>Programmers guide
### <a name="section3.1"></a>`NGSIRestHandler` class
TBD
Expand Down

0 comments on commit 728889a

Please sign in to comment.