Skip to content

Commit

Permalink
fix: reduce ws meta tracking memory consumption
Browse files Browse the repository at this point in the history
Fixes #1717.

Reduce memory consumption of string values used in tracking
what metadata we have sent for a ws connection by keeping
a cache of shared string values for this. This will deduplicate
the values and help with memory consumption with a large
number of AIS targets and multiple ws connections.

The cache is purged every 30 minutes to get rid of data
for AIS targets that have disappeared.
  • Loading branch information
tkurki committed Apr 14, 2024
1 parent 89baf2f commit 213cec1
Showing 1 changed file with 25 additions and 4 deletions.
29 changes: 25 additions & 4 deletions src/interfaces/ws.js
Original file line number Diff line number Diff line change
Expand Up @@ -505,18 +505,39 @@ function processUpdates(app, pathSources, spark, msg) {
})
}

/*
Keep a list of shared context-path strings that is shared across ws connections.
This way the string values are shared across ws connections and not recreated
for each context-path-ws combination. This reduces memory consumption for
multiple ws clients.
Nevertheless we need to purge this data eventually, otherwise the strings for
AIS targets will stay forever, so implement a simple total purge. This may cause
some thrashing, but is better than not sharing the values.
*/
let canonical_meta_contextpath_values = {}
const getContextPathMetaKey = (context, path ) => {
const contextPaths = canonical_meta_contextpath_values[context] || (canonical_meta_contextpath_values[context] = {})
const result = contextPaths[path] || (contextPaths[path] = `${context}.${path}`)
return result
}
setInterval(() => {
canonical_meta_contextpath_values = {}
}, 30 * 60 * 1000)

function handleValuesMeta(kp) {
if (kp.path && !this.spark.sentMetaData[this.context + '.' + kp.path]) {
const fullContextPathKey = getContextPathMetaKey(this.context, kp.path)
if (kp.path && !this.spark.sentMetaData[fullContextPathKey]) {
const split = kp.path.split('.')
for (let i = split.length; i > 1; i--) {
const path = split.slice(0, i).join('.')
if (this.spark.sentMetaData[this.context + '.' + path]) {
const partialContextPathKey = getContextPathMetaKey(this.context, path)
if (this.spark.sentMetaData[partialContextPathKey]) {
//stop backing up the path with first prefix that has already been handled
break
} else {
//always set to true, even if there is no meta for the path
this.spark.sentMetaData[this.context + '.' + path] = true
let meta = getMetadata(this.context + '.' + path)
this.spark.sentMetaData[partialContextPathKey] = true
let meta = getMetadata(partialContextPathKey)
if (meta) {
this.spark.write({
context: this.context,
Expand Down

0 comments on commit 213cec1

Please sign in to comment.