From ebd3d67a67c31d071a43370680f783b932f8115d Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Thu, 21 Mar 2019 17:28:14 -0700 Subject: [PATCH 01/34] Issue #105 - Send packet as JSON to frontend - Update tlm/realtime route to send packet to front end in JSON format - Update TelemetryStream on frontend to parse JSON formatted packet - Update plotting on frontend to work with JSON formatted packet - Update getting field values from packet on frontend to work with JSON --- ait/gui/__init__.py | 15 ++++++++++++++- ait/gui/static/js/ait/gui/Field.js | 2 +- ait/gui/static/js/ait/gui/Plot.js | 19 ++++++++++--------- ait/gui/static/js/ait/tlm.js | 18 +++++++++--------- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index e8080f02..2cfb85bf 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -696,10 +696,23 @@ def handle(): bottle.abort(400, 'Expected WebSocket request.') try: + tlmdict = ait.core.tlm.getDefaultDict() while not wsock.closed: try: uid, data = session.telemetry.popleft(timeout=30) - wsock.send(pad + struct.pack('>I', uid) + data) + pkt_defn = None + for k, v in tlmdict.iteritems(): + if v.uid == uid: + pkt_defn = v + break + else: + continue + + wsock.send(json.dumps({ + 'packet': pkt_defn.name, + 'data': ait.core.tlm.Packet(pkt_defn, data=data).toJSON() + })) + except IndexError: # If no telemetry has been received by the GUI # server after timeout seconds, "probe" the client diff --git a/ait/gui/static/js/ait/gui/Field.js b/ait/gui/static/js/ait/gui/Field.js index dac53ae6..41af65e8 100644 --- a/ait/gui/static/js/ait/gui/Field.js +++ b/ait/gui/static/js/ait/gui/Field.js @@ -83,7 +83,7 @@ const Field = * retrieving the packet value. */ getValue (packet, raw=false) { - return packet && packet.__get__(this._fname, raw) + return packet && packet[this._fname] }, diff --git a/ait/gui/static/js/ait/gui/Plot.js b/ait/gui/static/js/ait/gui/Plot.js index de12e9c5..8695182b 100644 --- a/ait/gui/static/js/ait/gui/Plot.js +++ b/ait/gui/static/js/ait/gui/Plot.js @@ -78,13 +78,14 @@ class DygraphsBackend } } - plot (packet) { - const pname = packet._defn.name + plot (data) { + const pname = data['packet'] + let delta = data['data'] const names = this._plot._packets[pname] if (!names) return - let row = [ this._plot._time.get(packet) ] + let row = [ this._plot._time.get(delta) ] // For each series of data, if it's in the current packet // that we're updating, add the associated point. Otherwise, @@ -92,7 +93,7 @@ class DygraphsBackend // to the plot maintains the same "shape" as the labels. this._series.forEach((id) => { if (id.startsWith(pname)) { - row.push(packet.__get__(id.split('.')[1])) + row.push(delta[id.split('.')[1]]) } else { row.push(null) } @@ -207,8 +208,8 @@ class HighchartsBackend Object.assign(options, overrides) } - plot(packet) { - const pname = packet._defn.name + plot(delta) { + const pname = delta['packet'] const names = this._plot._packets[pname] if (!names) return @@ -354,10 +355,10 @@ class HighchartsBackend const Plot = { /** - * Plots data from the given packet. + * Plots data from the given delta. */ - plot (packet) { - this._backend.plot(packet) + plot (delta) { + this._backend.plot(delta) }, diff --git a/ait/gui/static/js/ait/tlm.js b/ait/gui/static/js/ait/tlm.js index 07ce9410..168ceb60 100644 --- a/ait/gui/static/js/ait/tlm.js +++ b/ait/gui/static/js/ait/tlm.js @@ -381,11 +381,12 @@ class TelemetryStream onMessage (event) { - if ( !(event.data instanceof ArrayBuffer) ) return + if ( !(typeof event.data == "string") ) return - let uid = new DataView(event.data, 1, 4).getUint32(0) - let data = new DataView(event.data, 5) - let defn = this._dict[uid] + let now = Date.now() + let data = JSON.parse(event.data) + let packet_name = data['packet'] + let delta = data['data'] // Since WebSockets can stay open indefinitely, the AIT GUI // server will occasionally probe for dropped client @@ -399,16 +400,15 @@ class TelemetryStream // It's also possible that the packet UID is not in the client // telemetry dictionary (defn === undefined). Without a // packet definition, the packet cannot be processed further. - if ((uid == 0 && data.byteLength == 0) || !defn) return - - let packet = Packet.create(defn, data) clearInterval(this._interval) this._stale = 0 this._interval = setInterval(this.onStale.bind(this), 5000) - ait.packets.insert(defn.name, packet) - this._emit('packet', packet) + console.log(packet_name) + console.log(delta) + ait.packets.insert(packet_name, delta) + this._emit('packet', data) } From 9f4836191143d145e61f862b0a6c741cc6efa058 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Fri, 22 Mar 2019 11:30:22 -0700 Subject: [PATCH 02/34] Issue #105 - Update highcharts plot function for JSON formatted packet --- ait/gui/__init__.py | 4 ---- ait/gui/static/js/ait/gui/Plot.js | 9 +++++---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index 2cfb85bf..0ba9dc3d 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -660,8 +660,6 @@ def handle(): if v.uid == uid: pkt_defn = v break - else: - continue wsock.send(json.dumps({ 'packet': pkt_defn.name, @@ -705,8 +703,6 @@ def handle(): if v.uid == uid: pkt_defn = v break - else: - continue wsock.send(json.dumps({ 'packet': pkt_defn.name, diff --git a/ait/gui/static/js/ait/gui/Plot.js b/ait/gui/static/js/ait/gui/Plot.js index 8695182b..2789d478 100644 --- a/ait/gui/static/js/ait/gui/Plot.js +++ b/ait/gui/static/js/ait/gui/Plot.js @@ -208,8 +208,9 @@ class HighchartsBackend Object.assign(options, overrides) } - plot(delta) { - const pname = delta['packet'] + plot(data) { + const pname = data['packet'] + let delta = data['data'] const names = this._plot._packets[pname] if (!names) return @@ -217,8 +218,8 @@ class HighchartsBackend const series = this._plot._chart.get(pname + '.' + name) if (series) { - const x = this._plot._time.get(packet).getTime() - const y = packet.__get__(name) + const x = this._plot._time.get(delta).getTime() + const y = delta[name] series.addPoint([x, y], false) From 1f930d41b6afc8674c33c807e8e894d328c76f8b Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Fri, 22 Mar 2019 13:55:35 -0700 Subject: [PATCH 03/34] Issue #105 - Store last packet on backend; send only changes to front end --- ait/gui/__init__.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index 0ba9dc3d..be35717a 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -682,6 +682,33 @@ def handle(): pass +last_packets = { } +def get_packet_delta(pkt_defn, packet): + """ + Keeps track of last packets recieved of all types recieved + and checks returns only fields that have changed since last + packet. + + Params: + pkt_defn: Packet definition + packet: Packet in JSON format + Returns: + delta: JSON of packet fields that have changed + """ + if pkt_defn.name not in last_packets: + last_packets[pkt_defn.name] = packet + delta = packet + else: + delta = { } + for field, new_value in packet.items(): + last_value = last_packets[pkt_defn.name][field] + if new_value != last_value: + delta[field] = new_value + last_packets[pkt_defn.name][field] = new_value + + return delta + + @App.route('/tlm/realtime') def handle(): """Return telemetry packets in realtime to client""" @@ -704,9 +731,12 @@ def handle(): pkt_defn = v break + json_pkt = ait.core.tlm.Packet(pkt_defn, data=data).toJSON() + delta = get_packet_delta(pkt_defn, json_pkt) + wsock.send(json.dumps({ 'packet': pkt_defn.name, - 'data': ait.core.tlm.Packet(pkt_defn, data=data).toJSON() + 'data': delta })) except IndexError: From c5887fb618b21b4c54de0604c1d8f5bdfcd9906e Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Wed, 18 Sep 2019 11:08:43 -0700 Subject: [PATCH 04/34] Add delta to current packet state before emitting event (#105) --- ait/gui/static/js/ait/gui/Plot.js | 18 +++++++++--------- ait/gui/static/js/ait/tlm.js | 20 ++++++++++++++++---- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/ait/gui/static/js/ait/gui/Plot.js b/ait/gui/static/js/ait/gui/Plot.js index 2789d478..a41a9b10 100644 --- a/ait/gui/static/js/ait/gui/Plot.js +++ b/ait/gui/static/js/ait/gui/Plot.js @@ -80,12 +80,12 @@ class DygraphsBackend plot (data) { const pname = data['packet'] - let delta = data['data'] + let packet = data['data'] const names = this._plot._packets[pname] if (!names) return - let row = [ this._plot._time.get(delta) ] + let row = [ this._plot._time.get(packet) ] // For each series of data, if it's in the current packet // that we're updating, add the associated point. Otherwise, @@ -93,7 +93,7 @@ class DygraphsBackend // to the plot maintains the same "shape" as the labels. this._series.forEach((id) => { if (id.startsWith(pname)) { - row.push(delta[id.split('.')[1]]) + row.push(packet[id.split('.')[1]]) } else { row.push(null) } @@ -210,7 +210,7 @@ class HighchartsBackend plot(data) { const pname = data['packet'] - let delta = data['data'] + let packet = data['data'] const names = this._plot._packets[pname] if (!names) return @@ -218,8 +218,8 @@ class HighchartsBackend const series = this._plot._chart.get(pname + '.' + name) if (series) { - const x = this._plot._time.get(delta).getTime() - const y = delta[name] + const x = this._plot._time.get(packet).getTime() + const y = packet[name] series.addPoint([x, y], false) @@ -356,10 +356,10 @@ class HighchartsBackend const Plot = { /** - * Plots data from the given delta. + * Plots data from the given packet. */ - plot (delta) { - this._backend.plot(delta) + plot (packet) { + this._backend.plot(packet) }, diff --git a/ait/gui/static/js/ait/tlm.js b/ait/gui/static/js/ait/tlm.js index 168ceb60..e750f0bb 100644 --- a/ait/gui/static/js/ait/tlm.js +++ b/ait/gui/static/js/ait/tlm.js @@ -356,6 +356,7 @@ class TelemetryStream this._socket = new WebSocket(url) this._stale = 0 this._url = url + this._pkt_states = { } // Re-map telemetry dictionary to be keyed by a PacketDefinition // 'id' instead of 'name'. @@ -388,6 +389,18 @@ class TelemetryStream let packet_name = data['packet'] let delta = data['data'] + // add delta to last full packet + // want full packet inserted in packet buffer & emitted as event + if ( Object.keys(delta).length !== 0 ) { + if ( packet_name in this._pkt_states ) { + for ( var field in delta ) { + this._pkt_states[packet_name][field] = this._pkt_states[packet_name][field] + delta[field] + } + } else { + this._pkt_states[packet_name] = delta + } + } + // Since WebSockets can stay open indefinitely, the AIT GUI // server will occasionally probe for dropped client // connections by sending empty packets (data.byteLength == 0) @@ -405,10 +418,9 @@ class TelemetryStream this._stale = 0 this._interval = setInterval(this.onStale.bind(this), 5000) - console.log(packet_name) - console.log(delta) - ait.packets.insert(packet_name, delta) - this._emit('packet', data) + ait.packets.insert(packet_name, this._pkt_states[packet_name]) + this._emit('packet', {'packet': packet_name, + 'data': this._pkt_states[packet_name]}) } From df28e936623e5c98fb1dd112df12bd315f61b686 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Wed, 2 Oct 2019 10:57:40 -0700 Subject: [PATCH 05/34] Send new dntoeu values from backend to frontend (#105) --- ait/gui/__init__.py | 48 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index be35717a..7f09dcad 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -682,31 +682,57 @@ def handle(): pass +def add_dntoeu_value(field, packet, dntoeus): + """ + Returns dntoeu value of field from packet. + + Params: + field: str name of field to evaluate + packet: AIT Packet + dntoeus: dict of dntoeu values to add to + """ + field_defn = packet._defn.fieldmap[field] + if field_defn.dntoeu: + dntoeus[field] = field_defn.dntoeu.eval(packet) + + return dntoeus + + last_packets = { } def get_packet_delta(pkt_defn, packet): """ Keeps track of last packets recieved of all types recieved - and checks returns only fields that have changed since last - packet. + and returns only fields that have changed since last packet. Params: pkt_defn: Packet definition - packet: Packet in JSON format + packet: Binary packet data Returns: delta: JSON of packet fields that have changed """ + ait_pkt = ait.core.tlm.Packet(pkt_defn, data=packet) + json_pkt = ait_pkt.toJSON() + + # first packet of this type if pkt_defn.name not in last_packets: - last_packets[pkt_defn.name] = packet - delta = packet + last_packets[pkt_defn.name] = json_pkt + delta = json_pkt + + dntoeus = {} + for field, value in json_pkt.items(): + dntoeus = add_dntoeu_value(field, ait_pkt, dntoeus) + + # previous packets of this type received else: - delta = { } - for field, new_value in packet.items(): + delta, dntoeus = {}, {} + for field, new_value in json_pkt.items(): last_value = last_packets[pkt_defn.name][field] if new_value != last_value: delta[field] = new_value last_packets[pkt_defn.name][field] = new_value + dntoeus = add_dntoeu_value(field, ait_pkt, dntoeus) - return delta + return delta, dntoeus @App.route('/tlm/realtime') @@ -731,12 +757,12 @@ def handle(): pkt_defn = v break - json_pkt = ait.core.tlm.Packet(pkt_defn, data=data).toJSON() - delta = get_packet_delta(pkt_defn, json_pkt) + delta, dntoeus = get_packet_delta(pkt_defn, data) wsock.send(json.dumps({ 'packet': pkt_defn.name, - 'data': delta + 'data': delta, + 'dntoeus': dntoeus })) except IndexError: From 8841588c87bfe9dd07e8f90976c4c624874ad2e6 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Wed, 2 Oct 2019 17:10:49 -0700 Subject: [PATCH 06/34] Request full packet from backend on page refresh --- ait/gui/__init__.py | 24 ++++++++++++++++++++++++ ait/gui/static/js/ait/gui/Plot.js | 2 +- ait/gui/static/js/ait/tlm.js | 24 ++++++++++++++++++++---- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index 7f09dcad..5b5fd27c 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -781,6 +781,30 @@ def handle(): pass +@App.route('/tlm/latest', method='GET') +def handle(): + """Return latest telemetry packet to client""" + with Sessions.current() as session: + tlmdict = ait.core.tlm.getDefaultDict() + uid, data = session.telemetry.popleft(timeout=30) + pkt_defn = None + for k, v in tlmdict.iteritems(): + if v.uid == uid: + pkt_defn = v + break + + ait_pkt = ait.core.tlm.Packet(pkt_defn, data=data) + json_pkt = ait_pkt.toJSON() + for field, value in json_pkt.items(): + dntoeus = add_dntoeu_value(field, ait_pkt, {}) + + return json.dumps({ + 'packet': pkt_defn.name, + 'data': json_pkt, + 'dntoeus': dntoeus + }) + + @App.route('/tlm/query', method='POST') def handle(): """""" diff --git a/ait/gui/static/js/ait/gui/Plot.js b/ait/gui/static/js/ait/gui/Plot.js index a41a9b10..2464fceb 100644 --- a/ait/gui/static/js/ait/gui/Plot.js +++ b/ait/gui/static/js/ait/gui/Plot.js @@ -86,7 +86,7 @@ class DygraphsBackend if (!names) return let row = [ this._plot._time.get(packet) ] - + console.log(packet) // For each series of data, if it's in the current packet // that we're updating, add the associated point. Otherwise, // add a null value. Dygraphs requires that the data added diff --git a/ait/gui/static/js/ait/tlm.js b/ait/gui/static/js/ait/tlm.js index e750f0bb..fc68967a 100644 --- a/ait/gui/static/js/ait/tlm.js +++ b/ait/gui/static/js/ait/tlm.js @@ -388,17 +388,33 @@ class TelemetryStream let data = JSON.parse(event.data) let packet_name = data['packet'] let delta = data['data'] + let dntoeus = data['dntoeus'] + + console.log(this._pkt_states) // add delta to last full packet // want full packet inserted in packet buffer & emitted as event - if ( Object.keys(delta).length !== 0 ) { - if ( packet_name in this._pkt_states ) { + if ( packet_name in this._pkt_states ) { + if ( Object.keys(delta).length !== 0 ) { for ( var field in delta ) { this._pkt_states[packet_name][field] = this._pkt_states[packet_name][field] + delta[field] } - } else { - this._pkt_states[packet_name] = delta } + } else { + console.log('name not in states') + // delta is empty - request full packet from backend + if ( Object.keys(delta).length == 0 ) { + m.request({ url: '/tlm/latest' }).then( (latest) => { + packet_name = latest['packet'] + delta = latest['data'] + dntoeus = latest['dntoeus'] + console.log('here') + console.log(delta) + }) + console.log(delta) + } + + this._pkt_states[packet_name] = delta } // Since WebSockets can stay open indefinitely, the AIT GUI From 96d223d487a527f150ee2d1dd75b4f3d853b1ec8 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Tue, 8 Oct 2019 16:37:47 -0700 Subject: [PATCH 07/34] Update backend endpoint for returning latest packet states (#105) --- ait/gui/__init__.py | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index 5b5fd27c..276424d8 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -640,6 +640,7 @@ def handle(): __setResponseToEventStream() yield 'data: %s\n\n' % json.dumps(msg) + @App.route('/tlm/realtime/openmct') def handle(): """Return telemetry packets in realtime to client""" @@ -698,7 +699,7 @@ def add_dntoeu_value(field, packet, dntoeus): return dntoeus -last_packets = { } +packet_states = { } def get_packet_delta(pkt_defn, packet): """ Keeps track of last packets recieved of all types recieved @@ -714,8 +715,8 @@ def get_packet_delta(pkt_defn, packet): json_pkt = ait_pkt.toJSON() # first packet of this type - if pkt_defn.name not in last_packets: - last_packets[pkt_defn.name] = json_pkt + if pkt_defn.name not in packet_states: + packet_states[pkt_defn.name] = json_pkt delta = json_pkt dntoeus = {} @@ -726,10 +727,10 @@ def get_packet_delta(pkt_defn, packet): else: delta, dntoeus = {}, {} for field, new_value in json_pkt.items(): - last_value = last_packets[pkt_defn.name][field] + last_value = packet_states[pkt_defn.name][field] if new_value != last_value: delta[field] = new_value - last_packets[pkt_defn.name][field] = new_value + packet_states[pkt_defn.name][field] = new_value dntoeus = add_dntoeu_value(field, ait_pkt, dntoeus) return delta, dntoeus @@ -784,25 +785,7 @@ def handle(): @App.route('/tlm/latest', method='GET') def handle(): """Return latest telemetry packet to client""" - with Sessions.current() as session: - tlmdict = ait.core.tlm.getDefaultDict() - uid, data = session.telemetry.popleft(timeout=30) - pkt_defn = None - for k, v in tlmdict.iteritems(): - if v.uid == uid: - pkt_defn = v - break - - ait_pkt = ait.core.tlm.Packet(pkt_defn, data=data) - json_pkt = ait_pkt.toJSON() - for field, value in json_pkt.items(): - dntoeus = add_dntoeu_value(field, ait_pkt, {}) - - return json.dumps({ - 'packet': pkt_defn.name, - 'data': json_pkt, - 'dntoeus': dntoeus - }) + return json.dumps(packet_states) @App.route('/tlm/query', method='POST') From 7740371a9bf1d626a3b807d9e3d06d17033b1cfb Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Tue, 8 Oct 2019 16:38:47 -0700 Subject: [PATCH 08/34] Get latest packet state from backend on frontend initialization (#105) --- ait/gui/static/js/ait/gui/Plot.js | 1 + ait/gui/static/js/ait/gui/index.js | 6 ++++++ ait/gui/static/js/ait/tlm.js | 20 +++++++++++++------- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/ait/gui/static/js/ait/gui/Plot.js b/ait/gui/static/js/ait/gui/Plot.js index 2464fceb..f3bd9cc8 100644 --- a/ait/gui/static/js/ait/gui/Plot.js +++ b/ait/gui/static/js/ait/gui/Plot.js @@ -86,6 +86,7 @@ class DygraphsBackend if (!names) return let row = [ this._plot._time.get(packet) ] + console.log("Received by plotting:") console.log(packet) // For each series of data, if it's in the current packet // that we're updating, add the associated point. Otherwise, diff --git a/ait/gui/static/js/ait/gui/index.js b/ait/gui/static/js/ait/gui/index.js index 10279102..15360bc2 100644 --- a/ait/gui/static/js/ait/gui/index.js +++ b/ait/gui/static/js/ait/gui/index.js @@ -197,6 +197,12 @@ function init () { ait.events.emit(e.name, e.data) }); + m.request({ url: '/tlm/latest' }).then((latest) => { + ait.tlm.state = latest + console.log('requested latest tlm') + console.log(ait.tlm.state) + }) + m.mount(root, { view: () => createMithrilNodes(elems) }) }) } diff --git a/ait/gui/static/js/ait/tlm.js b/ait/gui/static/js/ait/tlm.js index fc68967a..28f588d4 100644 --- a/ait/gui/static/js/ait/tlm.js +++ b/ait/gui/static/js/ait/tlm.js @@ -356,7 +356,12 @@ class TelemetryStream this._socket = new WebSocket(url) this._stale = 0 this._url = url - this._pkt_states = { } + m.request({ url: '/tlm/latest' }).then((latest) => { + this._pkt_states = latest + console.log('requested latest tlm from within stream') + console.log(ait.tlm.state) + }) + console.log(this._pkt_states) // Re-map telemetry dictionary to be keyed by a PacketDefinition // 'id' instead of 'name'. @@ -390,6 +395,7 @@ class TelemetryStream let delta = data['data'] let dntoeus = data['dntoeus'] + console.log('packet states:') console.log(this._pkt_states) // add delta to last full packet @@ -405,13 +411,13 @@ class TelemetryStream // delta is empty - request full packet from backend if ( Object.keys(delta).length == 0 ) { m.request({ url: '/tlm/latest' }).then( (latest) => { - packet_name = latest['packet'] - delta = latest['data'] - dntoeus = latest['dntoeus'] - console.log('here') - console.log(delta) + console.log('requesting latest tlm') + console.log(latest) + //ait.tlm.state = latest + //packet_name = latest['packet'] + //delta = latest['data'] + //dntoeus = latest['dntoeus'] }) - console.log(delta) } this._pkt_states[packet_name] = delta From ab010e28949c9b63b6cc5b8376716bfd2c0f7121 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Wed, 9 Oct 2019 09:22:50 -0700 Subject: [PATCH 09/34] Encode datetimes as ISO formatted strings for sending to frontend (#105) --- ait/gui/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index 276424d8..e058814a 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -736,6 +736,16 @@ def get_packet_delta(pkt_defn, packet): return delta, dntoeus +def replace_datetimes(delta): + """Replace datetime objects with ISO formatted + strings for JSON serialization""" + for key, val in delta.items(): + if type(val) is datetime: + delta[key] = val.isoformat() + + return delta + + @App.route('/tlm/realtime') def handle(): """Return telemetry packets in realtime to client""" @@ -759,6 +769,8 @@ def handle(): break delta, dntoeus = get_packet_delta(pkt_defn, data) + delta = replace_datetimes(delta) + log.info(delta) wsock.send(json.dumps({ 'packet': pkt_defn.name, From 62b97e41e93a5bba21e68d56c26f5fd1c7002e56 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Wed, 9 Oct 2019 11:46:53 -0700 Subject: [PATCH 10/34] Implement counter method for checking packet delta order on frontend (#105) --- ait/gui/__init__.py | 16 ++++++--- ait/gui/static/js/ait/tlm.js | 69 ++++++++++++++++++++++-------------- 2 files changed, 54 insertions(+), 31 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index e058814a..47a1dbbc 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -700,6 +700,7 @@ def add_dntoeu_value(field, packet, dntoeus): packet_states = { } +counters = { } def get_packet_delta(pkt_defn, packet): """ Keeps track of last packets recieved of all types recieved @@ -716,6 +717,7 @@ def get_packet_delta(pkt_defn, packet): # first packet of this type if pkt_defn.name not in packet_states: + counters[pkt_defn.name] = 0 packet_states[pkt_defn.name] = json_pkt delta = json_pkt @@ -725,6 +727,7 @@ def get_packet_delta(pkt_defn, packet): # previous packets of this type received else: + counters[pkt_defn.name] += 1 delta, dntoeus = {}, {} for field, new_value in json_pkt.items(): last_value = packet_states[pkt_defn.name][field] @@ -733,7 +736,7 @@ def get_packet_delta(pkt_defn, packet): packet_states[pkt_defn.name][field] = new_value dntoeus = add_dntoeu_value(field, ait_pkt, dntoeus) - return delta, dntoeus + return delta, dntoeus, counters[pkt_defn.name] def replace_datetimes(delta): @@ -768,14 +771,15 @@ def handle(): pkt_defn = v break - delta, dntoeus = get_packet_delta(pkt_defn, data) + delta, dntoeus, counter = get_packet_delta(pkt_defn, data) delta = replace_datetimes(delta) log.info(delta) wsock.send(json.dumps({ 'packet': pkt_defn.name, 'data': delta, - 'dntoeus': dntoeus + 'dntoeus': dntoeus, + 'counter': counter })) except IndexError: @@ -797,7 +801,11 @@ def handle(): @App.route('/tlm/latest', method='GET') def handle(): """Return latest telemetry packet to client""" - return json.dumps(packet_states) + for pkt_type, state in packet_states.items(): + packet_states[pkt_type] = replace_datetimes(state) + + return json.dumps({'states': packet_states, + 'counters': counters}) @App.route('/tlm/query', method='POST') diff --git a/ait/gui/static/js/ait/tlm.js b/ait/gui/static/js/ait/tlm.js index 28f588d4..f5025dc6 100644 --- a/ait/gui/static/js/ait/tlm.js +++ b/ait/gui/static/js/ait/tlm.js @@ -356,12 +356,9 @@ class TelemetryStream this._socket = new WebSocket(url) this._stale = 0 this._url = url - m.request({ url: '/tlm/latest' }).then((latest) => { - this._pkt_states = latest - console.log('requested latest tlm from within stream') - console.log(ait.tlm.state) - }) + this.getFullPacketStates() console.log(this._pkt_states) + console.log(this._counters) // Re-map telemetry dictionary to be keyed by a PacketDefinition // 'id' instead of 'name'. @@ -385,6 +382,24 @@ class TelemetryStream this._emit('close', this) } + getFullPacketStates () { + m.request({ url: '/tlm/latest' }).then( (latest) => { + console.log('requesting latest tlm') + console.log(latest) + this._pkt_states = latest['states'] + this._counters = latest['counters'] + }) + } + + checkCounter (packetName, counter) { + // returns true if counter is as expected, false if not + let lastCounter = this._counters[packetName] + if ( counter == lastCounter + 1 ) { + return true + } + console.log('counter mismatch: had ' + lastCounter + ' , got ' + counter) + return false + } onMessage (event) { if ( !(typeof event.data == "string") ) return @@ -394,35 +409,35 @@ class TelemetryStream let packet_name = data['packet'] let delta = data['data'] let dntoeus = data['dntoeus'] + let counter = data['counter'] - console.log('packet states:') - console.log(this._pkt_states) - - // add delta to last full packet - // want full packet inserted in packet buffer & emitted as event if ( packet_name in this._pkt_states ) { - if ( Object.keys(delta).length !== 0 ) { - for ( var field in delta ) { - this._pkt_states[packet_name][field] = this._pkt_states[packet_name][field] + delta[field] + // check counter is as expected and request full packet states if not + let gotNextCounter = this.checkCounter(packet_name, counter) + if ( !gotNextCounter ) { + this.getFullPacketStates() + } else { + // add delta to current packet state and update counter + if ( Object.keys(delta).length !== 0 ) { + console.log('adding delta to pkt state') + for ( var field in delta ) { + this._pkt_states[packet_name][field] = this._pkt_states[packet_name][field] + delta[field] + } } + this._counters[packet_name] = counter } - } else { - console.log('name not in states') - // delta is empty - request full packet from backend + } else { // new packet type + console.log('new packet type') if ( Object.keys(delta).length == 0 ) { - m.request({ url: '/tlm/latest' }).then( (latest) => { - console.log('requesting latest tlm') - console.log(latest) - //ait.tlm.state = latest - //packet_name = latest['packet'] - //delta = latest['data'] - //dntoeus = latest['dntoeus'] - }) - } - - this._pkt_states[packet_name] = delta + // empty delta - request full packet from backend + this.getFullPacketStates() + } else { + this._pkt_states[packet_name] = delta + this._counters[packet_name] = counter + } } + console.log("counter: ", this._counters[packet_name]) // Since WebSockets can stay open indefinitely, the AIT GUI // server will occasionally probe for dropped client // connections by sending empty packets (data.byteLength == 0) From 58d6801eaa81537259dedd0f7b8977b1631c0a17 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Mon, 14 Oct 2019 16:01:19 -0700 Subject: [PATCH 11/34] Add max 32 bit int as max packet counter (#105) --- ait/gui/__init__.py | 5 ++++- ait/gui/static/js/ait/tlm.js | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index 47a1dbbc..d9e31104 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -727,7 +727,9 @@ def get_packet_delta(pkt_defn, packet): # previous packets of this type received else: - counters[pkt_defn.name] += 1 + count = counters[pkt_defn.name] + count = count + 1 if count < 2**31 - 1 else 0 + counters[pkt_defn.name] = count delta, dntoeus = {}, {} for field, new_value in json_pkt.items(): last_value = packet_states[pkt_defn.name][field] @@ -774,6 +776,7 @@ def handle(): delta, dntoeus, counter = get_packet_delta(pkt_defn, data) delta = replace_datetimes(delta) log.info(delta) + log.info('Packet #{}'.format(counter)) wsock.send(json.dumps({ 'packet': pkt_defn.name, diff --git a/ait/gui/static/js/ait/tlm.js b/ait/gui/static/js/ait/tlm.js index f5025dc6..e2845dbb 100644 --- a/ait/gui/static/js/ait/tlm.js +++ b/ait/gui/static/js/ait/tlm.js @@ -394,9 +394,11 @@ class TelemetryStream checkCounter (packetName, counter) { // returns true if counter is as expected, false if not let lastCounter = this._counters[packetName] - if ( counter == lastCounter + 1 ) { + if ( counter == lastCounter + 1) { return true - } + } else if ( lastCounter == Math.pow(2, 31) - 1 && counter == 0 ) { + return true + } console.log('counter mismatch: had ' + lastCounter + ' , got ' + counter) return false } From e277c42fdd6939c2bdb6c7ed0c6f453c3bff37d7 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Thu, 17 Oct 2019 17:40:55 -0700 Subject: [PATCH 12/34] Deep copy packet state for inserting into ait.packets so fields update (#105) --- ait/gui/static/js/ait/gui/Field.js | 3 ++- ait/gui/static/js/ait/gui/Plot.js | 2 -- ait/gui/static/js/ait/tlm.js | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ait/gui/static/js/ait/gui/Field.js b/ait/gui/static/js/ait/gui/Field.js index 41af65e8..edcd23a6 100644 --- a/ait/gui/static/js/ait/gui/Field.js +++ b/ait/gui/static/js/ait/gui/Field.js @@ -285,7 +285,8 @@ const Field = }, onbeforeupdate (vnode, old) { - return this.hasChanged() + let changed = this.hasChanged() + return changed }, view (vnode) { diff --git a/ait/gui/static/js/ait/gui/Plot.js b/ait/gui/static/js/ait/gui/Plot.js index f3bd9cc8..2a631227 100644 --- a/ait/gui/static/js/ait/gui/Plot.js +++ b/ait/gui/static/js/ait/gui/Plot.js @@ -86,8 +86,6 @@ class DygraphsBackend if (!names) return let row = [ this._plot._time.get(packet) ] - console.log("Received by plotting:") - console.log(packet) // For each series of data, if it's in the current packet // that we're updating, add the associated point. Otherwise, // add a null value. Dygraphs requires that the data added diff --git a/ait/gui/static/js/ait/tlm.js b/ait/gui/static/js/ait/tlm.js index e2845dbb..f42f547e 100644 --- a/ait/gui/static/js/ait/tlm.js +++ b/ait/gui/static/js/ait/tlm.js @@ -457,7 +457,8 @@ class TelemetryStream this._stale = 0 this._interval = setInterval(this.onStale.bind(this), 5000) - ait.packets.insert(packet_name, this._pkt_states[packet_name]) + let current_pkt = JSON.parse(JSON.stringify(this._pkt_states[packet_name])) + ait.packets.insert(packet_name, current_pkt) this._emit('packet', {'packet': packet_name, 'data': this._pkt_states[packet_name]}) } From 52a614f33782217d9d86d229f39da8c2e6a61b6d Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Fri, 18 Oct 2019 08:50:54 -0700 Subject: [PATCH 13/34] Make counters session dependent (#105) --- ait/gui/__init__.py | 43 +++++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index d9e31104..6ec9c02d 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -42,6 +42,7 @@ def __init__ (self, store=None, maxlen=100): self.events = api.GeventDeque(maxlen=maxlen) self.messages = api.GeventDeque(maxlen=maxlen) self.telemetry = api.GeventDeque(maxlen=maxlen) + self.tlm_counters = { } self._maxlen = maxlen self._store = store self._numConnections = 0 @@ -80,11 +81,36 @@ def __init__ (self, *args, **kwargs): """Creates a new SessionStore.""" dict.__init__(self, *args, **kwargs) + def getPacketName(self, uid): + """ + Returns packet defn from tlm dict matching uid. + Logs error if no defn matching uid is found. + """ + tlmdict = ait.core.tlm.getDefaultDict() + for k, v in tlmdict.iteritems(): + if v.uid == uid: + return v + + log.error('No packet defn matching UID {}'.format(uid)) + def addTelemetry (self, uid, packet): """Adds a telemetry packet to all Sessions in the store.""" item = (uid, packet) SessionStore.History.telemetry.append(item) + for session in self.values(): + print(session) + + pkt_name = self.getPacketName(uid).name + if pkt_name not in session.tlm_counters: + session.tlm_counters[pkt_name] = 0 + else: + count = session.tlm_counters[pkt_name] + count = count + 1 if count < 2**31 - 1 else 0 + session.tlm_counters[pkt_name] = count + + print(session.tlm_counters) + item = (uid, packet, session.tlm_counters[pkt_name]) session.telemetry.append(item) def addMessage (self, msg): @@ -700,7 +726,6 @@ def add_dntoeu_value(field, packet, dntoeus): packet_states = { } -counters = { } def get_packet_delta(pkt_defn, packet): """ Keeps track of last packets recieved of all types recieved @@ -717,7 +742,6 @@ def get_packet_delta(pkt_defn, packet): # first packet of this type if pkt_defn.name not in packet_states: - counters[pkt_defn.name] = 0 packet_states[pkt_defn.name] = json_pkt delta = json_pkt @@ -727,9 +751,6 @@ def get_packet_delta(pkt_defn, packet): # previous packets of this type received else: - count = counters[pkt_defn.name] - count = count + 1 if count < 2**31 - 1 else 0 - counters[pkt_defn.name] = count delta, dntoeus = {}, {} for field, new_value in json_pkt.items(): last_value = packet_states[pkt_defn.name][field] @@ -738,7 +759,7 @@ def get_packet_delta(pkt_defn, packet): packet_states[pkt_defn.name][field] = new_value dntoeus = add_dntoeu_value(field, ait_pkt, dntoeus) - return delta, dntoeus, counters[pkt_defn.name] + return delta, dntoeus def replace_datetimes(delta): @@ -766,14 +787,14 @@ def handle(): tlmdict = ait.core.tlm.getDefaultDict() while not wsock.closed: try: - uid, data = session.telemetry.popleft(timeout=30) + uid, data, counter = session.telemetry.popleft(timeout=30) pkt_defn = None for k, v in tlmdict.iteritems(): if v.uid == uid: pkt_defn = v break - delta, dntoeus, counter = get_packet_delta(pkt_defn, data) + delta, dntoeus = get_packet_delta(pkt_defn, data) delta = replace_datetimes(delta) log.info(delta) log.info('Packet #{}'.format(counter)) @@ -807,8 +828,10 @@ def handle(): for pkt_type, state in packet_states.items(): packet_states[pkt_type] = replace_datetimes(state) - return json.dumps({'states': packet_states, - 'counters': counters}) + with Sessions.current() as session: + counters = session.tlm_counters + return json.dumps({'states': packet_states, + 'counters': counters}) @App.route('/tlm/query', method='POST') From 7fddbf8b6de6df77106f70405a554002ab0e3139 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Fri, 18 Oct 2019 16:16:54 -0700 Subject: [PATCH 14/34] Modularize getting packet definition and name (#105) --- ait/gui/__init__.py | 53 +++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index 6ec9c02d..8549cc06 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -69,6 +69,30 @@ def id (self): """A unique identifier for this Session.""" return str( id(self) ) + def updateCounter(self, pkt_name) + if pkt_name not in self.tlm_counters: + self.tlm_counters[pkt_name] = 0 + else: + count = self.tlm_counters[pkt_name] + count = count + 1 if count < 2**31 - 1 else 0 + self.tlm_counters[pkt_name] = count + + return self.tlm_counters[pkt_name] + + +def getPacketDefn(uid): + """ + Returns packet defn from tlm dict matching uid. + Logs warning and returns None if no defn matching uid is found. + """ + tlmdict = ait.core.tlm.getDefaultDict() + for k, v in tlmdict.iteritems(): + if v.uid == uid: + return v + + log.warn('No packet defn matching UID {}'.format(uid)) + return None + class SessionStore (dict): """SessionStore @@ -81,18 +105,6 @@ def __init__ (self, *args, **kwargs): """Creates a new SessionStore.""" dict.__init__(self, *args, **kwargs) - def getPacketName(self, uid): - """ - Returns packet defn from tlm dict matching uid. - Logs error if no defn matching uid is found. - """ - tlmdict = ait.core.tlm.getDefaultDict() - for k, v in tlmdict.iteritems(): - if v.uid == uid: - return v - - log.error('No packet defn matching UID {}'.format(uid)) - def addTelemetry (self, uid, packet): """Adds a telemetry packet to all Sessions in the store.""" item = (uid, packet) @@ -101,13 +113,8 @@ def addTelemetry (self, uid, packet): for session in self.values(): print(session) - pkt_name = self.getPacketName(uid).name - if pkt_name not in session.tlm_counters: - session.tlm_counters[pkt_name] = 0 - else: - count = session.tlm_counters[pkt_name] - count = count + 1 if count < 2**31 - 1 else 0 - session.tlm_counters[pkt_name] = count + pkt_name = getPacketDefn(uid).name + session.updateCounter(pkt_name) print(session.tlm_counters) item = (uid, packet, session.tlm_counters[pkt_name]) @@ -784,16 +791,10 @@ def handle(): bottle.abort(400, 'Expected WebSocket request.') try: - tlmdict = ait.core.tlm.getDefaultDict() while not wsock.closed: try: uid, data, counter = session.telemetry.popleft(timeout=30) - pkt_defn = None - for k, v in tlmdict.iteritems(): - if v.uid == uid: - pkt_defn = v - break - + pkt_defn = getPacketDefn(uid) delta, dntoeus = get_packet_delta(pkt_defn, data) delta = replace_datetimes(delta) log.info(delta) From 3b7ce2dab385904e7afda212b35361281ceb921f Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Fri, 18 Oct 2019 16:20:18 -0700 Subject: [PATCH 15/34] Fix bug in adding delta to packet state on frontend (#105) --- ait/gui/static/js/ait/tlm.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ait/gui/static/js/ait/tlm.js b/ait/gui/static/js/ait/tlm.js index f42f547e..e9bdf59d 100644 --- a/ait/gui/static/js/ait/tlm.js +++ b/ait/gui/static/js/ait/tlm.js @@ -423,7 +423,7 @@ class TelemetryStream if ( Object.keys(delta).length !== 0 ) { console.log('adding delta to pkt state') for ( var field in delta ) { - this._pkt_states[packet_name][field] = this._pkt_states[packet_name][field] + delta[field] + this._pkt_states[packet_name][field] = delta[field] } } this._counters[packet_name] = counter From 9175608c1e261197e7902359c34689640bdf0551 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Fri, 18 Oct 2019 16:31:00 -0700 Subject: [PATCH 16/34] Store deltas in queue per session (#105) --- ait/gui/__init__.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index 8549cc06..f69ada9f 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -42,6 +42,7 @@ def __init__ (self, store=None, maxlen=100): self.events = api.GeventDeque(maxlen=maxlen) self.messages = api.GeventDeque(maxlen=maxlen) self.telemetry = api.GeventDeque(maxlen=maxlen) + self.deltas = api.GeventDeque(maxlen=maxlen) self.tlm_counters = { } self._maxlen = maxlen self._store = store @@ -69,7 +70,7 @@ def id (self): """A unique identifier for this Session.""" return str( id(self) ) - def updateCounter(self, pkt_name) + def updateCounter(self, pkt_name): if pkt_name not in self.tlm_counters: self.tlm_counters[pkt_name] = 0 else: @@ -110,13 +111,17 @@ def addTelemetry (self, uid, packet): item = (uid, packet) SessionStore.History.telemetry.append(item) + pkt_defn = getPacketDefn(uid) + pkt_name = pkt_defn.name + delta, dntoeus = get_packet_delta(pkt_defn, packet) + delta = replace_datetimes(delta) + for session in self.values(): print(session) - - pkt_name = getPacketDefn(uid).name - session.updateCounter(pkt_name) - + counter = session.updateCounter(pkt_name) print(session.tlm_counters) + item = (pkt_name, delta, dntoeus, counter) + session.deltas.append(item) item = (uid, packet, session.tlm_counters[pkt_name]) session.telemetry.append(item) @@ -793,15 +798,12 @@ def handle(): try: while not wsock.closed: try: - uid, data, counter = session.telemetry.popleft(timeout=30) - pkt_defn = getPacketDefn(uid) - delta, dntoeus = get_packet_delta(pkt_defn, data) - delta = replace_datetimes(delta) + name, delta, dntoeus, counter = session.deltas.popleft(timeout=30) log.info(delta) - log.info('Packet #{}'.format(counter)) + log.info('packet #{}'.format(counter)) wsock.send(json.dumps({ - 'packet': pkt_defn.name, + 'packet': name, 'data': delta, 'dntoeus': dntoeus, 'counter': counter From 05e41715b404d986bdb4ff331d30365c8d4e70df Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Tue, 22 Oct 2019 11:13:26 -0700 Subject: [PATCH 17/34] Add tracking of dntoeus on backend (#105) --- ait/gui/__init__.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index f69ada9f..1b8d5e70 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -117,9 +117,7 @@ def addTelemetry (self, uid, packet): delta = replace_datetimes(delta) for session in self.values(): - print(session) counter = session.updateCounter(pkt_name) - print(session.tlm_counters) item = (pkt_name, delta, dntoeus, counter) session.deltas.append(item) item = (uid, packet, session.tlm_counters[pkt_name]) @@ -732,7 +730,11 @@ def add_dntoeu_value(field, packet, dntoeus): """ field_defn = packet._defn.fieldmap[field] if field_defn.dntoeu: - dntoeus[field] = field_defn.dntoeu.eval(packet) + value = field_defn.dntoeu.eval(packet) + # to send to frontend + dntoeus[field] = value + # track on backend + packet_states[packet._defn.name]['dntoeu'][field] = value return dntoeus @@ -754,10 +756,12 @@ def get_packet_delta(pkt_defn, packet): # first packet of this type if pkt_defn.name not in packet_states: - packet_states[pkt_defn.name] = json_pkt + packet_states[pkt_defn.name] = {} + packet_states[pkt_defn.name]['raw'] = json_pkt delta = json_pkt dntoeus = {} + packet_states[pkt_defn.name]['dntoeu'] = {} for field, value in json_pkt.items(): dntoeus = add_dntoeu_value(field, ait_pkt, dntoeus) @@ -765,10 +769,10 @@ def get_packet_delta(pkt_defn, packet): else: delta, dntoeus = {}, {} for field, new_value in json_pkt.items(): - last_value = packet_states[pkt_defn.name][field] + last_value = packet_states[pkt_defn.name]['raw'][field] if new_value != last_value: delta[field] = new_value - packet_states[pkt_defn.name][field] = new_value + packet_states[pkt_defn.name]['raw'][field] = new_value dntoeus = add_dntoeu_value(field, ait_pkt, dntoeus) return delta, dntoeus @@ -799,7 +803,6 @@ def handle(): while not wsock.closed: try: name, delta, dntoeus, counter = session.deltas.popleft(timeout=30) - log.info(delta) log.info('packet #{}'.format(counter)) wsock.send(json.dumps({ @@ -829,8 +832,9 @@ def handle(): def handle(): """Return latest telemetry packet to client""" for pkt_type, state in packet_states.items(): - packet_states[pkt_type] = replace_datetimes(state) + packet_states[pkt_type]['raw'] = replace_datetimes(state['raw']) + log.info(packet_states) with Sessions.current() as session: counters = session.tlm_counters return json.dumps({'states': packet_states, From 2bde0783f3cd6f05bd1bc1675ec0561ce44d5fcb Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Tue, 22 Oct 2019 11:21:09 -0700 Subject: [PATCH 18/34] Add tracking of dntoeu states on frontend (#105) --- ait/gui/static/js/ait/gui/Field.js | 8 +++++++- ait/gui/static/js/ait/gui/Plot.js | 4 ++-- ait/gui/static/js/ait/gui/Search.js | 8 ++++++-- ait/gui/static/js/ait/tlm.js | 9 +++++++-- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/ait/gui/static/js/ait/gui/Field.js b/ait/gui/static/js/ait/gui/Field.js index edcd23a6..fbc1b373 100644 --- a/ait/gui/static/js/ait/gui/Field.js +++ b/ait/gui/static/js/ait/gui/Field.js @@ -83,7 +83,13 @@ const Field = * retrieving the packet value. */ getValue (packet, raw=false) { - return packet && packet[this._fname] + if ( !raw ) { + if ( packet && this._fname in packet['dntoeu']) { + return packet && packet['dntoeu'][this._fname] + } + } + + return packet && packet['raw'][this._fname] }, diff --git a/ait/gui/static/js/ait/gui/Plot.js b/ait/gui/static/js/ait/gui/Plot.js index 2a631227..e9f9c4bf 100644 --- a/ait/gui/static/js/ait/gui/Plot.js +++ b/ait/gui/static/js/ait/gui/Plot.js @@ -80,7 +80,7 @@ class DygraphsBackend plot (data) { const pname = data['packet'] - let packet = data['data'] + let packet = data['data']['raw'] const names = this._plot._packets[pname] if (!names) return @@ -209,7 +209,7 @@ class HighchartsBackend plot(data) { const pname = data['packet'] - let packet = data['data'] + let packet = data['data']['raw'] const names = this._plot._packets[pname] if (!names) return diff --git a/ait/gui/static/js/ait/gui/Search.js b/ait/gui/static/js/ait/gui/Search.js index 33b769d6..84a51a50 100644 --- a/ait/gui/static/js/ait/gui/Search.js +++ b/ait/gui/static/js/ait/gui/Search.js @@ -139,8 +139,12 @@ const MnemonicSearch = { ) if (curPacket !== null) { - val = curPacket.__get__(this._selection) - raw = curPacket.__get__(this._selection, true) + if ( this._selection in curPacket['dntoeu']) { + val = curPacket['dntoeu'][this._selection] + } else { + val = curPacket['raw'][this._selection] + } + raw = curPacket['raw'][this._selection] } let data = {} diff --git a/ait/gui/static/js/ait/tlm.js b/ait/gui/static/js/ait/tlm.js index e9bdf59d..6c6eab14 100644 --- a/ait/gui/static/js/ait/tlm.js +++ b/ait/gui/static/js/ait/tlm.js @@ -423,7 +423,10 @@ class TelemetryStream if ( Object.keys(delta).length !== 0 ) { console.log('adding delta to pkt state') for ( var field in delta ) { - this._pkt_states[packet_name][field] = delta[field] + this._pkt_states[packet_name]['raw'][field] = delta[field] + } + for ( var field in dntoeus ) { + this._pkt_states[packet_name]['dntoeu'][field] = dntoeus[field] } } this._counters[packet_name] = counter @@ -434,7 +437,9 @@ class TelemetryStream // empty delta - request full packet from backend this.getFullPacketStates() } else { - this._pkt_states[packet_name] = delta + this._pkt_states[packet_name] = {} + this._pkt_states[packet_name]['raw'] = delta + this._pkt_states[packet_name]['dntoeu'] = dntoeus this._counters[packet_name] = counter } } From 11855c9988b909a82c1caf564ca9e2c29cc59962 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Tue, 22 Oct 2019 14:29:38 -0700 Subject: [PATCH 19/34] Remove packet class from frontend (#105) --- ait/gui/static/js/ait/tlm.js | 79 ------------------------------------ 1 file changed, 79 deletions(-) diff --git a/ait/gui/static/js/ait/tlm.js b/ait/gui/static/js/ait/tlm.js index 6c6eab14..c70d665e 100644 --- a/ait/gui/static/js/ait/tlm.js +++ b/ait/gui/static/js/ait/tlm.js @@ -78,84 +78,6 @@ class FieldDefinition } -class Packet -{ - /** - * Creates and returns a new `Packet` (subclass) instance based on the - * given `PacketDefinition` and packet data. If the optional `raw` - * parameter is true, all field accesses will result in the raw value - * being returned (e.g. DN to EU conversions will be skipped). - * - * NOTE: This method will also create and register a new `Packet` - * subclass based on the given `PacketDefinition`, if this is the first - * time `create()` is called with that `PacketDefinition`. - */ - static - create (defn, data, raw=false) { - const name = defn.name - let ctor = this[name] - - if (ctor === undefined) { - ctor = Packet.createSubclass(defn) - this[name] = ctor - } - - return new ctor(data, raw) - } - - - /** - * Creates and returns a new (anonymous) `Packet` subclass at runtime. - */ - static - createSubclass (defn) { - let SubPacket = function (data, raw) { - this._defn = defn - this._data = data - this._raw = raw - } - - SubPacket.prototype = Object.create(Packet.prototype) - SubPacket.prototype.constructor = SubPacket - - for (const name in defn.fields) { - Object.defineProperty(SubPacket.prototype, name, { - get: function () { - return this.__get__(name) - } - }) - } - - return SubPacket - } - - - __get__ (name, raw=false) { - let value = undefined - - if (this._data instanceof DataView) { - const defn = this._defn.fields[name] - - if (defn) { - if (raw || this._raw || !defn.dntoeu) { - value = defn.decode(this._data, raw) - } - else if (defn.dntoeu && defn.dntoeu.equation) { - value = this._defn.scope.eval(this, defn.dntoeu.equation) - } - } - } - - return value - } - - - __clone__ (data, raw=false) { - return Packet.create(this._defn, data, raw) - } -} - - class PacketDefinition { constructor (obj) { @@ -487,7 +409,6 @@ class TelemetryStream export { FieldDefinition, - Packet, PacketDefinition, PacketScope, TelemetryDictionary, From a07458364d5270807667d382f701af6fcdfb7254 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Tue, 22 Oct 2019 14:44:00 -0700 Subject: [PATCH 20/34] Use new function for getting pkt defn in openmct endpoint (#105) --- ait/gui/__init__.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index 1b8d5e70..356b155c 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -692,11 +692,7 @@ def handle(): while not wsock.closed: try: uid, data = session.telemetry.popleft(timeout=30) - pkt_defn = None - for k, v in tlmdict.iteritems(): - if v.uid == uid: - pkt_defn = v - break + pkt_defn = getPacketDefn(uid) wsock.send(json.dumps({ 'packet': pkt_defn.name, From 1e7be377e34a80ed6d297aeb7833c6406001d227 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Wed, 18 Dec 2019 16:33:55 -0800 Subject: [PATCH 21/34] Remove unecessary changes (debug prints, etc.) (#105) --- ait/gui/__init__.py | 2 -- ait/gui/static/js/ait/gui/Field.js | 3 +-- ait/gui/static/js/ait/gui/index.js | 2 -- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index 356b155c..89e9e251 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -799,7 +799,6 @@ def handle(): while not wsock.closed: try: name, delta, dntoeus, counter = session.deltas.popleft(timeout=30) - log.info('packet #{}'.format(counter)) wsock.send(json.dumps({ 'packet': name, @@ -830,7 +829,6 @@ def handle(): for pkt_type, state in packet_states.items(): packet_states[pkt_type]['raw'] = replace_datetimes(state['raw']) - log.info(packet_states) with Sessions.current() as session: counters = session.tlm_counters return json.dumps({'states': packet_states, diff --git a/ait/gui/static/js/ait/gui/Field.js b/ait/gui/static/js/ait/gui/Field.js index fbc1b373..ee68b445 100644 --- a/ait/gui/static/js/ait/gui/Field.js +++ b/ait/gui/static/js/ait/gui/Field.js @@ -291,8 +291,7 @@ const Field = }, onbeforeupdate (vnode, old) { - let changed = this.hasChanged() - return changed + return this.hasChanged() }, view (vnode) { diff --git a/ait/gui/static/js/ait/gui/index.js b/ait/gui/static/js/ait/gui/index.js index 15360bc2..c7c91897 100644 --- a/ait/gui/static/js/ait/gui/index.js +++ b/ait/gui/static/js/ait/gui/index.js @@ -199,8 +199,6 @@ function init () { m.request({ url: '/tlm/latest' }).then((latest) => { ait.tlm.state = latest - console.log('requested latest tlm') - console.log(ait.tlm.state) }) m.mount(root, { view: () => createMithrilNodes(elems) }) From c0e05159d071d2ad498b011db759c41c542c04f8 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Wed, 18 Dec 2019 17:10:06 -0800 Subject: [PATCH 22/34] Add packet defn data structure to avoid loop on every call (#105) --- ait/gui/__init__.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index 89e9e251..9d2a5b88 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -81,18 +81,26 @@ def updateCounter(self, pkt_name): return self.tlm_counters[pkt_name] +packet_defns = {} def getPacketDefn(uid): """ Returns packet defn from tlm dict matching uid. Logs warning and returns None if no defn matching uid is found. """ - tlmdict = ait.core.tlm.getDefaultDict() - for k, v in tlmdict.iteritems(): - if v.uid == uid: - return v + global packet_defns - log.warn('No packet defn matching UID {}'.format(uid)) - return None + if uid in packet_defns: + return packet_defns[uid] + + else: + tlmdict = ait.core.tlm.getDefaultDict() + for k, v in tlmdict.iteritems(): + if v.uid == uid: + packet_defns[uid] = v + return v + + log.warn('No packet defn matching UID {}'.format(uid)) + return None class SessionStore (dict): From aa2807788c05e0a301d362428069221f7bf4f5b1 Mon Sep 17 00:00:00 2001 From: Anna Waldron Date: Thu, 19 Dec 2019 11:58:47 -0800 Subject: [PATCH 23/34] Remove initial packet conversion to JSON to save time (#105) --- ait/gui/__init__.py | 50 +++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index 9d2a5b88..ad600219 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -723,26 +723,6 @@ def handle(): pass -def add_dntoeu_value(field, packet, dntoeus): - """ - Returns dntoeu value of field from packet. - - Params: - field: str name of field to evaluate - packet: AIT Packet - dntoeus: dict of dntoeu values to add to - """ - field_defn = packet._defn.fieldmap[field] - if field_defn.dntoeu: - value = field_defn.dntoeu.eval(packet) - # to send to frontend - dntoeus[field] = value - # track on backend - packet_states[packet._defn.name]['dntoeu'][field] = value - - return dntoeus - - packet_states = { } def get_packet_delta(pkt_defn, packet): """ @@ -756,28 +736,36 @@ def get_packet_delta(pkt_defn, packet): delta: JSON of packet fields that have changed """ ait_pkt = ait.core.tlm.Packet(pkt_defn, data=packet) - json_pkt = ait_pkt.toJSON() # first packet of this type if pkt_defn.name not in packet_states: packet_states[pkt_defn.name] = {} - packet_states[pkt_defn.name]['raw'] = json_pkt - delta = json_pkt - dntoeus = {} + # get raw fields + raw_fields = {f.name: getattr(ait_pkt.raw, f.name) for f in pkt_defn.fields} + packet_states[pkt_defn.name]['raw'] = raw_fields + delta = raw_fields + + # get converted fields packet_states[pkt_defn.name]['dntoeu'] = {} - for field, value in json_pkt.items(): - dntoeus = add_dntoeu_value(field, ait_pkt, dntoeus) + dntoeus = {f.name: getattr(ait_pkt, f.name) for f in pkt_defn.fields if f.dntoeu is not None} # previous packets of this type received else: delta, dntoeus = {}, {} - for field, new_value in json_pkt.items(): - last_value = packet_states[pkt_defn.name]['raw'][field] + + for field in pkt_defn.fields: + new_value = getattr(ait_pkt.raw, field.name) + last_value = packet_states[pkt_defn.name]['raw'][field.name] + if new_value != last_value: - delta[field] = new_value - packet_states[pkt_defn.name]['raw'][field] = new_value - dntoeus = add_dntoeu_value(field, ait_pkt, dntoeus) + delta[field.name] = new_value + packet_states[pkt_defn.name]['raw'][field.name] = new_value + + if field.dntoeu is not None: + dntoeu_val = getattr(ait_pkt, field.name) + dntoeus[field.name] = dntoeu_val + packet_states[pkt_defn.name]['dntoeu'][field.name] = dntoeu_val return delta, dntoeus From a437eec55c61ff448a31849751e054a20c59cd40 Mon Sep 17 00:00:00 2001 From: Kayo Kallas Date: Thu, 14 May 2020 09:36:52 -0700 Subject: [PATCH 24/34] Update dict.iteritems to dict.items (#169) --- ait/gui/__init__.py | 6 +++--- test_scripts/notification_test.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ait/gui/__init__.py b/ait/gui/__init__.py index ad600219..70b36623 100644 --- a/ait/gui/__init__.py +++ b/ait/gui/__init__.py @@ -94,7 +94,7 @@ def getPacketDefn(uid): else: tlmdict = ait.core.tlm.getDefaultDict() - for k, v in tlmdict.iteritems(): + for k, v in tlmdict.items(): if v.uid == uid: packet_defns[uid] = v return v @@ -175,7 +175,7 @@ class Playback(object): be sent to the frontend during this. playback.enabled: True if historical data playback is enabled. This will be False if a database connection cannot be made or if data playback is disabled for - some other reason. + some other reason. """ def __init__(self): @@ -316,7 +316,7 @@ def process(self, input_data, topic=None): self.process_log_msg(input_data) processed = True - if not processed: + if not processed: raise ValueError('Topic of received message not recognized as telem or log stream.') def process_telem_msg(self, msg): diff --git a/test_scripts/notification_test.py b/test_scripts/notification_test.py index d04736f8..a4a2a6fa 100644 --- a/test_scripts/notification_test.py +++ b/test_scripts/notification_test.py @@ -98,7 +98,7 @@ def get_packet_and_defn(): def get_limit_dict(): limit_dict = defaultdict(dict) - for k, v in limits.getDefaultDict().iteritems(): + for k, v in limits.getDefaultDict().items(): packet, field = k.split('.') limit_dict[packet][field] = v From 7aa9c54ac1118dcd60f3a5b62979fe03961fe675 Mon Sep 17 00:00:00 2001 From: Kayo Kallas Date: Thu, 14 May 2020 10:40:45 -0700 Subject: [PATCH 25/34] Add dntoeu plot option --- ait/gui/static/index.html | 14 +++++++------- ait/gui/static/js/ait/gui/Plot.js | 22 ++++++++++++---------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/ait/gui/static/index.html b/ait/gui/static/index.html index 929c8e00..7395993a 100644 --- a/ait/gui/static/index.html +++ b/ait/gui/static/index.html @@ -122,7 +122,7 @@

Welcome to the AMMOS Instrument Toolkit Telemetry Monitoring Interface!

The AIT UI is created from a number of components. These components are written in JavaScript and provided by the project to cover basic functionality needed in the user interface. Users can customize their interface by making minimal HTML or configuration file changes. If a user needs more control they can create their own components and include them in the interface.

- The majority of the content that you're seeing is displayed in ait-tabset components. The tabset component helps keep our display split into logical blocks and displays additional status information to the user. + The majority of the content that you're seeing is displayed in ait-tabset components. The tabset component helps keep our display split into logical blocks and displays additional status information to the user. 

   <ait-tabset>
@@ -202,10 +202,10 @@ 
1553 HS Voltages

                 "height": 300
               }
             
-            
-            
-            
-            
+            
+            
+            
+            
           
         
         

@@ -226,7 +226,7 @@ 
1553 HS Currents

                 "height": 300
               }
             
-            
+            
           
         

       
@@ -259,7 +259,7 @@ 
1553 HS Currents

         
       
       
-         
+        
       
       
         

diff --git a/ait/gui/static/js/ait/gui/Plot.js b/ait/gui/static/js/ait/gui/Plot.js
index e9f9c4bf..6a981753 100644
--- a/ait/gui/static/js/ait/gui/Plot.js
+++ b/ait/gui/static/js/ait/gui/Plot.js
@@ -78,9 +78,9 @@ class DygraphsBackend
         }
     }
 
-    plot (data) {
+    plot (data, dntoeu) {
         const pname = data['packet']
-        let packet = data['data']['raw']
+        let packet = dntoeu ? data['data']['dntoeu']:data['data']['raw']
         const names = this._plot._packets[pname]
 
         if (!names) return
@@ -207,9 +207,9 @@ class HighchartsBackend
         Object.assign(options, overrides)
     }
 
-    plot(data) {
+    plot(data, dntoeu) {
         const pname = data['packet']
-        let packet = data['data']['raw']
+        let packet = dntoeu ? data['data']['dntoeu']:data['data']['raw']
         const names = this._plot._packets[pname]
         if (!names) return
 
@@ -291,7 +291,7 @@ class HighchartsBackend
  *   The name of the field in the packet that defines this series.
  *
  * Optional attributes:
- * 
+ *
  * type
  *   The type of series being displayed. This is not relevant for all plot
  *   backends. For instance, Highcharts would use this to define the type
@@ -322,7 +322,7 @@ class HighchartsBackend
  *      {
  *         "title": "Plot title",
  *         "xlabel": "X label",
- *         "ylabel": "Y label"  
+ *         "ylabel": "Y label"
  *      }
  *    
  *
@@ -350,15 +350,15 @@ class HighchartsBackend
  *   
  *   
  *   
- *  
+ * 
  */
 const Plot =
 {
     /**
      * Plots data from the given packet.
      */
-    plot (packet) {
-        this._backend.plot(packet)
+    plot (packet, dntoeu=false) {
+        this._backend.plot(packet, dntoeu)
     },
 
 
@@ -402,6 +402,8 @@ const Plot =
         // For each packet, maintain a list of fields to plot
         this._packets[packet] = this._packets[packet] || [ ]
         this._packets[packet].push(name)
+
+        this._dntoeu   = vnode.attrs.dntoeu
     },
 
 
@@ -455,7 +457,7 @@ const Plot =
             this._time = new PlotTimeField()
         }
 
-        ait.events.on('ait:tlm:packet', (p) => this.plot(p))
+        ait.events.on('ait:tlm:packet', (p) => this.plot(p, this._dntoeu))
         ait.events.on('ait:playback:on', () => this.redraw())
         ait.events.on('ait:playback:off', () => this.redraw())
     },

From 21c8c91fab6f0e7165b059bc2a20bd55b91a6658 Mon Sep 17 00:00:00 2001
From: Kayo Kallas 
Date: Thu, 14 May 2020 11:01:36 -0700
Subject: [PATCH 26/34] Add getPacket utility method

---
 ait/gui/static/js/ait/gui/Search.js | 13 +++++++------
 ait/gui/static/js/ait/util.js       |  8 +++++++-
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/ait/gui/static/js/ait/gui/Search.js b/ait/gui/static/js/ait/gui/Search.js
index 84a51a50..2be9e4ed 100644
--- a/ait/gui/static/js/ait/gui/Search.js
+++ b/ait/gui/static/js/ait/gui/Search.js
@@ -21,12 +21,14 @@ import map from 'lodash/map'
 import defaults from 'lodash/defaults'
 
 import * as format from 'ait/format'
+import { getPacket }     from '../util.js'
 import Field from './Field'
 import Clock from './Clock'
 
+
 /**
  * Search for a Packets's telemetry fields by name and display dictionary data
- * and data snapshot of selected value in a modal. 
+ * and data snapshot of selected value in a modal.
  *
  * Requires that ait-modal is included in the page for modal functionality
  * to work. Default styling via the display-border and invert-colors attributes
@@ -132,6 +134,7 @@ const MnemonicSearch = {
 
         let val = 'N/A'
         let raw = 'N/A'
+        let dneu = 'N/A'
         let curTime = format.datetime(new Date(), {utc: true, gps: false})
         let curPacket = (ait.packets[this._packet] ?
             ait.packets[this._packet].get(0) :
@@ -139,11 +142,9 @@ const MnemonicSearch = {
         )
 
         if (curPacket !== null) {
-            if ( this._selection in curPacket['dntoeu']) {
-                val = curPacket['dntoeu'][this._selection]
-            } else {
-                val = curPacket['raw'][this._selection]
-            }
+            dneu = this._selection in curPacket['dntoeu']
+            val = getPacket(curPacket, dneu)
+            val = val[this._selection]
             raw = curPacket['raw'][this._selection]
         }
 
diff --git a/ait/gui/static/js/ait/util.js b/ait/gui/static/js/ait/util.js
index 78919369..d6b3443b 100644
--- a/ait/gui/static/js/ait/util.js
+++ b/ait/gui/static/js/ait/util.js
@@ -36,5 +36,11 @@ function move (array, from, to) {
     array.splice(to, 0, array.splice(from, 1)[0])
 }
 
+/**
+ *
+ */
+function getPacket (data, dntoeu) {
+    return dntoeu ? data['dntoeu']:data['raw']
+}
 
-export { merge, move}
+export { merge, move, getPacket }

From c2007a1eaa1c39d3e9ecbd1102756c307cd87a19 Mon Sep 17 00:00:00 2001
From: Kayo Kallas 
Date: Thu, 14 May 2020 11:20:49 -0700
Subject: [PATCH 27/34] Remove spaces

---
 ait/gui/static/js/ait/gui/Search.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ait/gui/static/js/ait/gui/Search.js b/ait/gui/static/js/ait/gui/Search.js
index 2be9e4ed..3136632e 100644
--- a/ait/gui/static/js/ait/gui/Search.js
+++ b/ait/gui/static/js/ait/gui/Search.js
@@ -21,7 +21,7 @@ import map from 'lodash/map'
 import defaults from 'lodash/defaults'
 
 import * as format from 'ait/format'
-import { getPacket }     from '../util.js'
+import { getPacket } from '../util.js'
 import Field from './Field'
 import Clock from './Clock'
 

From 610d47ad8e065358bbb737b34dfbcb18aeaf01ac Mon Sep 17 00:00:00 2001
From: Kayo Kallas 
Date: Thu, 14 May 2020 11:22:33 -0700
Subject: [PATCH 28/34] Use getPacket utility method

---
 ait/gui/static/js/ait/gui/Plot.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ait/gui/static/js/ait/gui/Plot.js b/ait/gui/static/js/ait/gui/Plot.js
index 6a981753..509817c6 100644
--- a/ait/gui/static/js/ait/gui/Plot.js
+++ b/ait/gui/static/js/ait/gui/Plot.js
@@ -16,6 +16,7 @@
 
 import m from 'mithril'
 import Dygraph from 'dygraphs';
+import { getPacket } from '../util.js'
 
 /*
  * FIXME: The two Backend classes (Dygraphs and Highcharts) are not cleanly
@@ -80,7 +81,7 @@ class DygraphsBackend
 
     plot (data, dntoeu) {
         const pname = data['packet']
-        let packet = dntoeu ? data['data']['dntoeu']:data['data']['raw']
+        let packet = getPacket(data['data'], dntoeu)
         const names = this._plot._packets[pname]
 
         if (!names) return
@@ -209,7 +210,7 @@ class HighchartsBackend
 
     plot(data, dntoeu) {
         const pname = data['packet']
-        let packet = dntoeu ? data['data']['dntoeu']:data['data']['raw']
+        let packet = getPacket(data['data'], dntoeu)
         const names = this._plot._packets[pname]
         if (!names) return
 

From 7f7b35caf4846cb1d0d92aacd8bee6904c8c0e40 Mon Sep 17 00:00:00 2001
From: Kayo Kallas 
Date: Mon, 18 May 2020 22:58:05 -0700
Subject: [PATCH 29/34] Update dntoeu plot option (#105)

---
 ait/gui/static/index.html         | 12 ++++++------
 ait/gui/static/js/ait/gui/Plot.js | 30 +++++++++++++++++-------------
 2 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/ait/gui/static/index.html b/ait/gui/static/index.html
index 7395993a..d62c5101 100644
--- a/ait/gui/static/index.html
+++ b/ait/gui/static/index.html
@@ -202,10 +202,10 @@ 
1553 HS Voltages

                 "height": 300
               }
             
-            
-            
-            
-            
+            
+            
+            
+            
           
         

         

@@ -226,13 +226,13 @@ 
1553 HS Currents

                 "height": 300
               }
             
-            
+            
           
         

       
     
   
-  
+
 
   

     
diff --git a/ait/gui/static/js/ait/gui/Plot.js b/ait/gui/static/js/ait/gui/Plot.js
index 509817c6..1d0b835e 100644
--- a/ait/gui/static/js/ait/gui/Plot.js
+++ b/ait/gui/static/js/ait/gui/Plot.js
@@ -79,9 +79,9 @@ class DygraphsBackend
         }
     }
 
-    plot (data, dntoeu) {
+    plot (data, raw) {
         const pname = data['packet']
-        let packet = getPacket(data['data'], dntoeu)
+        let packet = getPacket(data['data'], raw)
         const names = this._plot._packets[pname]
 
         if (!names) return
@@ -208,9 +208,9 @@ class HighchartsBackend
         Object.assign(options, overrides)
     }
 
-    plot(data, dntoeu) {
+    plot(data, raw) {
         const pname = data['packet']
-        let packet = getPacket(data['data'], dntoeu)
+        let packet = getPacket(data['data'], raw)
         const names = this._plot._packets[pname]
         if (!names) return
 
@@ -306,9 +306,13 @@ class HighchartsBackend
  *   point to be removed from the queue. This is only used by the Dygraphs
  *   backend.
  *
+ * raw
+ *   If the `raw` parameter is true, the raw value will be returned.
+ *   DN to EU conversions will be skipped. (Default: raw=false)
+ *
  * .. code:: Javascript
  *
- *    
+ *    
  *
  * **ait-plot-config:**
  *
@@ -347,10 +351,10 @@ class HighchartsBackend
  *       "height": 300
  *     }
  *   
- *   
- *   
- *   
- *   
+ *   
+ *   
+ *   
+ *   
  * 
  */
 const Plot =
@@ -358,8 +362,8 @@ const Plot =
     /**
      * Plots data from the given packet.
      */
-    plot (packet, dntoeu=false) {
-        this._backend.plot(packet, dntoeu)
+    plot (packet, raw=false) {
+        this._backend.plot(packet, raw)
     },
 
 
@@ -404,7 +408,7 @@ const Plot =
         this._packets[packet] = this._packets[packet] || [ ]
         this._packets[packet].push(name)
 
-        this._dntoeu   = vnode.attrs.dntoeu
+        this._raw   = vnode.attrs.raw
     },
 
 
@@ -458,7 +462,7 @@ const Plot =
             this._time = new PlotTimeField()
         }
 
-        ait.events.on('ait:tlm:packet', (p) => this.plot(p, this._dntoeu))
+        ait.events.on('ait:tlm:packet', (p) => this.plot(p, this._raw))
         ait.events.on('ait:playback:on', () => this.redraw())
         ait.events.on('ait:playback:off', () => this.redraw())
     },

From b89e233c9aade6cbb5ade28ca905fa3df0fc1b50 Mon Sep 17 00:00:00 2001
From: Kayo Kallas 
Date: Tue, 19 May 2020 00:01:43 -0700
Subject: [PATCH 30/34] Add digitsAfterDecimal option to legend (#105)

---
 ait/gui/static/js/ait/gui/Plot.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ait/gui/static/js/ait/gui/Plot.js b/ait/gui/static/js/ait/gui/Plot.js
index 1d0b835e..5abaa1ba 100644
--- a/ait/gui/static/js/ait/gui/Plot.js
+++ b/ait/gui/static/js/ait/gui/Plot.js
@@ -66,7 +66,8 @@ class DygraphsBackend
             legend:     'always',
             labelsSeparateLines: false,
             labelsDiv: this._plot_id,
-            showRangeSelector: true
+            showRangeSelector: true,
+            digitsAfterDecimal: 5
         }
     }
 

From 658cceaadb78033c4b39128730029eef77d637ce Mon Sep 17 00:00:00 2001
From: Kayo Kallas 
Date: Tue, 19 May 2020 09:56:41 -0700
Subject: [PATCH 31/34] Update getPacket utility method (#105)

---
 ait/gui/static/js/ait/util.js | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/ait/gui/static/js/ait/util.js b/ait/gui/static/js/ait/util.js
index d6b3443b..99197bd0 100644
--- a/ait/gui/static/js/ait/util.js
+++ b/ait/gui/static/js/ait/util.js
@@ -37,10 +37,23 @@ function move (array, from, to) {
 }
 
 /**
- *
+ * Returns DN to EU value if it exists unless the `raw` parameter is true,
+ * otherwise return raw value.
  */
-function getPacket (data, dntoeu) {
-    return dntoeu ? data['dntoeu']:data['raw']
+function getPacket (data, raw=false) {
+    let value = undefined
+
+    if (!data['dntoeu']){
+        value = data['raw']
+    }
+    else if (data['dntoeu'] && raw) {
+        value = data['raw']
+    }
+    else {
+        value = data['dntoeu']
+    }
+
+    return value
 }
 
 export { merge, move, getPacket }

From 4a5bf0e2ced13d8231afcd2049af44bc718a0b43 Mon Sep 17 00:00:00 2001
From: Kayo Kallas 
Date: Tue, 19 May 2020 09:57:20 -0700
Subject: [PATCH 32/34] Apply getPacket utility method (#105)

---
 ait/gui/static/js/ait/gui/Search.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/ait/gui/static/js/ait/gui/Search.js b/ait/gui/static/js/ait/gui/Search.js
index 3136632e..3fa6c869 100644
--- a/ait/gui/static/js/ait/gui/Search.js
+++ b/ait/gui/static/js/ait/gui/Search.js
@@ -134,7 +134,6 @@ const MnemonicSearch = {
 
         let val = 'N/A'
         let raw = 'N/A'
-        let dneu = 'N/A'
         let curTime = format.datetime(new Date(), {utc: true, gps: false})
         let curPacket = (ait.packets[this._packet] ?
             ait.packets[this._packet].get(0) :
@@ -142,8 +141,7 @@ const MnemonicSearch = {
         )
 
         if (curPacket !== null) {
-            dneu = this._selection in curPacket['dntoeu']
-            val = getPacket(curPacket, dneu)
+            val = getPacket(curPacket, raw=false)
             val = val[this._selection]
             raw = curPacket['raw'][this._selection]
         }

From dd52c2adbf04ade3ada58101a7bb818f441e7f65 Mon Sep 17 00:00:00 2001
From: Kayo Kallas 
Date: Tue, 19 May 2020 13:36:12 -0700
Subject: [PATCH 33/34] Check if field exists in dntoeu (#105)

---
 ait/gui/static/js/ait/gui/Search.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ait/gui/static/js/ait/gui/Search.js b/ait/gui/static/js/ait/gui/Search.js
index 3fa6c869..a0a8ce8c 100644
--- a/ait/gui/static/js/ait/gui/Search.js
+++ b/ait/gui/static/js/ait/gui/Search.js
@@ -134,6 +134,7 @@ const MnemonicSearch = {
 
         let val = 'N/A'
         let raw = 'N/A'
+        let dneu = 'N/A'
         let curTime = format.datetime(new Date(), {utc: true, gps: false})
         let curPacket = (ait.packets[this._packet] ?
             ait.packets[this._packet].get(0) :
@@ -141,7 +142,8 @@ const MnemonicSearch = {
         )
 
         if (curPacket !== null) {
-            val = getPacket(curPacket, raw=false)
+            dneu = this._selection in curPacket['dntoeu']
+            val = getPacket(curPacket, !dneu)
             val = val[this._selection]
             raw = curPacket['raw'][this._selection]
         }

From d8c568f54642f6a9e711ac77808b3bbe60bc7ca6 Mon Sep 17 00:00:00 2001
From: Kayo Kallas 
Date: Tue, 26 May 2020 11:44:41 -0700
Subject: [PATCH 34/34] Update uttlity method (#105)

---
 ait/gui/static/js/ait/gui/Plot.js   |  6 +++---
 ait/gui/static/js/ait/gui/Search.js |  4 ++--
 ait/gui/static/js/ait/util.js       | 17 +++++++----------
 3 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/ait/gui/static/js/ait/gui/Plot.js b/ait/gui/static/js/ait/gui/Plot.js
index 5abaa1ba..7f39020d 100644
--- a/ait/gui/static/js/ait/gui/Plot.js
+++ b/ait/gui/static/js/ait/gui/Plot.js
@@ -16,7 +16,7 @@
 
 import m from 'mithril'
 import Dygraph from 'dygraphs';
-import { getPacket } from '../util.js'
+import { getFieldType } from '../util.js'
 
 /*
  * FIXME: The two Backend classes (Dygraphs and Highcharts) are not cleanly
@@ -82,7 +82,7 @@ class DygraphsBackend
 
     plot (data, raw) {
         const pname = data['packet']
-        let packet = getPacket(data['data'], raw)
+        let packet = getFieldType(data['data'], raw)
         const names = this._plot._packets[pname]
 
         if (!names) return
@@ -211,7 +211,7 @@ class HighchartsBackend
 
     plot(data, raw) {
         const pname = data['packet']
-        let packet = getPacket(data['data'], raw)
+        let packet = getFieldType(data['data'], raw)
         const names = this._plot._packets[pname]
         if (!names) return
 
diff --git a/ait/gui/static/js/ait/gui/Search.js b/ait/gui/static/js/ait/gui/Search.js
index a0a8ce8c..a52cae98 100644
--- a/ait/gui/static/js/ait/gui/Search.js
+++ b/ait/gui/static/js/ait/gui/Search.js
@@ -21,7 +21,7 @@ import map from 'lodash/map'
 import defaults from 'lodash/defaults'
 
 import * as format from 'ait/format'
-import { getPacket } from '../util.js'
+import { getFieldType } from '../util.js'
 import Field from './Field'
 import Clock from './Clock'
 
@@ -143,7 +143,7 @@ const MnemonicSearch = {
 
         if (curPacket !== null) {
             dneu = this._selection in curPacket['dntoeu']
-            val = getPacket(curPacket, !dneu)
+            val = getFieldTyoe(curPacket, !dneu)
             val = val[this._selection]
             raw = curPacket['raw'][this._selection]
         }
diff --git a/ait/gui/static/js/ait/util.js b/ait/gui/static/js/ait/util.js
index 99197bd0..4f14146d 100644
--- a/ait/gui/static/js/ait/util.js
+++ b/ait/gui/static/js/ait/util.js
@@ -40,20 +40,17 @@ function move (array, from, to) {
  * Returns DN to EU value if it exists unless the `raw` parameter is true,
  * otherwise return raw value.
  */
-function getPacket (data, raw=false) {
-    let value = undefined
+function getFieldType (data, raw=false) {
 
-    if (!data['dntoeu']){
-        value = data['raw']
-    }
-    else if (data['dntoeu'] && raw) {
-        value = data['raw']
-    }
-    else {
+    // Default to the raw value since it will always be avilable
+    let value = data['raw']
+
+    // If raw==false and a DN to EU value exists, grab that
+    if (!raw && data['dntoeu']) {
         value = data['dntoeu']
     }
 
     return value
 }
 
-export { merge, move, getPacket }
+export { merge, move, getFieldType }