Issue #105_kkallas - Rework of Backend/Frontend telemetry processing

ait/gui/__init__.py

@@ -42,6 +42,8 @@ def __init__ (self, store=None, maxlen=100):
         self.events          = api.GeventDeque(maxlen=maxlen)
         self.messages        = api.GeventDeque(maxlen=maxlen)
         self.telemetry       = api.GeventDeque(maxlen=maxlen)
+        self.deltas          = api.GeventDeque(maxlen=maxlen)
+        self.tlm_counters    = { }
         self._maxlen         = maxlen
         self._store          = store
         self._numConnections = 0
@@ -68,6 +70,38 @@ def id (self):
         """A unique identifier for this Session."""
         return str( id(self) )
 
+    def updateCounter(self, pkt_name):
+        if pkt_name not in self.tlm_counters:
+            self.tlm_counters[pkt_name] = 0
+        else:
+            count = self.tlm_counters[pkt_name]
+            count = count + 1 if count < 2**31 - 1 else 0
+            self.tlm_counters[pkt_name] = count
+
+        return self.tlm_counters[pkt_name]
+
+
+packet_defns = {}
+def getPacketDefn(uid):
+    """
+    Returns packet defn from tlm dict matching uid.
+    Logs warning and returns None if no defn matching uid is found.
+    """
+    global packet_defns
+
+    if uid in packet_defns:
+        return packet_defns[uid]
+
+    else:
+        tlmdict = ait.core.tlm.getDefaultDict()
+        for k, v in tlmdict.items():
+            if v.uid == uid:
+                packet_defns[uid] = v
+                return v
+
+        log.warn('No packet defn matching UID {}'.format(uid))
+        return None
+
 
 class SessionStore (dict):
     """SessionStore
@@ -84,7 +118,17 @@ def addTelemetry (self, uid, packet):
         """Adds a telemetry packet to all Sessions in the store."""
         item = (uid, packet)
         SessionStore.History.telemetry.append(item)
+
+        pkt_defn = getPacketDefn(uid)
+        pkt_name = pkt_defn.name
+        delta, dntoeus = get_packet_delta(pkt_defn, packet)
+        delta = replace_datetimes(delta)
+
         for session in self.values():
+            counter = session.updateCounter(pkt_name)
+            item = (pkt_name, delta, dntoeus, counter)
+            session.deltas.append(item)
+            item = (uid, packet, session.tlm_counters[pkt_name])
             session.telemetry.append(item)
 
     def addMessage (self, msg):
@@ -131,7 +175,7 @@ class Playback(object):
         be sent to the frontend during this.
     playback.enabled: True if historical data playback is enabled. This will be False
         if a database connection cannot be made or if data playback is disabled for
-        some other reason. 
+        some other reason.
     """
 
     def __init__(self):
@@ -272,7 +316,7 @@ def process(self, input_data, topic=None):
                 self.process_log_msg(input_data)
                 processed = True
 
-        if not processed: 
+        if not processed:
             raise ValueError('Topic of received message not recognized as telem or log stream.')
 
     def process_telem_msg(self, msg):
@@ -640,6 +684,7 @@ def handle():
             __setResponseToEventStream()
             yield 'data: %s\n\n' % json.dumps(msg)
 
+
 @App.route('/tlm/realtime/openmct')
 def handle():
     """Return telemetry packets in realtime to client"""
@@ -655,13 +700,7 @@ def handle():
         while not wsock.closed:
             try:
                 uid, data = session.telemetry.popleft(timeout=30)
-                pkt_defn = None
-                for k, v in tlmdict.iteritems():
-                    if v.uid == uid:
-                        pkt_defn = v
-                        break
-                else:
-                    continue
+                pkt_defn = getPacketDefn(uid)
 
                 wsock.send(json.dumps({
                     'packet': pkt_defn.name,
@@ -684,6 +723,63 @@ def handle():
         pass
 
 
+packet_states = { }
+def get_packet_delta(pkt_defn, packet):
+    """
+    Keeps track of last packets recieved of all types recieved
+    and returns only fields that have changed since last packet.
+
+    Params:
+        pkt_defn:  Packet definition
+        packet:    Binary packet data
+    Returns:
+        delta:     JSON of packet fields that have changed
+    """
+    ait_pkt = ait.core.tlm.Packet(pkt_defn, data=packet)
+
+    # first packet of this type
+    if pkt_defn.name not in packet_states:
+        packet_states[pkt_defn.name] = {}
+
+        # get raw fields
+        raw_fields = {f.name: getattr(ait_pkt.raw, f.name) for f in pkt_defn.fields}
+        packet_states[pkt_defn.name]['raw'] = raw_fields
+        delta = raw_fields
+
+        # get converted fields
+        packet_states[pkt_defn.name]['dntoeu'] = {}
+        dntoeus = {f.name: getattr(ait_pkt, f.name) for f in pkt_defn.fields if f.dntoeu is not None}
+
+    # previous packets of this type received
+    else:
+        delta, dntoeus = {}, {}
+
+        for field in pkt_defn.fields:
+            new_value = getattr(ait_pkt.raw, field.name)
+            last_value = packet_states[pkt_defn.name]['raw'][field.name]
+
+            if new_value != last_value:
+                delta[field.name] = new_value
+                packet_states[pkt_defn.name]['raw'][field.name] = new_value
+
+                if field.dntoeu is not None:
+                    dntoeu_val = getattr(ait_pkt, field.name)
+                    dntoeus[field.name] = dntoeu_val
+                    packet_states[pkt_defn.name]['dntoeu'][field.name] = dntoeu_val
+
+    return delta, dntoeus
+
+
+def replace_datetimes(delta):
+    """Replace datetime objects with ISO formatted
+    strings for JSON serialization"""
+    for key, val in delta.items():
+        if type(val) is datetime:
+            delta[key] = val.isoformat()
+
+    return delta
+
+
 @App.route('/tlm/realtime')
 def handle():
     """Return telemetry packets in realtime to client"""
@@ -698,8 +794,15 @@ def handle():
         try:
             while not wsock.closed:
                 try:
-                    uid, data = session.telemetry.popleft(timeout=30)
-                    wsock.send(pad + struct.pack('>I', uid) + data)
+                    name, delta, dntoeus, counter = session.deltas.popleft(timeout=30)
+
+                    wsock.send(json.dumps({
+                        'packet': name,
+                        'data': delta,
+                        'dntoeus': dntoeus,
+                        'counter': counter
+                    }))
+
                 except IndexError:
                     # If no telemetry has been received by the GUI
                     # server after timeout seconds, "probe" the client
@@ -716,6 +819,18 @@ def handle():
             pass
 
 
+@App.route('/tlm/latest', method='GET')
+def handle():
+    """Return latest telemetry packet to client"""
+    for pkt_type, state in packet_states.items():
+        packet_states[pkt_type]['raw'] = replace_datetimes(state['raw'])
+
+    with Sessions.current() as session:
+        counters = session.tlm_counters
+        return json.dumps({'states': packet_states,
+                           'counters': counters})
+
+
 @App.route('/tlm/query', method='POST')
 def handle():
     """"""

ait/gui/static/index.html

@@ -122,7 +122,7 @@ <h1>Welcome to the AMMOS Instrument Toolkit Telemetry Monitoring Interface!</h1>
               The AIT UI is created from a number of <strong>components</strong>. These components are written in JavaScript and provided by the project to cover basic functionality needed in the user interface. Users can customize their interface by making minimal HTML or configuration file changes. If a user needs more control they can create their own components and include them in the interface.
             </p>
             <p>
-              The majority of the content that you're seeing is displayed in <strong>ait-tabset</strong> components. The tabset component helps keep our display split into logical blocks and displays additional status information to the user. 
+              The majority of the content that you're seeing is displayed in <strong>ait-tabset</strong> components. The tabset component helps keep our display split into logical blocks and displays additional status information to the user.
 
 <pre>
   &lt;ait-tabset&gt;
@@ -202,10 +202,10 @@ <h4 class="telem-group-title">1553 HS Voltages</h4>
                 "height": 300
               }
             </ait-plot-config>
-            <ait-plot-series packet="1553_HS_Packet" name="Voltage_A"></ait-plot-series>
-            <ait-plot-series packet="1553_HS_Packet" name="Voltage_B"></ait-plot-series>
-            <ait-plot-series packet="1553_HS_Packet" name="Voltage_C"></ait-plot-series>
-            <ait-plot-series packet="1553_HS_Packet" name="Voltage_D"></ait-plot-series>
+            <ait-plot-series packet="1553_HS_Packet" name="Voltage_A" raw=true></ait-plot-series>
+            <ait-plot-series packet="1553_HS_Packet" name="Voltage_B" raw=true></ait-plot-series>
+            <ait-plot-series packet="1553_HS_Packet" name="Voltage_C" raw=true></ait-plot-series>
+            <ait-plot-series packet="1553_HS_Packet" name="Voltage_D" raw=true></ait-plot-series>
           </ait-plot>
         </div>
         <div class="col-sm-6">
@@ -232,7 +232,7 @@ <h4 class="telem-group-title">1553 HS Currents</h4>
       </ait-tab>
     </ait-tabset>
   </div>
-  
+
 
   <div class='ctrlcontainer'>
     <ait-tabset class='nav-tabs'>
@@ -259,7 +259,7 @@ <h4 class="telem-group-title">1553 HS Currents</h4>
         <ait-command-history></ait-command-history>
       </ait-tab>
       <ait-tab title='Sequences'>
-        <ait-sequence></ait-sequence> 
+        <ait-sequence></ait-sequence>
       </ait-tab>
       <ait-tab title="Script Control">
         <div class="row">

ait/gui/static/js/ait/gui/Field.js

@@ -83,7 +83,13 @@ const Field =
      * retrieving the packet value.
      */
     getValue (packet, raw=false) {
-        return packet && packet.__get__(this._fname, raw)
+        if ( !raw ) {
+            if ( packet && this._fname in packet['dntoeu']) {
+                return packet && packet['dntoeu'][this._fname]
+            }
+        }
+
+        return packet && packet['raw'][this._fname]
     },
 
 
@@ -285,7 +291,7 @@ const Field =
     },
 
     onbeforeupdate (vnode, old) {
-        return this.hasChanged()
+        return this.hasChanged() 
     },
 
     view (vnode) {