integration-tests: implement framework for simulator-based integratio…

…n tests

The py/viaems directory now contains code to:
 - interface with the hosted-mode simulator
 - define the concept of a test scenario with inputs
 - a Nminus1+cam test trigger for scenarios
 - validation for output angles and durations

 Additionally, py/integration-tests/smoke-tests.py an initial example of
 using this infrastructure to create a basic smoke test for engine
 cranking and startup.

.github/workflows/checks.yaml

@@ -13,13 +13,12 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: recursive
-      - name: install pip deps
-        run: pip install -r integration/requirements.txt
-
       - name: run unit tests
         run: make PLATFORM=test check
       - name: build hosted platform
         run: make PLATFORM=hosted
+      - name: install pip deps
+        run: ( cd py; python3 -m pip install --upgrade pip; python3 -m pip install -r requirements.txt )
       - name: run hosted integration checks
         run: make PLATFORM=hosted integration
       - name: run lint

integration/interface-tests.py → py/integration-tests/interface-tests.py

@@ -1,11 +1,7 @@
 #!/usr/bin/env python3
 import unittest
-import cbor
-import subprocess
-import random
-import os
 
-from viaems import ViaemsWrapper
+from viaems.connector import ViaemsWrapper
 
 def _leaves_have_types(obj):
     if type(obj) == dict:

py/integration-tests/smoke-tests.py

@@ -0,0 +1,101 @@
+import unittest
+import sys
+import cbor
+
+from viaems.connector import ViaemsWrapper
+from viaems.decoder import CrankNMinus1PlusCam_Wheel
+from viaems.scenario import Scenario
+from viaems.testcase import TestCase
+from viaems.util import ticks_for_rpm_degrees, ms_ticks
+from viaems.validation import validate_outputs
+
+
+
+class NMinus1DecoderTests(TestCase):
+
+    def test_start_stop_start(self):
+      scenario = Scenario("start_stop_start", CrankNMinus1PlusCam_Wheel(36))
+      scenario.set_brv(12.5)
+      scenario.set_map(102);
+      scenario.wait_milliseconds(1000)
+      t1 = scenario.now()
+
+      # simulate a crank condition
+      scenario.set_rpm(300);
+      scenario.set_brv(9.0)
+      scenario.set_map(80.0)
+      scenario.wait_milliseconds(1000)
+
+      # engine catch ramp-up
+      t2 = scenario.now()
+      for rpm in range(300, 800, 100):
+          scenario.set_rpm(rpm)
+          scenario.wait_milliseconds(100)
+
+      t3 = scenario.now()
+
+      scenario.set_map(35)
+      scenario.set_brv(14.4)
+      scenario.wait_milliseconds(1000)
+
+      t4 = scenario.now()
+
+      scenario.set_rpm(0);
+      scenario.set_map(102)
+      scenario.set_brv(12)
+      scenario.wait_milliseconds(5000)
+
+      t5 = scenario.now()
+      scenario.end()
+
+      results = self.conn.execute_scenario(scenario)
+
+#     t1 - t2 is cranking. validate:
+#       - we gain sync within 500 ms
+#       - we get first output within 1 cycles
+#       - (after sync) advance is reasonable
+#       - (after sync) pw is reasonable
+
+      first_sync = next(filter(lambda i: i.values['sync'] == 1,
+                               results.filter_between(t1, t2).filter_feeds()))
+
+      self.assertLessEqual(first_sync.time - t1, ms_ticks(500))
+
+      for f in results.filter_between(first_sync.time, t2).filter_feeds():
+          self.assertWithin(f.values['advance'], 10, 20)
+          self.assertEqual(f.values['sync'], 1)
+          self.assertWithin(f.values['fuel_pulsewidth_us'], 4400, 4500)
+
+      first_output = next(results.filter_between(t1, t2).filter_outputs())
+      self.assertEqual(first_output.cycle, 1)
+
+
+#    t2-t4:
+#     - should have sync
+#     - fully validate output
+      for f in results.filter_between(t2, t4).filter_feeds():
+          self.assertEqual(f.values['sync'], 1)
+
+#    t4-t5:
+#     - we lose sync within X ms, and it stays lost
+#     - last output within Y ms
+      last_sync = next(filter(lambda i: i.values['sync'] == 0,
+                               results.filter_between(t4, t5).filter_feeds()))
+
+      self.assertWithin(last_sync.time - t4, ms_ticks(1), ms_ticks(100))
+      remaining_syncs = filter(lambda i: i.values['sync'] == 1,
+                               results.filter_between(last_sync.time, t5).filter_feeds())
+      self.assertEqual(len(list(remaining_syncs)), 0)
+
+      outputs = list(results.filter_between(t4, t5).filter_outputs())
+      if len(outputs) > 0:
+          self.assertWithin(outputs[-1].time, ms_ticks(1), ms_ticks(100))
+
+      # Finally, validate all the outputs are associated with an event and match
+      # the expected angles/durations
+      is_valid, msg = validate_outputs(results.filter_between(t1, t5))
+      self.assertTrue(is_valid, msg)
+
+
+if __name__ == "__main__":
+    unittest.main()

py/pyproject.toml

@@ -0,0 +1,3 @@
+[project]
+name = "viaems"
+version = "0.0.0"

integration/requirements.txt → py/requirements.txt

@@ -1,2 +1,5 @@
 pip
 cbor
+pyvcd
+
+.

integration/get-config.py → py/scripts/get-config.py

@@ -1,4 +1,4 @@
-from viaems import ViaemsWrapper
+from viaems.connector import ViaemsWrapper
 import json
 
 conn = ViaemsWrapper("obj/hosted/viaems")

integration/get-structure.py → py/scripts/get-structure.py

@@ -1,4 +1,4 @@
-from viaems import ViaemsWrapper
+from viaems.connector import ViaemsWrapper
 import json
 
 conn = ViaemsWrapper("obj/hosted/viaems")

integration/viaems.py → py/viaems/connector.py

@@ -1,18 +1,27 @@
-
 import cbor
+import json
 import subprocess
+import tempfile
 import random
 import os
 
+from viaems.vcd import dump_vcd
+from viaems.validation import enrich_log
+
 class ViaemsWrapper:
     def __init__(self, binary):
         self.binary = binary
         self.process = None
 
-    def start(self):
-        self.process = subprocess.Popen([self.binary], bufsize=-1,
+    def start(self, replay=None):
+        args = [self.binary]
+        if replay:
+          args.append("-i")
+          args.append(replay)
+
+        self.process = subprocess.Popen(args, bufsize=-1,
                 stdin=subprocess.PIPE, stdout=subprocess.PIPE,
-                stderr=subprocess.DEVNULL)
+                stderr=subprocess.PIPE)
 
     def kill(self):
         if not self.process:
@@ -72,3 +81,26 @@ def set(self, path, value):
         result = self.recv_until_id(id)
         return result
 
+    def execute_scenario(self, scenario):
+      tf = open(f"scenario_{scenario.name}.inputs", "w")
+      for ev in scenario.events:
+        tf.write(ev.render())
+      tf.close()
+
+      self.start(replay=tf.name)
+      results = []
+      while True:
+        try:
+          msg = self.recv()
+          results.append(msg)
+        except EOFError:
+          break
+
+      for line in self.process.stderr:
+          print(line)
+
+
+      results.sort(key=lambda x: x["time"])
+      dump_vcd(results, f"scenario_{scenario.name}.vcd")
+      enriched_log = enrich_log(scenario.events, results)
+      return enriched_log

py/viaems/decoder.py

@@ -0,0 +1,39 @@
+from viaems.util import ticks_for_rpm_degrees, clamp_angle
+
+class CrankNMinus1PlusCam_Wheel:
+  def __init__(self, N):
+    self.N = N
+    self.degrees_per_tooth = 360.0 / N
+    self.cycle = 0
+    self.index = 0
+
+    # Populate even tooth wheel for trigger 0
+    self.wheel = [(0, x * self.degrees_per_tooth) 
+        for x in range(N * 2)
+        # Except for a missing tooth each rev
+        # The first tooth *after* the gap is the 0 degree mark, and we want
+        # the first tooth to be angle 0
+        if x != 35 and x != (N + 35)
+        ]
+
+    # Add a cam sync at 45 degrees
+    self.wheel.append((1, 45.0))
+    self.wheel.sort(key=lambda x: x[1])
+
+  def _next_index(self):
+    return (self.index + 1) % len(self.wheel)
+
+  def time_to_next_trigger(self, rpm):
+    current_angle = self.wheel[self.index][1]
+    next_angle = self.wheel[self._next_index()][1]
+    diff = clamp_angle(next_angle - current_angle)
+    return ticks_for_rpm_degrees(rpm, diff)
+
+  def next(self, rpm):
+    self.index = self._next_index()
+    trigger, angle = self.wheel[self.index]
+    if angle == 0:
+      self.cycle += 1
+    return (trigger, angle)
+
+

py/viaems/scenario.py

@@ -0,0 +1,126 @@
+
+class ToothEvent:
+  def __init__(self, time, delay, trigger, angle=None, rpm=None, cycle=None):
+    self.time = time
+    self.delay = delay
+    self.angle = angle
+    self.rpm = rpm
+    self.trigger = trigger
+    self.cycle = cycle
+
+  def render(self):
+      return f"t {int(self.delay)} {self.trigger}\n"
+
+class EndEvent:
+  def __init__(self, delay):
+    self.delay = delay
+
+  def render(self):
+    return f"e {int(self.delay)}\n"
+
+class AdcEvent:
+  def __init__(self, delay, values):
+    self.delay = delay
+    self.values = [x for x in values] # deep copy
+
+  def render(self):
+    vals = " ".join([str(x) for x in self.values])
+    return f"a {int(self.delay)} {vals}\n"
+
+
+class Scenario:
+  def __init__(self, name, decoder):
+    self.name = name
+    self.events = []
+    self.decoder = decoder
+    self.time = 0
+    self.rpm = 0
+    self.adc = [0.0] * 16
+    self.adc_sample_rate = 5000
+    self.adc_delay = 4000000 / self.adc_sample_rate
+
+    self._last_event = None
+    self._last_trigger_time = 0
+    self._last_adc_time = 0
+
+  # TODO: factor these out, maybe parse a config to set them
+  def set_brv(self, brv):
+      self.adc[2] = brv / 24.5 * 5.0
+
+  def set_map(self, map_kpa):
+      self.adc[3] = (map_kpa - 12) / (420 - 12) * 5.0
+
+  def set_iat(self, iat):
+      pass
+
+  def set_clt(self, clt):
+      pass
+
+  def set_clt(self, clt):
+      pass
+
+  def _advance(self, max_delay=None):
+    # Advance to the next known point in time, which is either the next
+    # trigger/tooth, or the next adc sample event
+
+    if self.rpm == 0:
+        next_trigger_time = 3600 * 4000000 # Far future
+    else:
+        next_trigger_time = self._last_trigger_time + self.decoder.time_to_next_trigger(self.rpm)
+        # for when rpm becomes nonzero
+        if next_trigger_time < self.time:
+            next_trigger_time = self.time
+
+
+    next_adc_time = self._last_adc_time + self.adc_delay
+    if max_delay is not None:
+        max_time = self.time + max_delay
+
+        if max_time < next_adc_time and max_time < next_trigger_time:
+            self.time += max_delay
+            return
+
+    if next_trigger_time < next_adc_time:
+        tooth, angle = self.decoder.next(self.rpm)
+
+        delay = next_trigger_time - self.time
+        ev = ToothEvent(time=next_trigger_time,
+                        delay=delay, 
+                        trigger=tooth, 
+                        angle=angle,
+                        rpm=self.rpm,
+                        cycle=self.decoder.cycle)
+
+        self.events.append(ev)
+        self.time += delay
+        self._last_event = ev
+        self._last_trigger_time = next_trigger_time
+    else:
+        delay = next_adc_time - self.time
+        ev = AdcEvent(delay=delay, values=self.adc)
+
+        self.events.append(ev)
+        self.time += delay
+        self._last_event = ev
+        self._last_adc_time = next_adc_time
+
+  def set_rpm(self, rpm):
+    self.rpm = rpm
+
+  def wait_until_revolution(self, rev):
+    while self.decoder.revolutions <= rev:
+      self._advance()
+
+  def wait_milliseconds(self, ms):
+    targettime = self.time + 4000 * ms
+
+    while self.time <= targettime:
+      self._advance(4000 * ms)
+
+  def end(self, delay=1):
+    ev = EndEvent(delay=delay * 4000000)
+    self.events.append(ev)
+
+  def now(self):
+      return self.time
+

py/viaems/testcase.py

@@ -0,0 +1,16 @@
+import unittest
+
+from viaems.connector import ViaemsWrapper
+
+class TestCase(unittest.TestCase):
+
+    def assertWithin(self, val, lower, upper):
+        self.assertGreaterEqual(val, lower)
+        self.assertLessEqual(val, upper)
+
+
+    def setUp(self):
+        self.conn = ViaemsWrapper("obj/hosted/viaems")
+
+    def tearDown(self):
+        self.conn.kill()