Merge pull request #1145 from advancedtelematic/feat/OTA-2282/more-fa…

…ilure-simulation

Feat/ota 2282/more failure simulation

CHANGELOG.md

@@ -6,6 +6,10 @@ Our versioning scheme is `YEAR.N` where `N` is incremented whenever a new releas
 
 ## [??? (unreleased)]
 
+### Changed
+
+- Device installation failure result codes are deduced as concatenation of ECU failure result codes
+
 ## [2019.2] - 2019-02-21
 
 ### Added

CMakeLists.txt

@@ -165,6 +165,7 @@ if(FAULT_INJECTION)
     find_package(Libfiu REQUIRED)
     add_definitions(-DFIU_ENABLE)
     link_libraries(fiu dl)
+    install(PROGRAMS scripts/fiu DESTINATION bin COMPONENT aktualizr)
 endif(FAULT_INJECTION)
 
 # set symbols used when compiling

actions.md

@@ -160,6 +160,7 @@ These are the primary actions that a user of libaktualizr can perform through th
     - [x] Send EcuInstallationCompletedReport to server for secondaries (aktualizr_test.cc)
       - [x] Send an event report (see below)
   - [x] Store installation result for device (uptane_test.cc)
+  - [x] Compute device installation failure code as concatenation of ECU failure codes (aktualizr_test.cc)
   - [ ] Store negative device installation result when an ECU installation failed
   - [x] Send AllInstallsComplete event after all installations are finished (aktualizr_test.cc)
 - [x] Send installation report
@@ -245,6 +246,7 @@ These are internal requirements that are relatively opaque to the user and/or co
   - [x] Load and store Uptane roots in an SQL database (storage_common_test.cc)
   - [x] Load and store the device ID in an SQL database (storage_common_test.cc)
   - [x] Load and store ECU serials in an SQL database (storage_common_test.cc)
+    - [x] Preserve ECU ordering between store and load calls (storage_common_test.cc)
   - [x] Load and store a list of misconfigured ECUs in an SQL database (storage_common_test.cc)
   - [x] Load and store a flag indicating successful registration in an SQL database (storage_common_test.cc)
   - [x] Load and store an ecu installation result in an SQL database (storage_common_test.cc)

cmake-modules/AddAktualizrTest.cmake

@@ -1,32 +1,33 @@
 function(add_aktualizr_test)
     set(options PROJECT_WORKING_DIRECTORY NO_VALGRIND)
     set(oneValueArgs NAME)
-    set(multiValueArgs SOURCES LIBRARIES ARGS)
+    set(multiValueArgs SOURCES LIBRARIES ARGS LAUNCH_CMD)
     cmake_parse_arguments(AKTUALIZR_TEST "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-    add_executable(t_${AKTUALIZR_TEST_NAME} EXCLUDE_FROM_ALL ${AKTUALIZR_TEST_SOURCES} ${PROJECT_SOURCE_DIR}/tests/test_utils.cc)
-    target_link_libraries(t_${AKTUALIZR_TEST_NAME}
+    set(TEST_TARGET t_${AKTUALIZR_TEST_NAME})
+    add_executable(${TEST_TARGET} EXCLUDE_FROM_ALL ${AKTUALIZR_TEST_SOURCES} ${PROJECT_SOURCE_DIR}/tests/test_utils.cc)
+    target_link_libraries(${TEST_TARGET}
         ${AKTUALIZR_TEST_LIBRARIES}
         aktualizr_static_lib
         ${TEST_LIBS})
-    target_include_directories(t_${AKTUALIZR_TEST_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/tests)
+    target_include_directories(${TEST_TARGET} PUBLIC ${PROJECT_SOURCE_DIR}/tests)
 
     if(AKTUALIZR_TEST_PROJECT_WORKING_DIRECTORY)
         set(WD WORKING_DIRECTORY ${PROJECT_SOURCE_DIR})
     else()
         set(WD )
     endif()
 
-    # run tests under valgrind if the correct CMAKE_BUILD_TYPE is set
     if(TESTSUITE_VALGRIND AND (NOT AKTUALIZR_TEST_NO_VALGRIND))
-        add_test(NAME test_${AKTUALIZR_TEST_NAME}
-                 COMMAND ${RUN_VALGRIND} ${CMAKE_CURRENT_BINARY_DIR}/t_${AKTUALIZR_TEST_NAME} ${AKTUALIZR_TEST_ARGS} ${GOOGLE_TEST_OUTPUT}
-                 ${WD})
-    else()
-        add_test(NAME test_${AKTUALIZR_TEST_NAME}
-                 COMMAND t_${AKTUALIZR_TEST_NAME} ${AKTUALIZR_TEST_ARGS} ${GOOGLE_TEST_OUTPUT}
-                 ${WD})
+        list(INSERT CMD_PREFIX 0 ${RUN_VALGRIND})
+    endif()
+
+    if(AKTUALIZR_TEST_LAUNCH_CMD)
+        list(INSERT CMD_PREFIX 0 ${AKTUALIZR_TEST_LAUNCH_CMD})
     endif()
 
-    add_dependencies(build_tests t_${AKTUALIZR_TEST_NAME})
+    add_test(NAME test_${AKTUALIZR_TEST_NAME}
+             COMMAND ${CMD_PREFIX} $<TARGET_FILE:${TEST_TARGET}> ${AKTUALIZR_TEST_ARGS} ${GOOGLE_TEST_OUTPUT} ${WD})
+
+    add_dependencies(build_tests ${TEST_TARGET})
     set(TEST_SOURCES ${TEST_SOURCES} ${AKTUALIZR_TEST_SOURCES} PARENT_SCOPE)
 endfunction(add_aktualizr_test)

config/sql/migration/migrate.19.sql

@@ -0,0 +1,13 @@
+-- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
+SAVEPOINT MIGRATION;
+
+CREATE TABLE ecu_serials_migrate(id INTEGER PRIMARY KEY, serial TEXT UNIQUE, hardware_id TEXT NOT NULL, is_primary INTEGER NOT NULL DEFAULT 0 CHECK (is_primary IN (0,1)));
+INSERT INTO ecu_serials_migrate(serial, hardware_id, is_primary) SELECT ecu_serials.serial, ecu_serials.hardware_id, ecu_serials.is_primary FROM ecu_serials ORDER BY is_primary DESC, ecu_serials.rowid;
+
+DROP TABLE ecu_serials;
+ALTER TABLE ecu_serials_migrate RENAME TO ecu_serials;
+
+DELETE FROM version;
+INSERT INTO version VALUES(19);
+
+RELEASE MIGRATION;

config/sql/rollback/rollback.19.sql

@@ -0,0 +1,13 @@
+-- Don't modify this! Create a new migration instead--see docs/schema-migrations.adoc
+SAVEPOINT ROLLBACK_MIGRATION;
+
+CREATE TABLE ecu_serials_migrate(serial TEXT UNIQUE, hardware_id TEXT NOT NULL, is_primary INTEGER NOT NULL CHECK (is_primary IN (0,1)));
+INSERT INTO ecu_serials_migrate(serial, hardware_id, is_primary) SELECT ecu_serials.serial, ecu_serials.hardware_id, ecu_serials.is_primary FROM ecu_serials ORDER BY ecu_serials.id;
+
+DROP TABLE ecu_serials;
+ALTER TABLE ecu_serials_migrate RENAME TO ecu_serials;
+
+DELETE FROM version;
+INSERT INTO version VALUES(18);
+
+RELEASE ROLLBACK_MIGRATION;

config/sql/schema.sql

@@ -1,7 +1,7 @@
 CREATE TABLE version(version INTEGER);
-INSERT INTO version(rowid,version) VALUES(1,18);
+INSERT INTO version(rowid,version) VALUES(1,19);
 CREATE TABLE device_info(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), device_id TEXT, is_registered INTEGER NOT NULL DEFAULT 0 CHECK (is_registered IN (0,1)));
-CREATE TABLE ecu_serials(serial TEXT UNIQUE, hardware_id TEXT NOT NULL, is_primary INTEGER NOT NULL CHECK (is_primary IN (0,1)));
+CREATE TABLE ecu_serials(id INTEGER PRIMARY KEY, serial TEXT UNIQUE, hardware_id TEXT NOT NULL, is_primary INTEGER NOT NULL DEFAULT 0 CHECK (is_primary IN (0,1)));
 CREATE TABLE misconfigured_ecus(serial TEXT UNIQUE, hardware_id TEXT NOT NULL, state INTEGER NOT NULL CHECK (state IN (0,1)));
 CREATE TABLE installed_versions(ecu_serial TEXT NOT NULL, sha256 TEXT NOT NULL, name TEXT NOT NULL, hashes TEXT NOT NULL, length INTEGER NOT NULL DEFAULT 0, correlation_id TEXT NOT NULL DEFAULT '', is_current INTEGER NOT NULL CHECK (is_current IN (0,1)) DEFAULT 0, is_pending INTEGER NOT NULL CHECK (is_pending IN (0,1)) DEFAULT 0, UNIQUE(ecu_serial, sha256, name));
 CREATE TABLE primary_keys(unique_mark INTEGER PRIMARY KEY CHECK (unique_mark = 0), private TEXT, public TEXT);

docs/fault-injection.adoc

@@ -18,6 +18,12 @@ For example, when using the fake package manager:
 
     fiu-run -c 'enable name=fake_package_install' aktualizr -c . once
 
+Our wrapper script in `./scripts/fiu` can be used in place, with the added feature of passing strings in failinfo parameters.
+
+Usage is as follow:
+
+    ./scripts/fiu run -c 'enable name=fake_package_install,failinfo=reason' -- aktualizr -c . once
+
 == List of fail points
 
 Please try to keep this list up-to-date when inserting/removing fail points.
@@ -48,12 +54,12 @@ It runs the docker image inside a container with the same permissions as the loc
 
 Then, let's launch aktualizr with `fiu-run`:
 
-    docker run --name aktualizr-fiu -u $(id -u):$(id -g) -w $PWD -v $PWD:$PWD advancedtelematic/aktualizr-app fiu-run aktualizr -c .
+    docker run --name aktualizr-fiu --rm -u $(id -u):$(id -g) -w $PWD -v $PWD:$PWD advancedtelematic/aktualizr-app fiu run -- aktualizr -c .
 
-You can try to install a package now, which will succeed. To make all subsequent installations fail, use:
+You can try to install a package now, which will succeed. To make all subsequent installations fail with the reason "TEST_FAILURE", use:
 
-    docker exec aktualizr-fiu fiu-ctrl -c 'enable name=fake_package_install' 1
+    docker exec aktualizr-fiu fiu ctrl -c 'enable name=fake_package_install,failinfo=TEST_FAILURE' 1
 
 To make installations succeed again:
 
-    docker exec aktualizr-fiu fiu-ctrl -c 'disable name=fake_package_install' 1
+    docker exec aktualizr-fiu fiu ctrl -c 'disable name=fake_package_install' 1

scripts/fiu

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+
+"""fiu wrapper script for string failinfos
+
+This programs wraps fiu-run(1) and fiu-ctrl(1), with the added support of passing arbitrary
+strings (of limited length) to programs with appropriate support (e.g: using "fault_injection.h"
+in this repository).
+
+Usage is `fiu run` instead of `fiu-run` and `fiu ctrl` instead of `fiu-ctrl`, with the added
+caveat that the program invocation part should be separated by `--` if it contains options
+parsed by the corresponding fiu tool.
+
+For example:
+
+    fiu run -c 'enable name=xxx' -- aktualizr -c config_dir
+
+As programs compiled with fiu can only receive integers through the failinfo value, it uses an
+intermediary file as a storage and interprets the integer as some index in the file (actually,
+the id of a 256B sized block).
+
+Note: care should be taken to run multiple `fiu ctrl` sequentially (including from application's
+code!), as there is no lock on the data files.
+"""
+
+import argparse
+import os
+import re
+import tempfile
+
+
+def allocate_info(info_fn, info_str):
+    bs = 256
+    if len(info_str) > bs:
+        raise RuntimeError('Too big info')
+
+    with open(info_fn, 'ab') as f:
+        off = f.tell()
+        f.write(info_str.encode())
+        f.write((bs - len(info_str)) * b'\x00')
+    bid = off // bs
+
+    if bid >= 1 << 31:
+        raise RuntimeError('Info id too big: {}'.format(bid))
+
+    return bid
+
+
+def convert_failinfo(info_fn, instr, high_id=False):
+    def matchf(m):
+        info_id = allocate_info(info_fn, m.group('info'))
+        # high bit controls if the failure is in 'environment' or 'pid' file
+        if high_id:
+            info_id |= 1 << 31
+        return 'failinfo={}'.format(info_id)
+    r = re.sub(r'failinfo=(?P<info>[^,]*)', matchf, instr)
+
+    return r
+
+
+def collect_other_args(args):
+    others = []
+    if hasattr(args, 'f') and args.f is not None:
+        others += ['-f', args.f]
+    if hasattr(args, 'l') and args.l:
+        others += ['-l', args.l]
+    if hasattr(args, 'x') and args.x:
+        others += ['-x']
+    if hasattr(args, 'n') and args.n:
+        others += ['-n']
+
+    return others
+
+
+def do_run(args, *kargs):
+    # Pass failinfo strings in a temporary file whose name is passed through an
+    # environment variable.
+    # It cannot use the pid of the target process because it is not known yet
+    # and trying to get it after launch would lead to potential races.
+
+    # temporary file will be leaked, we can't clean after run because we're exec-ing
+    tf = tempfile.NamedTemporaryFile(prefix='fiu-ctrl-info-')
+    c_with_id = []
+    for c in args.c:
+        c_with_id += ['-c', convert_failinfo(tf.name, c, True)]
+
+    nenv = os.environ.copy()
+    nenv["FIU_INFO_FILE"] = tf.name
+
+    # use exec so that the pid is unchanged (as fiu-run does)
+    # it's especially useful in the dockerized case where keeping pid 1 makes
+    # everything smoother
+    cmd = ["fiu-run", *c_with_id, *collect_other_args(args), *kargs]
+    print('Running: ' + ' '.join(cmd))
+    os.execvpe(cmd[0], cmd, nenv)
+
+
+def do_ctrl(args, *kargs):
+    # Pass failinfo strings in a file whose name can be deducted from the
+    # program's pid
+    pids = args.pid
+    if len(pids) != 1:
+        raise RuntimeError('Can only do one pid at once!')
+    pid = pids[0]
+
+    info_fn = '/tmp/fiu-ctrl-info-{}'.format(pid)
+    c_with_id = []
+    for c in args.c:
+        c_with_id += ['-c', convert_failinfo(info_fn, c)]
+
+    cmd = ["fiu-ctrl", *c_with_id, *collect_other_args(args), pid]
+    print('Running: ' + ' '.join(cmd))
+    os.execvp(cmd[0], cmd)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Proxy for fiu-run and fiu-ctrl')
+    parser.set_defaults(func=None)
+    subparsers = parser.add_subparsers()
+
+    parser_run = subparsers.add_parser('run')
+    parser_run.add_argument('-c', type=str, action='append', default=[])
+    parser_run.add_argument('-n', action='store_true')
+    parser_run.add_argument('-x', action='store_true')
+    parser_run.add_argument('-f', type=str)
+    parser_run.add_argument('-l', type=str)
+    parser_run.set_defaults(func=do_run)
+
+    parser_ctrl = subparsers.add_parser('ctrl')
+    parser_ctrl.add_argument('-c', type=str, action='append', default=[])
+    parser_ctrl.add_argument('-n', action='store_true')
+    parser_ctrl.add_argument('-f', type=str)
+    parser_ctrl.add_argument('pid', type=str, nargs=1)
+    parser_ctrl.set_defaults(func=do_ctrl)
+    args, passthrough = parser.parse_known_args()
+
+    if args.func is None:
+        parser.print_help()
+        return 0
+
+    return args.func(args, *passthrough)
+
+
+if __name__ == '__main__':
+    main()

src/libaktualizr/package_manager/packagemanagerfake.cc

@@ -26,12 +26,12 @@ Uptane::Target PackageManagerFake::getCurrent() const {
 data::InstallationResult PackageManagerFake::install(const Uptane::Target &target) const {
   // fault injection: only enabled with FIU_ENABLE defined
   if (fiu_fail("fake_package_install") != 0) {
-    std::string failure_cause = fault_injection_get_parameter("fault_fake_package_install_cause");
+    std::string failure_cause = fault_injection_last_info();
     if (failure_cause.empty()) {
-      failure_cause = "Installation failed";
+      return data::InstallationResult(data::ResultCode::Numeric::kInstallFailed, "");
     }
     LOG_DEBUG << "Causing installation failure with message: " << failure_cause;
-    return data::InstallationResult(data::ResultCode::Numeric::kInstallFailed, failure_cause);
+    return data::InstallationResult(data::ResultCode(data::ResultCode::Numeric::kInstallFailed, failure_cause), "");
   }
 
   if (config.fake_need_reboot) {
@@ -46,7 +46,7 @@ data::InstallationResult PackageManagerFake::install(const Uptane::Target &targe
   }
 
   storage_->savePrimaryInstalledVersion(target, InstalledVersionUpdateMode::kCurrent);
-  return data::InstallationResult(data::ResultCode::Numeric::kOk, "Installing fake package was successful");
+  return data::InstallationResult(data::ResultCode::Numeric::kOk, "Installing package was successful");
 }
 
 void PackageManagerFake::completeInstall() const {

src/libaktualizr/package_manager/packagemanagerfake_test.cc

@@ -36,7 +36,6 @@ TEST(PackageManagerFake, FinalizeAfterReboot) {
 
 #ifdef FIU_ENABLE
 
-#include <fiu-control.h>
 #include "utilities/fault_injection.h"
 
 TEST(PackageManagerFake, FailureInjection) {
@@ -56,19 +55,16 @@ TEST(PackageManagerFake, FailureInjection) {
   EXPECT_EQ(result.result_code, data::ResultCode::Numeric::kOk);
 
   // fault
-  fiu_enable("fake_package_install", 1, nullptr, 0);
+  fault_injection_enable("fake_package_install", 1, "", 0);
   result = fakepm.install(target);
   EXPECT_EQ(result.result_code, data::ResultCode::Numeric::kInstallFailed);
-  fiu_disable("fake_package_install");
+  fault_injection_disable("fake_package_install");
 
-  // fault with custom data (through environment)
-  fiu_enable("fake_package_install", 1, nullptr, 0);
-  fault_injection_set_parameter("fault_fake_package_install_cause", "Random cause");
+  // fault with custom data (through pid file)
+  fault_injection_enable("fake_package_install", 1, "RANDOM_CAUSE", 0);
   result = fakepm.install(target);
-  EXPECT_EQ(result.result_code, data::ResultCode::Numeric::kInstallFailed);
-  EXPECT_EQ(result.description, "Random cause");
-  fault_injection_set_parameter("fault_fake_package_install_cause", "");
-  fiu_disable("fake_package_install");
+  EXPECT_EQ(result.result_code, data::ResultCode(data::ResultCode::Numeric::kInstallFailed, "RANDOM_CAUSE"));
+  fault_injection_disable("fake_package_install");
 }
 
 #endif  // FIU_ENABLE

src/libaktualizr/primary/aktualizr.h

@@ -187,6 +187,7 @@ class Aktualizr {
 
  private:
   FRIEND_TEST(Aktualizr, FullNoUpdates);
+  FRIEND_TEST(Aktualizr, DeviceInstallationResult);
   FRIEND_TEST(Aktualizr, FullWithUpdates);
   FRIEND_TEST(Aktualizr, FullWithUpdatesNeedReboot);
   FRIEND_TEST(Aktualizr, AutoRebootAfterUpdate);