diff --git a/core/tests/gateway_e2e/module_config_windows.c b/core/tests/gateway_e2e/module_config_windows.c index 714fe6fd..c4a097f0 100644 --- a/core/tests/gateway_e2e/module_config_windows.c +++ b/core/tests/gateway_e2e/module_config_windows.c @@ -1,11 +1,17 @@ -// Copyright (c) Microsoft. All rights reserved. +// Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. #include "gateway.h" +#ifdef _DEBUG const char* e2e_module_path_string = "..\\e2e_module\\Debug\\e2e_module.dll"; const char* iothub_path_string = "..\\..\\..\\..\\modules\\iothub\\Debug\\iothub.dll"; const char* identity_map_path_string = "..\\..\\..\\..\\modules\\identitymap\\Debug\\identity_map.dll"; +#else +const char* e2e_module_path_string = "..\\e2e_module\\Release\\e2e_module.dll"; +const char* iothub_path_string = "..\\..\\..\\..\\modules\\iothub\\Release\\iothub.dll"; +const char* identity_map_path_string = "..\\..\\..\\..\\modules\\identitymap\\Release\\identity_map.dll"; +#endif const char* e2e_module_path() { diff --git a/core/tests/performance_e2e/CMakeLists.txt b/core/tests/performance_e2e/CMakeLists.txt index 00cc9a35..1a05c1f6 100644 --- a/core/tests/performance_e2e/CMakeLists.txt +++ b/core/tests/performance_e2e/CMakeLists.txt @@ -60,6 +60,7 @@ if(install_modules) endif() +# This builds the command line tool. set(performance_e2e_sources ./src/main.cpp ) @@ -89,3 +90,65 @@ copy_gateway_dll(performance_e2e ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration) ) set_target_properties(performance_e2e PROPERTIES FOLDER "tests/E2ETests") + +# Run E2E as a test. + +set(theseTestsName performance_e2e) + +#Disable valgrind just for performance E2E test +set(run_valgrind OFF) + +compileAsC99() + +set(${theseTestsName}_test_files + ./src/${theseTestsName}.c +) + +#setting the dynamic_loader file based on OS that it is used +if(WIN32) + set(modules_c_file ./src/module_config_windows.c) +elseif(UNIX) # LINUX OR APPLE + set(modules_c_file ./src/module_config_linux.c) +endif() + + +set(${theseTestsName}_c_files + ${modules_c_file} +) + +set(${theseTestsName}_h_files + ./inc/module_config_resources.h +) + +build_c_test_artifacts(${theseTestsName} ON "tests/E2ETests") +set_target_properties( + ${theseTestsName}_exe PROPERTIES + BUILD_WITH_INSTALL_RPATH TRUE +) + +linkSharedUtil(${theseTestsName}_exe) + +if(WIN32) + if(TARGET ${theseTestsName}_dll) + target_link_libraries(${theseTestsName}_dll + gateway + ) + endif() + + if(TARGET ${theseTestsName}_exe) + target_link_libraries(${theseTestsName}_exe + gateway + ) + install_broker(${theseTestsName}_exe ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration) ) + copy_gateway_dll(${theseTestsName}_exe ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration) ) + endif() +else() + if(TARGET ${theseTestsName}_exe) + target_link_libraries(${theseTestsName}_exe + gateway + aziotsharedutil + pthread + ) + endif() + +endif() diff --git a/core/tests/performance_e2e/README.md b/core/tests/performance_e2e/README.md index 7bba5af2..729b27bd 100644 --- a/core/tests/performance_e2e/README.md +++ b/core/tests/performance_e2e/README.md @@ -386,9 +386,14 @@ resources allocated in `moduleHandle`. ## Running the performance test. +To run on the command line, use the `performance_e2e` executable. + The program accepts 2 command line arguments, the json configuration file and an optional test duration in seconds. The sample creates the gateway from the JSON file, starts all modules, pauses for a given time (default of 5 seconds), and stops the gateway. Stopping the gateway will trigger the metrics module to report message statistics. +A 5 second and 10 second performance test are run as part of the build tests. +run `ctest -C Debug -V -R performance_e2e` to execute those tests. + diff --git a/core/tests/performance_e2e/inc/module_config_resources.h b/core/tests/performance_e2e/inc/module_config_resources.h new file mode 100644 index 00000000..583906ac --- /dev/null +++ b/core/tests/performance_e2e/inc/module_config_resources.h @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#ifndef MODULE_CONFIG_RESOURCES_H +#define MODULE_CONFIG_RESOURCES_H + +#ifdef __cplusplus +extern "C" +{ +#endif +//Add here paths for any new Module. +const char* simulator_module_path(); +const char* metrics_module_path(); +#ifdef __cplusplus +} +#endif + +#endif /*MODULE_CONFIG_RESOURCES_H*/ \ No newline at end of file diff --git a/core/tests/performance_e2e/main.c b/core/tests/performance_e2e/main.c new file mode 100644 index 00000000..3ae2283a --- /dev/null +++ b/core/tests/performance_e2e/main.c @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#include "testrunnerswitcher.h" + +int main(void) +{ + size_t failedTestCount = 0; + RUN_TEST_SUITE(Performance_e2e, failedTestCount); + return failedTestCount; +} diff --git a/core/tests/performance_e2e/src/metrics.cpp b/core/tests/performance_e2e/src/metrics.cpp index 359244b1..bb7d81c3 100644 --- a/core/tests/performance_e2e/src/metrics.cpp +++ b/core/tests/performance_e2e/src/metrics.cpp @@ -241,6 +241,7 @@ static void MetricsModule_Destroy(MODULE_HANDLE moduleHandle) << "Messages Lost: " << (*d).second.messages_lost << std::endl; } } + delete (module->per_device_metrics); free(module); } } diff --git a/core/tests/performance_e2e/src/module_config_linux.c b/core/tests/performance_e2e/src/module_config_linux.c new file mode 100644 index 00000000..5a9a85f9 --- /dev/null +++ b/core/tests/performance_e2e/src/module_config_linux.c @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +const char* simulator_module_path_string = "libsimulator.so"; +const char* metrics_module_path_string = "libmetrics.so"; + +const char* simulator_module_path() +{ + return simulator_module_path_string; +} + +const char* metrics_module_path() +{ + return metrics_module_path_string; +} diff --git a/core/tests/performance_e2e/src/module_config_windows.c b/core/tests/performance_e2e/src/module_config_windows.c new file mode 100644 index 00000000..0b784b44 --- /dev/null +++ b/core/tests/performance_e2e/src/module_config_windows.c @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#include "gateway.h" + +#ifdef _DEBUG +const char* simulator_module_path_string = "Debug\\simulator.dll"; +const char* metrics_module_path_string = "Debug\\metrics.dll"; +#else +const char* simulator_module_path_string = "Release\\simulator.dll"; +const char* metrics_module_path_string = "Release\\metrics.dll"; +#endif + +const char* simulator_module_path() +{ + return simulator_module_path_string; +} + +const char* metrics_module_path() +{ + return metrics_module_path_string; +} \ No newline at end of file diff --git a/core/tests/performance_e2e/src/performance_e2e.c b/core/tests/performance_e2e/src/performance_e2e.c new file mode 100644 index 00000000..c4ca3065 --- /dev/null +++ b/core/tests/performance_e2e/src/performance_e2e.c @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +#include "gateway.h" +#include "module_config_resources.h" +#include "simulator.h" +#include "module_loader.h" +#include "module_loaders/dynamic_loader.h" +#include "azure_c_shared_utility/threadapi.h" + +#include "testrunnerswitcher.h" + +//============================================================================= +//Globals +//============================================================================= + +#ifdef WIN32 +static TEST_MUTEX_HANDLE g_dllByDll; +#endif +static TEST_MUTEX_HANDLE g_testByTest; + +BEGIN_TEST_SUITE(Performance_e2e) + +TEST_SUITE_INITIALIZE(TestClassInitialize) +{ + TEST_INITIALIZE_MEMORY_DEBUG(g_dllByDll); + g_testByTest = TEST_MUTEX_CREATE(); + ASSERT_IS_NOT_NULL(g_testByTest); + +} + +TEST_SUITE_CLEANUP(TestClassCleanup) +{ + TEST_MUTEX_DESTROY(g_testByTest); + TEST_DEINITIALIZE_MEMORY_DEBUG(g_dllByDll); +} + +TEST_FUNCTION_INITIALIZE(TestMethodInitialize) +{ + if (TEST_MUTEX_ACQUIRE(g_testByTest) != 0) + { + ASSERT_FAIL("our mutex is ABANDONED. Failure in test framework"); + } + +} + +TEST_FUNCTION_CLEANUP(TestMethodCleanup) +{ + TEST_MUTEX_RELEASE(g_testByTest); +} + +TEST_FUNCTION(Performance_e2e_5_second_run) +{ + ///arrange + GATEWAY_HANDLE e2eGatewayInstance; + + /* Setup: data for simulator module */ + SIMULATOR_MODULE_CONFIG simulator_config = + { + "device1", + 0, + 2, + 16, + 256 + }; + + GATEWAY_MODULES_ENTRY modules[2]; + DYNAMIC_LOADER_ENTRYPOINT loader_info[2]; + GATEWAY_LINK_ENTRY links[1]; + + // simulator + modules[0].module_name = "simulator1"; + modules[0].module_configuration = &simulator_config; + modules[0].module_loader_info.loader = DynamicLoader_Get(); + loader_info[0].moduleLibraryFileName = STRING_construct(simulator_module_path()); + modules[0].module_loader_info.entrypoint = (void*)&(loader_info[0]); + + // metrics + modules[1].module_name = "metrics1"; + modules[1].module_configuration = NULL; + modules[1].module_loader_info.loader = DynamicLoader_Get(); + loader_info[1].moduleLibraryFileName = STRING_construct(metrics_module_path()); + modules[1].module_loader_info.entrypoint = (void*)&(loader_info[1]); + + links[0].module_source = "simulator1"; + links[0].module_sink = "metrics1"; + + GATEWAY_PROPERTIES performance_gw_properties; + VECTOR_HANDLE gatewayProps = VECTOR_create(sizeof(GATEWAY_MODULES_ENTRY)); + VECTOR_HANDLE gatewayLinks = VECTOR_create(sizeof(GATEWAY_LINK_ENTRY)); + + VECTOR_push_back(gatewayProps, &modules, 2); + VECTOR_push_back(gatewayLinks, &links, 1); + + + + ///act + performance_gw_properties.gateway_modules = gatewayProps; + performance_gw_properties.gateway_links = gatewayLinks; + e2eGatewayInstance = Gateway_Create(&performance_gw_properties); + GATEWAY_START_RESULT start_result = Gateway_Start(e2eGatewayInstance); + + ///assert + ASSERT_IS_NOT_NULL(e2eGatewayInstance); + ASSERT_IS_TRUE((start_result == GATEWAY_START_SUCCESS)); + + + ThreadAPI_Sleep(5000); + + Gateway_Destroy(e2eGatewayInstance); + + VECTOR_destroy(gatewayProps); + VECTOR_destroy(gatewayLinks); + + for (int loader = 0; loader < 2; loader++) + { + STRING_delete(loader_info[loader].moduleLibraryFileName); + } +} + +TEST_FUNCTION(Performance_e2e_10_second_run) +{ + ///arrange + GATEWAY_HANDLE e2eGatewayInstance; + + /* Setup: data for simulator module */ + SIMULATOR_MODULE_CONFIG simulator_config = + { + "device1", + 0, + 2, + 16, + 256 + }; + + GATEWAY_MODULES_ENTRY modules[2]; + DYNAMIC_LOADER_ENTRYPOINT loader_info[2]; + GATEWAY_LINK_ENTRY links[1]; + + // simulator + modules[0].module_name = "simulator1"; + modules[0].module_configuration = &simulator_config; + modules[0].module_loader_info.loader = DynamicLoader_Get(); + loader_info[0].moduleLibraryFileName = STRING_construct(simulator_module_path()); + modules[0].module_loader_info.entrypoint = (void*)&(loader_info[0]); + + // metrics + modules[1].module_name = "metrics1"; + modules[1].module_configuration = NULL; + modules[1].module_loader_info.loader = DynamicLoader_Get(); + loader_info[1].moduleLibraryFileName = STRING_construct(metrics_module_path()); + modules[1].module_loader_info.entrypoint = (void*)&(loader_info[1]); + + links[0].module_source = "simulator1"; + links[0].module_sink = "metrics1"; + + GATEWAY_PROPERTIES performance_gw_properties; + VECTOR_HANDLE gatewayProps = VECTOR_create(sizeof(GATEWAY_MODULES_ENTRY)); + VECTOR_HANDLE gatewayLinks = VECTOR_create(sizeof(GATEWAY_LINK_ENTRY)); + + VECTOR_push_back(gatewayProps, &modules, 2); + VECTOR_push_back(gatewayLinks, &links, 1); + + + + ///act + performance_gw_properties.gateway_modules = gatewayProps; + performance_gw_properties.gateway_links = gatewayLinks; + e2eGatewayInstance = Gateway_Create(&performance_gw_properties); + GATEWAY_START_RESULT start_result = Gateway_Start(e2eGatewayInstance); + + ///assert + ASSERT_IS_NOT_NULL(e2eGatewayInstance); + ASSERT_IS_TRUE((start_result == GATEWAY_START_SUCCESS)); + + + ThreadAPI_Sleep(10000); + + Gateway_Destroy(e2eGatewayInstance); + + VECTOR_destroy(gatewayProps); + VECTOR_destroy(gatewayLinks); + + for (int loader = 0; loader < 2; loader++) + { + STRING_delete(loader_info[loader].moduleLibraryFileName); + } + +} + + +END_TEST_SUITE(Performance_e2e); diff --git a/core/tests/performance_e2e/src/simulator.cpp b/core/tests/performance_e2e/src/simulator.cpp index 763a7e60..a36b6c52 100644 --- a/core/tests/performance_e2e/src/simulator.cpp +++ b/core/tests/performance_e2e/src/simulator.cpp @@ -275,6 +275,14 @@ static int SimulatorModule_thread(void * context) if (thread_result != 0) { LogError("unable to continue with simulation"); + if (message_to_send.sourceProperties != NULL) + { + Map_Destroy(message_to_send.sourceProperties); + } + if (message_to_send.source != NULL) + { + free( (void*)message_to_send.source); + } } else { @@ -338,6 +346,14 @@ static int SimulatorModule_thread(void * context) } } } + if (message_to_send.sourceProperties != NULL) + { + Map_Destroy(message_to_send.sourceProperties); + } + if (message_to_send.source != NULL) + { + free((void*)message_to_send.source); + } } return thread_result; } diff --git a/jenkins/linux_c.sh b/jenkins/linux_c.sh index 14f330ac..c07f9737 100755 --- a/jenkins/linux_c.sh +++ b/jenkins/linux_c.sh @@ -6,6 +6,6 @@ build_root=$(cd "$(dirname "$0")/.." && pwd) cd $build_root # -- C -- -./tools/build.sh --run-unittests --enable-nodejs-binding --enable-java-binding --enable-java-remote-modules --enable-dotnet-core-binding "$@" #-x +./tools/build.sh --run-unittests --run-e2e-tests --enable-nodejs-binding --enable-java-binding --enable-java-remote-modules --enable-dotnet-core-binding "$@" #-x [ $? -eq 0 ] || exit $? \ No newline at end of file diff --git a/jenkins/ubuntu-14.04_c.sh b/jenkins/ubuntu-14.04_c.sh index 6bef8f1a..d61d365c 100755 --- a/jenkins/ubuntu-14.04_c.sh +++ b/jenkins/ubuntu-14.04_c.sh @@ -6,6 +6,6 @@ build_root=$(cd "$(dirname "$0")/.." && pwd) cd $build_root # -- C -- -./tools/build.sh --run-unittests --enable-nodejs-binding --enable-java-binding --disable-native-remote-modules --enable-dotnet-core-binding "$@" #-x +./tools/build.sh --run-unittests --run-e2e-tests --enable-nodejs-binding --enable-java-binding --disable-native-remote-modules --enable-dotnet-core-binding "$@" #-x [ $? -eq 0 ] || exit $? \ No newline at end of file diff --git a/tools/build.sh b/tools/build.sh index 53023a8b..aa6a7a20 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -193,10 +193,10 @@ then then #use doctored (-DPURIFY no-asm) openssl export LD_LIBRARY_PATH=/usr/local/ssl/lib - ctest -j $CORES --output-on-failure + ctest -V -j $CORES --output-on-failure export LD_LIBRARY_PATH= else - ctest -j $CORES -C $build_config --output-on-failure + ctest -V -j $CORES -C $build_config --output-on-failure fi fi