diff --git a/CMakeLists.txt b/CMakeLists.txt index 5296f0209c5..54f08c1f08d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -262,6 +262,7 @@ if (WITH_GUI) add_subdirectory(outlineviewer) if (LIBSSH_FOUND) add_subdirectory(roborioteamnumbersetter) + add_subdirectory(datalogtool) endif() endif() diff --git a/datalogtool/.styleguide b/datalogtool/.styleguide new file mode 100644 index 00000000000..87142fbc90a --- /dev/null +++ b/datalogtool/.styleguide @@ -0,0 +1,29 @@ +cppHeaderFileInclude { + \.h$ + \.inc$ + \.inl$ +} + +cppSrcFileInclude { + \.cpp$ +} + +generatedFileExclude { + src/main/native/resources/ + src/main/native/win/datalogtool.ico + src/main/native/mac/datalogtool.icns +} + +repoRootNameOverride { + datalogtool +} + +includeOtherLibs { + ^GLFW + ^fmt/ + ^glass/ + ^imgui + ^portable-file-dialog + ^wpi/ + ^wpigui +} diff --git a/datalogtool/CMakeLists.txt b/datalogtool/CMakeLists.txt new file mode 100644 index 00000000000..a5e23581e45 --- /dev/null +++ b/datalogtool/CMakeLists.txt @@ -0,0 +1,29 @@ +project(datalogtool) + +include(CompileWarnings) +include(GenResources) +include(LinkMacOSGUI) + +configure_file(src/main/generate/WPILibVersion.cpp.in WPILibVersion.cpp) +GENERATE_RESOURCES(src/main/native/resources generated/main/cpp DLT dlt datalogtool_resources_src) + +file(GLOB datalogtool_src src/main/native/cpp/*.cpp ${CMAKE_CURRENT_BINARY_DIR}/WPILibVersion.cpp) + +if (WIN32) + set(datalogtool_rc src/main/native/win/datalogtool.rc) +elseif(APPLE) + set(MACOSX_BUNDLE_ICON_FILE datalogtool.icns) + set(APP_ICON_MACOSX src/main/native/mac/datalogtool.icns) + set_source_files_properties(${APP_ICON_MACOSX} PROPERTIES MACOSX_PACKAGE_LOCATION "Resources") +endif() + +add_executable(datalogtool ${datalogtool_src} ${datalogtool_resources_src} ${datalogtool_rc} ${APP_ICON_MACOSX}) +wpilib_link_macos_gui(datalogtool) +target_link_libraries(datalogtool libglass ${LIBSSH_LIBRARIES}) +target_include_directories(datalogtool PRIVATE ${LIBSSH_INCLUDE_DIRS}) + +if (WIN32) + set_target_properties(datalogtool PROPERTIES WIN32_EXECUTABLE YES) +elseif(APPLE) + set_target_properties(datalogtool PROPERTIES MACOSX_BUNDLE YES OUTPUT_NAME "datalogTool") +endif() diff --git a/datalogtool/Info.plist b/datalogtool/Info.plist new file mode 100644 index 00000000000..db23e093738 --- /dev/null +++ b/datalogtool/Info.plist @@ -0,0 +1,32 @@ + + + + + CFBundleName + datalogTool + CFBundleExecutable + datalogtool + CFBundleDisplayName + datalogTool + CFBundleIdentifier + edu.wpi.first.tools.datalogTool + CFBundleIconFile + datalogtool.icns + CFBundlePackageType + APPL + CFBundleSupportedPlatforms + + MacOSX + + CFBundleInfoDictionaryVersion + 6.0 + CFBundleShortVersionString + 2021 + CFBundleVersion + 2021 + LSMinimumSystemVersion + 10.11 + NSHighResolutionCapable + + + diff --git a/datalogtool/build.gradle b/datalogtool/build.gradle new file mode 100644 index 00000000000..8c8fc6c7d46 --- /dev/null +++ b/datalogtool/build.gradle @@ -0,0 +1,134 @@ +import org.gradle.internal.os.OperatingSystem + +if (!project.hasProperty('onlylinuxathena') && !project.hasProperty('onlylinuxraspbian') && !project.hasProperty('onlylinuxaarch64bionic')) { + + description = "roboRIO Team Number Setter" + + apply plugin: 'cpp' + apply plugin: 'c' + apply plugin: 'google-test-test-suite' + apply plugin: 'visual-studio' + apply plugin: 'edu.wpi.first.NativeUtils' + + if (OperatingSystem.current().isWindows()) { + apply plugin: 'windows-resources' + } + + ext { + nativeName = 'datalogtool' + } + + apply from: "${rootDir}/shared/resources.gradle" + apply from: "${rootDir}/shared/config.gradle" + + def wpilibVersionFileInput = file("src/main/generate/WPILibVersion.cpp.in") + def wpilibVersionFileOutput = file("$buildDir/generated/main/cpp/WPILibVersion.cpp") + + nativeUtils { + nativeDependencyContainer { + libssh(getNativeDependencyTypeClass('WPIStaticMavenDependency')) { + groupId = "edu.wpi.first.thirdparty.frc2022" + artifactId = "libssh" + headerClassifier = "headers" + sourceClassifier = "sources" + ext = "zip" + version = '0.95-1' + targetPlatforms.addAll(nativeUtils.wpi.platforms.desktopPlatforms) + } + } + } + + task generateCppVersion() { + description = 'Generates the wpilib version class' + group = 'WPILib' + + outputs.file wpilibVersionFileOutput + inputs.file wpilibVersionFileInput + + if (wpilibVersioning.releaseMode) { + outputs.upToDateWhen { false } + } + + // We follow a simple set of checks to determine whether we should generate a new version file: + // 1. If the release type is not development, we generate a new version file + // 2. If there is no generated version number, we generate a new version file + // 3. If there is a generated build number, and the release type is development, then we will + // only generate if the publish task is run. + doLast { + def version = wpilibVersioning.version.get() + println "Writing version ${version} to $wpilibVersionFileOutput" + + if (wpilibVersionFileOutput.exists()) { + wpilibVersionFileOutput.delete() + } + def read = wpilibVersionFileInput.text.replace('${wpilib_version}', version) + wpilibVersionFileOutput.write(read) + } + } + + gradle.taskGraph.addTaskExecutionGraphListener { graph -> + def willPublish = graph.hasTask(publish) + if (willPublish) { + generateCppVersion.outputs.upToDateWhen { false } + } + } + + def generateTask = createGenerateResourcesTask('main', 'DLT', 'dlt', project) + + project(':').libraryBuild.dependsOn build + tasks.withType(CppCompile) { + dependsOn generateTask + dependsOn generateCppVersion + } + + model { + components { + // By default, a development executable will be generated. This is to help the case of + // testing specific functionality of the library. + "${nativeName}"(NativeExecutableSpec) { + baseName = 'datalogtool' + sources { + cpp { + source { + srcDirs 'src/main/native/cpp', "$buildDir/generated/main/cpp" + include '**/*.cpp' + } + exportedHeaders { + srcDirs 'src/main/native/include' + } + } + if (OperatingSystem.current().isWindows()) { + rc { + source { + srcDirs 'src/main/native/win' + include '*.rc' + } + } + } + } + binaries.all { + if (it.targetPlatform.name == nativeUtils.wpi.platforms.roborio || it.targetPlatform.name == nativeUtils.wpi.platforms.raspbian || it.targetPlatform.name == nativeUtils.wpi.platforms.aarch64bionic) { + it.buildable = false + return + } + it.cppCompiler.define("LIBSSH_STATIC") + lib project: ':glass', library: 'glass', linkage: 'static' + lib project: ':wpiutil', library: 'wpiutil', linkage: 'static' + lib project: ':wpigui', library: 'wpigui', linkage: 'static' + nativeUtils.useRequiredLibrary(it, 'imgui_static', 'libssh') + if (it.targetPlatform.operatingSystem.isWindows()) { + it.linker.args << 'Gdi32.lib' << 'Shell32.lib' << 'd3d11.lib' << 'd3dcompiler.lib' + it.linker.args << 'ws2_32.lib' << 'advapi32.lib' << 'crypt32.lib' << 'user32.lib' + } else if (it.targetPlatform.operatingSystem.isMacOsX()) { + it.linker.args << '-framework' << 'Metal' << '-framework' << 'MetalKit' << '-framework' << 'Cocoa' << '-framework' << 'IOKit' << '-framework' << 'CoreFoundation' << '-framework' << 'CoreVideo' << '-framework' << 'QuartzCore' + it.linker.args << '-framework' << 'Kerberos' + } else { + it.linker.args << '-lX11' + } + } + } + } + } + + apply from: 'publish.gradle' +} diff --git a/datalogtool/publish.gradle b/datalogtool/publish.gradle new file mode 100644 index 00000000000..131009cfc67 --- /dev/null +++ b/datalogtool/publish.gradle @@ -0,0 +1,107 @@ +apply plugin: 'maven-publish' + +def baseArtifactId = 'DataLogTool' +def artifactGroupId = 'edu.wpi.first.tools' +def zipBaseName = '_GROUP_edu_wpi_first_tools_ID_DataLogTool_CLS' + +def outputsFolder = file("$project.buildDir/outputs") + +model { + tasks { + // Create the run task. + $.components.datalogtool.binaries.each { bin -> + if (bin.buildable && bin.name.toLowerCase().contains("debug")) { + Task run = project.tasks.create("run", Exec) { + commandLine bin.tasks.install.runScriptFile.get().asFile.toString() + } + run.dependsOn bin.tasks.install + } + } + } + publishing { + def dataLogToolTaskList = [] + $.components.each { component -> + component.binaries.each { binary -> + if (binary in NativeExecutableBinarySpec && binary.component.name.contains("datalogtool")) { + if (binary.buildable && binary.name.contains("Release")) { + // We are now in the binary that we want. + // This is the default application path for the ZIP task. + def applicationPath = binary.executable.file + def icon = file("$project.projectDir/src/main/native/mac/datalogtool.icns") + + // Create the macOS bundle. + def bundleTask = project.tasks.create("bundleDataLogToolOsxApp", Copy) { + description("Creates a macOS application bundle for DataLogTool") + from(file("$project.projectDir/Info.plist")) + into(file("$project.buildDir/outputs/bundles/DataLogTool.app/Contents")) + into("MacOS") { with copySpec { from binary.executable.file } } + into("Resources") { with copySpec { from icon } } + + doLast { + if (project.hasProperty("developerID")) { + // Get path to binary. + exec { + workingDir rootDir + def args = [ + "sh", + "-c", + "codesign --force --strict --deep " + + "--timestamp --options=runtime " + + "--verbose -s ${project.findProperty("developerID")} " + + "$project.buildDir/outputs/bundles/DataLogTool.app/" + ] + commandLine args + } + } + } + } + + // Reset the application path if we are creating a bundle. + if (binary.targetPlatform.operatingSystem.isMacOsX()) { + applicationPath = file("$project.buildDir/outputs/bundles") + project.build.dependsOn bundleTask + } + + // Create the ZIP. + def task = project.tasks.create("copyDataLogToolExecutable", Zip) { + description("Copies the DataLogTool executable to the outputs directory.") + destinationDirectory = outputsFolder + + archiveBaseName = '_M_' + zipBaseName + duplicatesStrategy = 'exclude' + classifier = nativeUtils.getPublishClassifier(binary) + + from(licenseFile) { + into '/' + } + + from(applicationPath) + into(nativeUtils.getPlatformPath(binary)) + } + + if (binary.targetPlatform.operatingSystem.isMacOsX()) { + bundleTask.dependsOn binary.tasks.link + task.dependsOn(bundleTask) + } + + task.dependsOn binary.tasks.link + dataLogToolTaskList.add(task) + project.build.dependsOn task + project.artifacts { task } + addTaskToCopyAllOutputs(task) + } + } + } + } + + publications { + datalogtool(MavenPublication) { + dataLogToolTaskList.each { artifact it } + + artifactId = baseArtifactId + groupId = artifactGroupId + version wpilibVersioning.version.get() + } + } + } +} diff --git a/datalogtool/src/main/generate/WPILibVersion.cpp.in b/datalogtool/src/main/generate/WPILibVersion.cpp.in new file mode 100644 index 00000000000..b0a44905207 --- /dev/null +++ b/datalogtool/src/main/generate/WPILibVersion.cpp.in @@ -0,0 +1,7 @@ +/* + * Autogenerated file! Do not manually edit this file. This version is regenerated + * any time the publish task is run, or when this file is deleted. + */ +const char* GetWPILibVersion() { + return "${wpilib_version}"; +} diff --git a/datalogtool/src/main/native/cpp/App.cpp b/datalogtool/src/main/native/cpp/App.cpp new file mode 100644 index 00000000000..a4b466b57c3 --- /dev/null +++ b/datalogtool/src/main/native/cpp/App.cpp @@ -0,0 +1,156 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "App.h" + +#include + +#include +#include + +#define IMGUI_DEFINE_MATH_OPERATORS + +#include +#include +#include +#include +#include +#include + +#include "Downloader.h" +#include "Exporter.h" + +namespace gui = wpi::gui; + +const char* GetWPILibVersion(); + +namespace dlt { +std::string_view GetResource_dlt_16_png(); +std::string_view GetResource_dlt_32_png(); +std::string_view GetResource_dlt_48_png(); +std::string_view GetResource_dlt_64_png(); +std::string_view GetResource_dlt_128_png(); +std::string_view GetResource_dlt_256_png(); +std::string_view GetResource_dlt_512_png(); +} // namespace dlt + +bool gShutdown = false; + +static std::unique_ptr gDownloader; +static bool* gDownloadVisible; +static float gDefaultScale = 1.0; + +void SetNextWindowPos(const ImVec2& pos, ImGuiCond cond, const ImVec2& pivot) { + if ((cond & ImGuiCond_FirstUseEver) != 0) { + ImGui::SetNextWindowPos(pos * gDefaultScale, cond, pivot); + } else { + ImGui::SetNextWindowPos(pos, cond, pivot); + } +} + +void SetNextWindowSize(const ImVec2& size, ImGuiCond cond) { + if ((cond & ImGuiCond_FirstUseEver) != 0) { + ImGui::SetNextWindowSize(size * gDefaultScale, cond); + } else { + ImGui::SetNextWindowPos(size, cond); + } +} + +static void DisplayDownload() { + if (!*gDownloadVisible) { + return; + } + SetNextWindowPos(ImVec2{0, 250}, ImGuiCond_FirstUseEver); + SetNextWindowSize(ImVec2{375, 260}, ImGuiCond_FirstUseEver); + if (ImGui::Begin("Download", gDownloadVisible)) { + if (!gDownloader) { + gDownloader = std::make_unique( + glass::GetStorageRoot().GetChild("download")); + } + gDownloader->Display(); + } + ImGui::End(); +} + +static void DisplayMainMenu() { + ImGui::BeginMainMenuBar(); + + static glass::MainMenuBar mainMenu; + mainMenu.WorkspaceMenu(); + gui::EmitViewMenu(); + + if (ImGui::BeginMenu("Window")) { + ImGui::MenuItem("Download", nullptr, gDownloadVisible); + ImGui::EndMenu(); + } + + bool about = false; + if (ImGui::BeginMenu("Info")) { + if (ImGui::MenuItem("About")) { + about = true; + } + ImGui::EndMenu(); + } + + ImGui::EndMainMenuBar(); + + if (about) { + ImGui::OpenPopup("About"); + } + if (ImGui::BeginPopupModal("About")) { + ImGui::Text("Datalog Tool"); + ImGui::Separator(); + ImGui::Text("v%s", GetWPILibVersion()); + ImGui::Separator(); + ImGui::Text("Save location: %s", glass::GetStorageDir().c_str()); + if (ImGui::Button("Close")) { + ImGui::CloseCurrentPopup(); + } + ImGui::EndPopup(); + } +} + +static void DisplayGui() { + DisplayMainMenu(); + DisplayInputFiles(); + DisplayEntries(); + DisplayOutput(glass::GetStorageRoot().GetChild("output")); + DisplayDownload(); +} + +void Application(std::string_view saveDir) { + ssh_init(); + + gui::CreateContext(); + glass::CreateContext(); + + // Add icons + gui::AddIcon(dlt::GetResource_dlt_16_png()); + gui::AddIcon(dlt::GetResource_dlt_32_png()); + gui::AddIcon(dlt::GetResource_dlt_48_png()); + gui::AddIcon(dlt::GetResource_dlt_64_png()); + gui::AddIcon(dlt::GetResource_dlt_128_png()); + gui::AddIcon(dlt::GetResource_dlt_256_png()); + gui::AddIcon(dlt::GetResource_dlt_512_png()); + + glass::SetStorageName("datalogtool"); + glass::SetStorageDir(saveDir.empty() ? gui::GetPlatformSaveFileDir() + : saveDir); + + gui::AddWindowScaler([](float scale) { gDefaultScale = scale; }); + gui::AddLateExecute(DisplayGui); + gui::Initialize("Datalog Tool", 925, 510); + + gDownloadVisible = + &glass::GetStorageRoot().GetChild("download").GetBool("visible", true); + + gui::Main(); + + gShutdown = true; + glass::DestroyContext(); + gui::DestroyContext(); + + gDownloader.reset(); + ssh_finalize(); +} diff --git a/datalogtool/src/main/native/cpp/App.h b/datalogtool/src/main/native/cpp/App.h new file mode 100644 index 00000000000..9d1520c8a0a --- /dev/null +++ b/datalogtool/src/main/native/cpp/App.h @@ -0,0 +1,11 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include + +void SetNextWindowPos(const ImVec2& pos, ImGuiCond cond = 0, + const ImVec2& pivot = ImVec2(0, 0)); +void SetNextWindowSize(const ImVec2& size, ImGuiCond cond = 0); diff --git a/datalogtool/src/main/native/cpp/DataLogThread.cpp b/datalogtool/src/main/native/cpp/DataLogThread.cpp new file mode 100644 index 00000000000..90c8b196c62 --- /dev/null +++ b/datalogtool/src/main/native/cpp/DataLogThread.cpp @@ -0,0 +1,72 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "DataLogThread.h" + +#include + +DataLogThread::~DataLogThread() { + if (m_thread.joinable()) { + m_active = false; + m_thread.join(); + } +} + +void DataLogThread::ReadMain() { + for (auto record : m_reader) { + if (!m_active) { + break; + } + ++m_numRecords; + if (record.IsStart()) { + wpi::log::StartRecordData data; + if (record.GetStartData(&data)) { + std::scoped_lock lock{m_mutex}; + if (m_entries.find(data.entry) != m_entries.end()) { + fmt::print("...DUPLICATE entry ID, overriding\n"); + } + m_entries[data.entry] = data; + m_entryNames.emplace(data.name, data); + sigEntryAdded(data); + } else { + fmt::print("Start(INVALID)\n"); + } + } else if (record.IsFinish()) { + int entry; + if (record.GetFinishEntry(&entry)) { + std::scoped_lock lock{m_mutex}; + auto it = m_entries.find(entry); + if (it == m_entries.end()) { + fmt::print("...ID not found\n"); + } else { + m_entries.erase(it); + } + } else { + fmt::print("Finish(INVALID)\n"); + } + } else if (record.IsSetMetadata()) { + wpi::log::MetadataRecordData data; + if (record.GetSetMetadataData(&data)) { + std::scoped_lock lock{m_mutex}; + auto it = m_entries.find(data.entry); + if (it == m_entries.end()) { + fmt::print("...ID not found\n"); + } else { + it->second.metadata = data.metadata; + auto nameIt = m_entryNames.find(it->second.name); + if (nameIt != m_entryNames.end()) { + nameIt->second.metadata = data.metadata; + } + } + } else { + fmt::print("SetMetadata(INVALID)\n"); + } + } else if (record.IsControl()) { + fmt::print("Unrecognized control record\n"); + } + } + + sigDone(); + m_done = true; +} diff --git a/datalogtool/src/main/native/cpp/DataLogThread.h b/datalogtool/src/main/native/cpp/DataLogThread.h new file mode 100644 index 00000000000..0a273369ca2 --- /dev/null +++ b/datalogtool/src/main/native/cpp/DataLogThread.h @@ -0,0 +1,71 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +class DataLogThread { + public: + explicit DataLogThread(wpi::log::DataLogReader reader) + : m_reader{std::move(reader)}, m_thread{[=] { ReadMain(); }} {} + ~DataLogThread(); + + bool IsDone() const { return m_done; } + std::string_view GetBufferIdentifier() const { + return m_reader.GetBufferIdentifier(); + } + unsigned int GetNumRecords() const { return m_numRecords; } + unsigned int GetNumEntries() const { + std::scoped_lock lock{m_mutex}; + return m_entryNames.size(); + } + + // Passes wpi::log::StartRecordData to func + template + void ForEachEntryName(T&& func) { + std::scoped_lock lock{m_mutex}; + for (auto&& kv : m_entryNames) { + func(kv.second); + } + } + + wpi::log::StartRecordData GetEntry(std::string_view name) const { + std::scoped_lock lock{m_mutex}; + auto it = m_entryNames.find(name); + if (it == m_entryNames.end()) { + return {}; + } + return it->second; + } + + const wpi::log::DataLogReader& GetReader() const { return m_reader; } + + // note: these are called on separate thread + wpi::sig::Signal_mt sigEntryAdded; + wpi::sig::Signal_mt<> sigDone; + + private: + void ReadMain(); + + wpi::log::DataLogReader m_reader; + mutable wpi::mutex m_mutex; + std::atomic_bool m_active{true}; + std::atomic_bool m_done{false}; + std::atomic m_numRecords{0}; + std::map> m_entryNames; + wpi::DenseMap m_entries; + std::thread m_thread; +}; diff --git a/datalogtool/src/main/native/cpp/Downloader.cpp b/datalogtool/src/main/native/cpp/Downloader.cpp new file mode 100644 index 00000000000..452bf261a91 --- /dev/null +++ b/datalogtool/src/main/native/cpp/Downloader.cpp @@ -0,0 +1,393 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "Downloader.h" + +#include + +#ifdef _WIN32 +#include +#include +#else +#include +#endif + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "Sftp.h" + +Downloader::Downloader(glass::Storage& storage) + : m_serverTeam{storage.GetString("serverTeam")}, + m_remoteDir{storage.GetString("remoteDir", "/home/lvuser")}, + m_username{storage.GetString("username", "lvuser")}, + m_localDir{storage.GetString("localDir")}, + m_deleteAfter{storage.GetBool("deleteAfter", true)}, + m_thread{[this] { ThreadMain(); }} {} + +Downloader::~Downloader() { + { + std::scoped_lock lock{m_mutex}; + m_state = kExit; + } + m_cv.notify_all(); + m_thread.join(); +} + +void Downloader::DisplayConnect() { + // IP or Team Number text box + ImGui::SetNextItemWidth(ImGui::GetFontSize() * 12); + ImGui::InputText("Team Number / Address", &m_serverTeam); + + // Username/password + ImGui::SetNextItemWidth(ImGui::GetFontSize() * 12); + ImGui::InputText("Username", &m_username); + ImGui::SetNextItemWidth(ImGui::GetFontSize() * 12); + ImGui::InputText("Password", &m_password, ImGuiInputTextFlags_Password); + + // Connect button + if (ImGui::Button("Connect")) { + m_state = kConnecting; + m_cv.notify_all(); + } +} + +void Downloader::DisplayDisconnectButton() { + if (ImGui::Button("Disconnect")) { + m_state = kDisconnecting; + m_cv.notify_all(); + } +} + +void Downloader::DisplayRemoteDirSelector() { + ImGui::SameLine(); + if (ImGui::Button("Refresh")) { + m_state = kGetFiles; + m_cv.notify_all(); + } + + // Remote directory text box + ImGui::SetNextItemWidth(ImGui::GetFontSize() * 20); + if (ImGui::InputText("Remote Dir", &m_remoteDir, + ImGuiInputTextFlags_EnterReturnsTrue)) { + m_state = kGetFiles; + m_cv.notify_all(); + } + + // List directories + for (auto&& dir : m_dirList) { + if (ImGui::Selectable(dir.c_str())) { + if (dir == "..") { + if (wpi::ends_with(m_remoteDir, '/')) { + m_remoteDir.resize(m_remoteDir.size() - 1); + } + m_remoteDir = wpi::rsplit(m_remoteDir, '/').first; + if (m_remoteDir.empty()) { + m_remoteDir = "/"; + } + } else { + if (!wpi::ends_with(m_remoteDir, '/')) { + m_remoteDir += '/'; + } + m_remoteDir += dir; + } + m_state = kGetFiles; + m_cv.notify_all(); + } + } +} + +void Downloader::DisplayLocalDirSelector() { + // Local directory text / select button + if (ImGui::Button("Select Download Folder...")) { + m_localDirSelector = + std::make_unique("Select Download Folder"); + } + ImGui::TextUnformatted(m_localDir.c_str()); + + // Delete after download (checkbox) + ImGui::Checkbox("Delete after download", &m_deleteAfter); + + // Download button + if (!m_localDir.empty()) { + if (ImGui::Button("Download")) { + m_state = kDownload; + m_cv.notify_all(); + } + } +} + +size_t Downloader::DisplayFiles() { + // List of files (multi-select) (changes to progress bar for downloading) + size_t fileCount = 0; + if (ImGui::BeginTable( + "files", 3, + ImGuiTableFlags_Borders | ImGuiTableFlags_SizingStretchProp)) { + ImGui::TableSetupColumn("File"); + ImGui::TableSetupColumn("Size"); + ImGui::TableSetupColumn("Download"); + ImGui::TableHeadersRow(); + for (auto&& download : m_downloadList) { + if ((m_state == kDownload || m_state == kDownloadDone) && + !download.enabled) { + continue; + } + + ++fileCount; + + ImGui::TableNextRow(); + ImGui::TableNextColumn(); + ImGui::TextUnformatted(download.name.c_str()); + ImGui::TableNextColumn(); + auto sizeText = fmt::format("{}", download.size); + ImGui::TextUnformatted(sizeText.c_str()); + ImGui::TableNextColumn(); + if (m_state == kDownload || m_state == kDownloadDone) { + if (!download.status.empty()) { + ImGui::TextUnformatted(download.status.c_str()); + } else { + ImGui::ProgressBar(download.complete); + } + } else { + auto checkboxLabel = fmt::format("##{}", download.name); + ImGui::Checkbox(checkboxLabel.c_str(), &download.enabled); + } + } + ImGui::EndTable(); + } + + return fileCount; +} + +void Downloader::Display() { + if (m_localDirSelector && m_localDirSelector->ready(0)) { + m_localDir = m_localDirSelector->result(); + m_localDirSelector.reset(); + } + + std::scoped_lock lock{m_mutex}; + + if (!m_error.empty()) { + ImGui::TextUnformatted(m_error.c_str()); + } + + switch (m_state) { + case kDisconnected: + DisplayConnect(); + break; + case kConnecting: + DisplayDisconnectButton(); + ImGui::Text("Connecting to %s...", m_serverTeam.c_str()); + break; + case kDisconnecting: + ImGui::TextUnformatted("Disconnecting..."); + break; + case kConnected: + case kGetFiles: + DisplayDisconnectButton(); + DisplayRemoteDirSelector(); + if (DisplayFiles() > 0) { + DisplayLocalDirSelector(); + } + break; + case kDownload: + case kDownloadDone: + DisplayDisconnectButton(); + DisplayFiles(); + if (m_state == kDownloadDone) { + if (ImGui::Button("Download complete!")) { + m_state = kGetFiles; + m_cv.notify_all(); + } + } + break; + default: + break; + } +} + +void Downloader::ThreadMain() { + std::unique_ptr session; + + static constexpr size_t kBufSize = 32 * 1024; + std::unique_ptr copyBuf = std::make_unique(kBufSize); + + std::unique_lock lock{m_mutex}; + while (m_state != kExit) { + State prev = m_state; + m_cv.wait(lock, [&] { return m_state != prev; }); + m_error.clear(); + try { + switch (m_state) { + case kConnecting: + if (auto team = wpi::parse_integer(m_serverTeam, 10)) { + // team number + session = std::make_unique( + fmt::format("roborio-{}-frc.local", team.value()), 22, + m_username, m_password); + } else { + session = std::make_unique(m_serverTeam, 22, + m_username, m_password); + } + lock.unlock(); + try { + session->Connect(); + } catch (...) { + lock.lock(); + throw; + } + lock.lock(); + // FALLTHROUGH + case kGetFiles: { + std::string dir = m_remoteDir; + std::vector fileList; + lock.unlock(); + try { + fileList = session->ReadDir(dir); + } catch (sftp::Exception& ex) { + lock.lock(); + if (ex.err == SSH_FX_OK || ex.err == SSH_FX_CONNECTION_LOST) { + throw; + } + m_error = ex.what(); + m_dirList.clear(); + m_downloadList.clear(); + m_state = kConnected; + break; + } + std::sort( + fileList.begin(), fileList.end(), + [](const auto& l, const auto& r) { return l.name < r.name; }); + lock.lock(); + + m_dirList.clear(); + m_downloadList.clear(); + for (auto&& attr : fileList) { + if (attr.type == SSH_FILEXFER_TYPE_DIRECTORY) { + if (attr.name != ".") { + m_dirList.emplace_back(attr.name); + } + } else if (attr.type == SSH_FILEXFER_TYPE_REGULAR && + (attr.flags & SSH_FILEXFER_ATTR_SIZE) != 0 && + wpi::ends_with(attr.name, ".wpilog")) { + m_downloadList.emplace_back(attr.name, attr.size); + } + } + + m_state = kConnected; + break; + } + case kDisconnecting: + session.reset(); + m_state = kDisconnected; + break; + case kDownload: { + for (auto&& download : m_downloadList) { + if (m_state != kDownload) { + // user aborted + break; + } + if (!download.enabled) { + continue; + } + + auto remoteFilename = fmt::format( + "{}{}{}", m_remoteDir, + wpi::ends_with(m_remoteDir, '/') ? "" : "/", download.name); + auto localFilename = fs::path{m_localDir} / download.name; + uint64_t fileSize = download.size; + + lock.unlock(); + + // open local file + std::error_code ec; + fs::file_t of = fs::OpenFileForWrite(localFilename, ec, + fs::CD_CreateNew, fs::OF_None); + if (ec) { + // failed to open + lock.lock(); + download.status = ec.message(); + continue; + } + int ofd = fs::FileToFd(of, ec, fs::OF_None); + if (ofd == -1 || ec) { + // failed to convert to fd + lock.lock(); + download.status = ec.message(); + continue; + } + + try { + // open remote file + sftp::File f = session->Open(remoteFilename, O_RDONLY, 0); + + // copy in chunks + uint64_t total = 0; + while (total < fileSize) { + uint64_t toCopy = (std::min)(fileSize - total, + static_cast(kBufSize)); + auto copied = f.Read(copyBuf.get(), toCopy); + if (write(ofd, copyBuf.get(), copied) != + static_cast(copied)) { + // error writing + close(ofd); + fs::remove(localFilename, ec); + lock.lock(); + download.status = "error writing local file"; + goto err; + } + total += copied; + lock.lock(); + download.complete = static_cast(total) / fileSize; + lock.unlock(); + } + + // close local file + close(ofd); + ofd = -1; + + // delete remote file (if enabled) + if (m_deleteAfter) { + f = sftp::File{}; + session->Unlink(remoteFilename); + } + } catch (sftp::Exception& ex) { + if (ofd != -1) { + // close local file and delete it (due to failure) + close(ofd); + fs::remove(localFilename, ec); + } + lock.lock(); + download.status = ex.what(); + if (ex.err == SSH_FX_OK || ex.err == SSH_FX_CONNECTION_LOST) { + throw; + } + continue; + } + lock.lock(); + err : {} + } + if (m_state == kDownload) { + m_state = kDownloadDone; + } + break; + } + default: + break; + } + } catch (sftp::Exception& ex) { + m_error = ex.what(); + session.reset(); + m_state = kDisconnected; + } + } +} diff --git a/datalogtool/src/main/native/cpp/Downloader.h b/datalogtool/src/main/native/cpp/Downloader.h new file mode 100644 index 00000000000..f9bb32ff391 --- /dev/null +++ b/datalogtool/src/main/native/cpp/Downloader.h @@ -0,0 +1,78 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include +#include +#include + +#include +#include + +namespace glass { +class Storage; +} // namespace glass + +namespace pfd { +class select_folder; +} // namespace pfd + +class Downloader { + public: + explicit Downloader(glass::Storage& storage); + ~Downloader(); + + void Display(); + + private: + void DisplayConnect(); + void DisplayDisconnectButton(); + void DisplayRemoteDirSelector(); + void DisplayLocalDirSelector(); + size_t DisplayFiles(); + + void ThreadMain(); + + wpi::mutex m_mutex; + enum State { + kDisconnected, + kConnecting, + kConnected, + kDisconnecting, + kGetFiles, + kDownload, + kDownloadDone, + kExit + } m_state = kDisconnected; + std::condition_variable m_cv; + + std::string& m_serverTeam; + std::string& m_remoteDir; + std::string& m_username; + std::string m_password; + + std::string& m_localDir; + std::unique_ptr m_localDirSelector; + + bool& m_deleteAfter; + + std::vector m_dirList; + struct DownloadState { + DownloadState(std::string_view name, uint64_t size) + : name{name}, size{size} {} + + std::string name; + uint64_t size; + bool enabled = true; + float complete = 0.0; + std::string status; + }; + std::vector m_downloadList; + + std::string m_error; + + std::thread m_thread; +}; diff --git a/datalogtool/src/main/native/cpp/Exporter.cpp b/datalogtool/src/main/native/cpp/Exporter.cpp new file mode 100644 index 00000000000..0fdfa8f695c --- /dev/null +++ b/datalogtool/src/main/native/cpp/Exporter.cpp @@ -0,0 +1,655 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "Exporter.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "App.h" +#include "DataLogThread.h" + +namespace { +struct InputFile { + explicit InputFile(std::unique_ptr datalog); + + InputFile(std::string_view filename, std::string_view status) + : filename{filename}, + stem{fs::path{filename}.stem().string()}, + status{status} {} + + ~InputFile(); + + std::string filename; + std::string stem; + std::unique_ptr datalog; + std::string status; + bool highlight = false; +}; + +struct Entry { + explicit Entry(const wpi::log::StartRecordData& srd) + : name{srd.name}, type{srd.type}, metadata{srd.metadata} {} + + std::string name; + std::string type; + std::string metadata; + std::set inputFiles; + bool typeConflict = false; + bool metadataConflict = false; + bool selected = true; + + // used only during export + int column = -1; +}; + +struct EntryTreeNode { + explicit EntryTreeNode(std::string_view name) : name{name} {} + std::string name; // name of just this node + std::string path; // full path if entry is nullptr + Entry* entry = nullptr; + std::vector children; // children, sorted by name + int selected = 1; +}; +} // namespace + +static std::map, std::less<>> + gInputFiles; +static wpi::mutex gEntriesMutex; +static std::map, std::less<>> gEntries; +static std::vector gEntryTree; +std::atomic_int gExportCount{0}; + +// must be called with gEntriesMutex held +static void RebuildEntryTree() { + gEntryTree.clear(); + wpi::SmallVector parts; + for (auto& kv : gEntries) { + parts.clear(); + // split on first : if one is present + auto [prefix, mainpart] = wpi::split(kv.first, ':'); + if (mainpart.empty() || wpi::contains(prefix, '/')) { + mainpart = kv.first; + } else { + parts.emplace_back(prefix); + } + wpi::split(mainpart, parts, '/', -1, false); + + // ignore a raw "/" key + if (parts.empty()) { + continue; + } + + // get to leaf + auto nodes = &gEntryTree; + for (auto part : wpi::drop_back(wpi::span{parts.begin(), parts.end()})) { + auto it = + std::find_if(nodes->begin(), nodes->end(), + [&](const auto& node) { return node.name == part; }); + if (it == nodes->end()) { + nodes->emplace_back(part); + // path is from the beginning of the string to the end of the current + // part; this works because part is a reference to the internals of + // kv.first + nodes->back().path.assign(kv.first.data(), + part.data() + part.size() - kv.first.data()); + it = nodes->end() - 1; + } + nodes = &it->children; + } + + auto it = std::find_if(nodes->begin(), nodes->end(), [&](const auto& node) { + return node.name == parts.back(); + }); + if (it == nodes->end()) { + nodes->emplace_back(parts.back()); + // no need to set path, as it's identical to kv.first + it = nodes->end() - 1; + } + it->entry = kv.second.get(); + } +} + +InputFile::InputFile(std::unique_ptr datalog_) + : filename{datalog_->GetBufferIdentifier()}, + stem{fs::path{filename}.stem().string()}, + datalog{std::move(datalog_)} { + datalog->sigEntryAdded.connect([this](const wpi::log::StartRecordData& srd) { + std::scoped_lock lock{gEntriesMutex}; + auto it = gEntries.find(srd.name); + if (it == gEntries.end()) { + it = gEntries.emplace(srd.name, std::make_unique(srd)).first; + RebuildEntryTree(); + } else { + if (it->second->type != srd.type) { + it->second->typeConflict = true; + } + if (it->second->metadata != srd.metadata) { + it->second->metadataConflict = true; + } + } + it->second->inputFiles.emplace(this); + }); +} + +InputFile::~InputFile() { + if (gShutdown || !datalog) { + return; + } + std::scoped_lock lock{gEntriesMutex}; + bool changed = false; + for (auto it = gEntries.begin(); it != gEntries.end();) { + it->second->inputFiles.erase(this); + if (it->second->inputFiles.empty()) { + it = gEntries.erase(it); + changed = true; + } else { + ++it; + } + } + if (changed) { + RebuildEntryTree(); + } +} + +static std::unique_ptr LoadDataLog(std::string_view filename) { + std::error_code ec; + auto buf = wpi::MemoryBuffer::GetFile(filename, ec); + std::string fn{filename}; + if (ec) { + return std::make_unique( + fn, fmt::format("Could not open file: {}", ec.message())); + } + + wpi::log::DataLogReader reader{std::move(buf)}; + if (!reader.IsValid()) { + return std::make_unique(fn, "Not a valid datalog file"); + } + + return std::make_unique( + std::make_unique(std::move(reader))); +} + +void DisplayInputFiles() { + static std::unique_ptr dataFileSelector; + + SetNextWindowPos(ImVec2{0, 20}, ImGuiCond_FirstUseEver); + SetNextWindowSize(ImVec2{375, 230}, ImGuiCond_FirstUseEver); + if (ImGui::Begin("Input Files")) { + if (ImGui::Button("Open File(s)...")) { + dataFileSelector = std::make_unique( + "Select Data Log", "", + std::vector{"DataLog Files", "*.wpilog"}, + pfd::opt::multiselect); + } + ImGui::BeginTable( + "Input Files", 3, + ImGuiTableFlags_Borders | ImGuiTableFlags_SizingStretchProp); + ImGui::TableSetupColumn("File"); + ImGui::TableSetupColumn("Status"); + ImGui::TableSetupColumn("X", ImGuiTableColumnFlags_WidthFixed | + ImGuiTableColumnFlags_NoHeaderLabel | + ImGuiTableColumnFlags_NoHeaderWidth); + ImGui::TableHeadersRow(); + for (auto it = gInputFiles.begin(); it != gInputFiles.end();) { + ImGui::TableNextRow(); + ImGui::TableNextColumn(); + if (it->second->highlight) { + ImGui::TableSetBgColor(ImGuiTableBgTarget_RowBg0, + IM_COL32(0, 64, 0, 255)); + it->second->highlight = false; + } + ImGui::TextUnformatted(it->first.c_str()); + if (ImGui::IsItemHovered()) { + ImGui::SetTooltip("%s", it->second->filename.c_str()); + } + + ImGui::TableNextColumn(); + if (it->second->datalog) { + ImGui::Text("%u records, %u entries%s", + it->second->datalog->GetNumRecords(), + it->second->datalog->GetNumEntries(), + it->second->datalog->IsDone() ? "" : " (working)"); + } else { + ImGui::TextUnformatted(it->second->status.c_str()); + } + + ImGui::TableNextColumn(); + ImGui::PushID(it->first.c_str()); + if (ImGui::SmallButton("X")) { + it = gInputFiles.erase(it); + gExportCount = 0; + } else { + ++it; + } + ImGui::PopID(); + } + ImGui::EndTable(); + } + ImGui::End(); + + // Load data file(s) + if (dataFileSelector && dataFileSelector->ready(0)) { + auto result = dataFileSelector->result(); + for (auto&& filename : result) { + // don't allow duplicates + std::string stem = fs::path{filename}.stem().string(); + auto it = gInputFiles.find(stem); + if (it == gInputFiles.end()) { + gInputFiles.emplace(std::move(stem), LoadDataLog(filename)); + gExportCount = 0; + } + } + dataFileSelector.reset(); + } +} + +static bool EmitEntry(const std::string& name, Entry& entry) { + ImGui::TableNextColumn(); + bool rv = ImGui::Checkbox(name.c_str(), &entry.selected); + if (ImGui::IsItemHovered() && gInputFiles.size() > 1) { + for (auto inputFile : entry.inputFiles) { + inputFile->highlight = true; + } + } + + ImGui::TableNextColumn(); + if (entry.typeConflict) { + ImGui::TextUnformatted("(Inconsistent)"); + if (ImGui::IsItemHovered()) { + ImGui::BeginTooltip(); + for (auto inputFile : entry.inputFiles) { + ImGui::Text( + "%s: %s", inputFile->stem.c_str(), + std::string{inputFile->datalog->GetEntry(entry.name).type}.c_str()); + } + ImGui::EndTooltip(); + } + } else { + ImGui::TextUnformatted(entry.type.c_str()); + } + + ImGui::TableNextColumn(); + if (entry.metadataConflict) { + ImGui::TextUnformatted("(Inconsistent)"); + if (ImGui::IsItemHovered()) { + ImGui::BeginTooltip(); + for (auto inputFile : entry.inputFiles) { + ImGui::Text( + "%s: %s", inputFile->stem.c_str(), + std::string{inputFile->datalog->GetEntry(entry.name).metadata} + .c_str()); + } + ImGui::EndTooltip(); + } + } else { + ImGui::TextUnformatted(entry.metadata.c_str()); + } + return rv; +} + +static bool EmitEntryTree(std::vector& tree) { + bool rv = false; + for (auto&& node : tree) { + if (node.entry) { + if (EmitEntry(node.name, *node.entry)) { + rv = true; + } + } + + if (!node.children.empty()) { + ImGui::TableNextColumn(); + auto label = fmt::format("##check_{}", node.name); + if (node.selected == -1) { + ImGui::PushItemFlag(ImGuiItemFlags_MixedValue, true); + bool b = false; + if (ImGui::Checkbox(label.c_str(), &b)) { + node.selected = 3; // 3 = enable group + rv = true; + } + ImGui::PopItemFlag(); + } else { + bool b = node.selected == 1 || node.selected == 3; + if (ImGui::Checkbox(label.c_str(), &b)) { + node.selected = b ? 3 : 2; // 2 = disable group + rv = true; + } + } + ImGui::SameLine(); + bool open = ImGui::TreeNodeEx(node.name.c_str(), + ImGuiTreeNodeFlags_SpanFullWidth); + ImGui::TableNextColumn(); + ImGui::TableNextColumn(); + if (open) { + if (EmitEntryTree(node.children)) { + rv = true; + } + ImGui::TreePop(); + } + } + } + return rv; +} + +static void RefreshTreeCheckboxes(std::vector& tree, + int* selected) { + bool first = true; + for (auto&& node : tree) { + if (node.entry) { + if (first && *selected == -1) { + *selected = node.entry->selected ? 1 : 0; + } + if ((*selected == 0 && node.entry->selected) || + (*selected == 1 && !node.entry->selected)) { + *selected = -1; // inconsistent + } else if (*selected == 2) { // disable group + node.entry->selected = false; + } else if (*selected == 3) { // enable group + node.entry->selected = true; + } + } + + if (!node.children.empty()) { + if (*selected == 2) { // disable group + node.selected = 2; + } else if (*selected == 3) { // enable group + node.selected = 3; + } + RefreshTreeCheckboxes(node.children, &node.selected); + if (node.selected == 2) { + node.selected = 0; + } else if (node.selected == 3) { + node.selected = 1; + } + if (first && *selected == -1) { + *selected = node.selected; + } else if (node.selected == -1 || + (*selected == 0 && node.selected == 1) || + (*selected == 1 && node.selected == 0)) { + *selected = -1; // inconsistent + } + } + + first = false; + } +} + +void DisplayEntries() { + SetNextWindowPos(ImVec2{380, 20}, ImGuiCond_FirstUseEver); + SetNextWindowSize(ImVec2{540, 365}, ImGuiCond_FirstUseEver); + if (ImGui::Begin("Entries")) { + static bool treeView = true; + if (ImGui::BeginPopupContextItem()) { + ImGui::MenuItem("Tree View", "", &treeView); + ImGui::EndPopup(); + } + std::scoped_lock lock{gEntriesMutex}; + ImGui::BeginTable( + "Entries", 3, + ImGuiTableFlags_Borders | ImGuiTableFlags_SizingStretchProp); + ImGui::TableSetupColumn("Name"); + ImGui::TableSetupColumn("Type"); + ImGui::TableSetupColumn("Metadata"); + ImGui::TableHeadersRow(); + if (treeView) { + if (EmitEntryTree(gEntryTree)) { + int selected = -1; + RefreshTreeCheckboxes(gEntryTree, &selected); + } + } else { + for (auto&& kv : gEntries) { + EmitEntry(kv.first, *kv.second); + } + } + ImGui::EndTable(); + } + ImGui::End(); +} + +static wpi::mutex gExportMutex; +static std::vector gExportErrors; + +static void PrintEscapedCsvString(wpi::raw_ostream& os, std::string_view str) { + auto s = str; + while (!s.empty()) { + std::string_view fragment; + std::tie(fragment, s) = wpi::split(s, '"'); + os << fragment; + if (!s.empty()) { + os << '"' << '"'; + } + } + if (wpi::ends_with(str, '"')) { + os << '"' << '"'; + } +} + +static void ValueToCsv(wpi::raw_ostream& os, const Entry& entry, + const wpi::log::DataLogRecord& record) { + // handle systemTime specially + if (entry.name == "systemTime" && entry.type == "int64") { + int64_t val; + if (record.GetInteger(&val)) { + std::time_t timeval = val / 1000000; + fmt::print(os, "{:%Y-%m-%d %H:%M:%S}.{:06}", *std::localtime(&timeval), + val % 1000000); + return; + } + } else if (entry.type == "double") { + double val; + if (record.GetDouble(&val)) { + fmt::print(os, "{}", val); + return; + } + } else if (entry.type == "int64") { + int64_t val; + if (record.GetInteger(&val)) { + fmt::print(os, "{}", val); + return; + } + } else if (entry.type == "string" || entry.type == "json") { + std::string_view val; + record.GetString(&val); + os << '"'; + PrintEscapedCsvString(os, val); + os << '"'; + return; + } else if (entry.type == "boolean") { + bool val; + if (record.GetBoolean(&val)) { + fmt::print(os, "{}", val); + return; + } + } else if (entry.type == "double[]") { + std::vector val; + if (record.GetDoubleArray(&val)) { + fmt::print(os, "{}", fmt::join(val, ";")); + return; + } + } else if (entry.type == "float[]") { + std::vector val; + if (record.GetFloatArray(&val)) { + fmt::print(os, "{}", fmt::join(val, ";")); + return; + } + } else if (entry.type == "int64[]") { + std::vector val; + if (record.GetIntegerArray(&val)) { + fmt::print(os, "{}", fmt::join(val, ";")); + return; + } + } else if (entry.type == "string[]") { + std::vector val; + if (record.GetStringArray(&val)) { + os << '"'; + bool first = true; + for (auto&& v : val) { + if (!first) { + os << ';'; + } + first = false; + PrintEscapedCsvString(os, v); + } + os << '"'; + return; + } + } + fmt::print(os, ""); +} + +static void ExportCsvFile(InputFile& f, wpi::raw_ostream& os, int style) { + // header + if (style == 0) { + os << "Timestamp,Name,Value\n"; + } else if (style == 1) { + // scan for exported fields for this file to print header and assign columns + os << "Timestamp"; + int columnNum = 0; + for (auto&& entry : gEntries) { + if (entry.second->selected && + entry.second->inputFiles.find(&f) != entry.second->inputFiles.end()) { + os << ',' << '"'; + PrintEscapedCsvString(os, entry.first); + os << '"'; + entry.second->column = columnNum++; + } else { + entry.second->column = -1; + } + } + os << '\n'; + } + + wpi::DenseMap nameMap; + for (auto&& record : f.datalog->GetReader()) { + if (record.IsStart()) { + wpi::log::StartRecordData data; + if (record.GetStartData(&data)) { + auto it = gEntries.find(data.name); + if (it != gEntries.end() && it->second->selected) { + nameMap[data.entry] = it->second.get(); + } + } + } else if (record.IsFinish()) { + int entry; + if (record.GetFinishEntry(&entry)) { + nameMap.erase(entry); + } + } else if (!record.IsControl()) { + auto entryIt = nameMap.find(record.GetEntry()); + if (entryIt == nameMap.end()) { + continue; + } + Entry* entry = entryIt->second; + + if (style == 0) { + fmt::print(os, "{},\"", record.GetTimestamp() / 1000000.0); + PrintEscapedCsvString(os, entry->name); + os << '"' << ','; + ValueToCsv(os, *entry, record); + os << '\n'; + } else if (style == 1 && entry->column != -1) { + fmt::print(os, "{},", record.GetTimestamp() / 1000000.0); + for (int i = 0; i < entry->column; ++i) { + os << ','; + } + ValueToCsv(os, *entry, record); + os << '\n'; + } + } + } +} + +static void ExportCsv(std::string_view outputFolder, int style) { + fs::path outPath{outputFolder}; + for (auto&& f : gInputFiles) { + if (f.second->datalog) { + std::error_code ec; + auto of = fs::OpenFileForWrite( + outPath / fs::path{f.first}.replace_extension("csv"), ec, + fs::CD_CreateNew, fs::OF_Text); + if (ec) { + std::scoped_lock lock{gExportMutex}; + gExportErrors.emplace_back( + fmt::format("{}: {}", f.first, ec.message())); + ++gExportCount; + continue; + } + wpi::raw_fd_ostream os{fs::FileToFd(of, ec, fs::OF_Text), true}; + ExportCsvFile(*f.second, os, style); + } + ++gExportCount; + } +} + +void DisplayOutput(glass::Storage& storage) { + static std::string& outputFolder = storage.GetString("outputFolder"); + static std::unique_ptr outputFolderSelector; + + SetNextWindowPos(ImVec2{380, 390}, ImGuiCond_FirstUseEver); + SetNextWindowSize(ImVec2{540, 120}, ImGuiCond_FirstUseEver); + if (ImGui::Begin("Output")) { + if (ImGui::Button("Select Output Folder...")) { + outputFolderSelector = + std::make_unique("Select Output Folder"); + } + ImGui::TextUnformatted(outputFolder.c_str()); + + static const char* const options[] = {"List", "Table"}; + static int style = 0; + ImGui::SetNextItemWidth(ImGui::GetFontSize() * 8); + ImGui::Combo("Style", &style, options, + sizeof(options) / sizeof(const char*)); + + static std::future exporter; + if (!gInputFiles.empty() && !outputFolder.empty() && + ImGui::Button("Export CSV") && + (gExportCount == 0 || + gExportCount == static_cast(gInputFiles.size()))) { + gExportCount = 0; + gExportErrors.clear(); + exporter = std::async(std::launch::async, ExportCsv, outputFolder, style); + } + if (exporter.valid()) { + ImGui::SameLine(); + ImGui::Text("Exported %d/%d", gExportCount.load(), + static_cast(gInputFiles.size())); + } + { + std::scoped_lock lock{gExportMutex}; + for (auto&& err : gExportErrors) { + ImGui::TextUnformatted(err.c_str()); + } + } + } + ImGui::End(); + + if (outputFolderSelector && outputFolderSelector->ready(0)) { + outputFolder = outputFolderSelector->result(); + outputFolderSelector.reset(); + } +} diff --git a/datalogtool/src/main/native/cpp/Exporter.h b/datalogtool/src/main/native/cpp/Exporter.h new file mode 100644 index 00000000000..8078243bb7d --- /dev/null +++ b/datalogtool/src/main/native/cpp/Exporter.h @@ -0,0 +1,15 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +namespace glass { +class Storage; +} // namespace glass + +void DisplayInputFiles(); +void DisplayEntries(); +void DisplayOutput(glass::Storage& storage); + +extern bool gShutdown; diff --git a/datalogtool/src/main/native/cpp/Sftp.cpp b/datalogtool/src/main/native/cpp/Sftp.cpp new file mode 100644 index 00000000000..f6e627c2bf6 --- /dev/null +++ b/datalogtool/src/main/native/cpp/Sftp.cpp @@ -0,0 +1,215 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include "Sftp.h" + +#include + +using namespace sftp; + +Attributes::Attributes(sftp_attributes&& attr) + : name{attr->name}, flags{attr->flags}, type{attr->type}, size{attr->size} { + sftp_attributes_free(attr); +} + +static std::string GetError(sftp_session sftp) { + switch (sftp_get_error(sftp)) { + case SSH_FX_EOF: + return "end of file"; + case SSH_FX_NO_SUCH_FILE: + return "no such file"; + case SSH_FX_PERMISSION_DENIED: + return "permission denied"; + case SSH_FX_FAILURE: + return "SFTP failure"; + case SSH_FX_BAD_MESSAGE: + return "SFTP bad message"; + case SSH_FX_NO_CONNECTION: + return "SFTP no connection"; + case SSH_FX_CONNECTION_LOST: + return "SFTP connection lost"; + case SSH_FX_OP_UNSUPPORTED: + return "SFTP operation unsupported"; + case SSH_FX_INVALID_HANDLE: + return "SFTP invalid handle"; + case SSH_FX_NO_SUCH_PATH: + return "no such path"; + case SSH_FX_FILE_ALREADY_EXISTS: + return "file already exists"; + case SSH_FX_WRITE_PROTECT: + return "write protected filesystem"; + case SSH_FX_NO_MEDIA: + return "no media inserted"; + default: + return ssh_get_error(sftp->session); + } +} + +Exception::Exception(sftp_session sftp) + : runtime_error{GetError(sftp)}, err{sftp_get_error(sftp)} {} + +File::~File() { + if (m_handle) { + sftp_close(m_handle); + } +} + +Attributes File::Stat() const { + sftp_attributes attr = sftp_fstat(m_handle); + if (!attr) { + throw Exception{m_handle->sftp}; + } + return Attributes{std::move(attr)}; +} + +size_t File::Read(void* buf, uint32_t count) { + auto rv = sftp_read(m_handle, buf, count); + if (rv < 0) { + throw Exception{m_handle->sftp}; + } + return rv; +} + +File::AsyncId File::AsyncReadBegin(uint32_t len) const { + int rv = sftp_async_read_begin(m_handle, len); + if (rv < 0) { + throw Exception{m_handle->sftp}; + } + return rv; +} + +size_t File::AsyncRead(void* data, uint32_t len, AsyncId id) { + auto rv = sftp_async_read(m_handle, data, len, id); + if (rv == SSH_ERROR) { + throw Exception{ssh_get_error(m_handle->sftp->session)}; + } + if (rv == SSH_AGAIN) { + return 0; + } + return rv; +} + +size_t File::Write(wpi::span data) { + auto rv = sftp_write(m_handle, data.data(), data.size()); + if (rv < 0) { + throw Exception{m_handle->sftp}; + } + return rv; +} + +void File::Seek(uint64_t offset) { + if (sftp_seek64(m_handle, offset) < 0) { + throw Exception{m_handle->sftp}; + } +} + +uint64_t File::Tell() const { + return sftp_tell64(m_handle); +} + +void File::Rewind() { + sftp_rewind(m_handle); +} + +void File::Sync() { + if (sftp_fsync(m_handle) < 0) { + throw Exception{m_handle->sftp}; + } +} + +Session::Session(std::string_view host, int port, std::string_view user, + std::string_view pass) + : m_host{host}, m_port{port}, m_username{user}, m_password{pass} { + // Create a new SSH session. + m_session = ssh_new(); + if (!m_session) { + throw Exception{"The SSH session could not be allocated."}; + } + + // Set the host, user, and port. + ssh_options_set(m_session, SSH_OPTIONS_HOST, m_host.c_str()); + ssh_options_set(m_session, SSH_OPTIONS_USER, m_username.c_str()); + ssh_options_set(m_session, SSH_OPTIONS_PORT, &m_port); + + // Set timeout to 3 seconds. + int64_t timeout = 3L; + ssh_options_set(m_session, SSH_OPTIONS_TIMEOUT, &timeout); + + // Set other miscellaneous options. + ssh_options_set(m_session, SSH_OPTIONS_STRICTHOSTKEYCHECK, "no"); +} + +Session::~Session() { + if (m_sftp) { + sftp_free(m_sftp); + } + if (m_session) { + ssh_free(m_session); + } +} + +void Session::Connect() { + // Connect to the server. + int rc = ssh_connect(m_session); + if (rc != SSH_OK) { + throw Exception{ssh_get_error(m_session)}; + } + + // Authenticate with password. + rc = ssh_userauth_password(m_session, nullptr, m_password.c_str()); + if (rc != SSH_AUTH_SUCCESS) { + throw Exception{ssh_get_error(m_session)}; + } + + // Allocate the SFTP session. + m_sftp = sftp_new(m_session); + if (!m_sftp) { + throw Exception{ssh_get_error(m_session)}; + } + + // Initialize. + rc = sftp_init(m_sftp); + if (rc != SSH_OK) { + sftp_free(m_sftp); + m_sftp = nullptr; + throw Exception{ssh_get_error(m_session)}; + } +} + +void Session::Disconnect() { + if (m_sftp) { + sftp_free(m_sftp); + m_sftp = nullptr; + } + ssh_disconnect(m_session); +} + +std::vector Session::ReadDir(const std::string& path) { + sftp_dir dir = sftp_opendir(m_sftp, path.c_str()); + if (!dir) { + throw Exception{m_sftp}; + } + + std::vector rv; + while (sftp_attributes attr = sftp_readdir(m_sftp, dir)) { + rv.emplace_back(std::move(attr)); + } + + sftp_closedir(dir); + return rv; +} + +void Session::Unlink(const std::string& filename) { + if (sftp_unlink(m_sftp, filename.c_str()) < 0) { + throw Exception{m_sftp}; + } +} + +File Session::Open(const std::string& filename, int accesstype, mode_t mode) { + sftp_file f = sftp_open(m_sftp, filename.c_str(), accesstype, mode); + if (!f) { + throw Exception{m_sftp}; + } + return File{std::move(f)}; +} diff --git a/datalogtool/src/main/native/cpp/Sftp.h b/datalogtool/src/main/native/cpp/Sftp.h new file mode 100644 index 00000000000..e57a747fad3 --- /dev/null +++ b/datalogtool/src/main/native/cpp/Sftp.h @@ -0,0 +1,144 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#pragma once + +#include +#include + +#include +#include +#include +#include + +#include + +namespace sftp { + +struct Attributes { + Attributes() = default; + explicit Attributes(sftp_attributes&& attr); + + std::string name; + uint32_t flags = 0; + uint8_t type = 0; + uint64_t size = 0; +}; + +/** + * This is the exception that will be thrown if something goes wrong. + */ +class Exception : public std::runtime_error { + public: + explicit Exception(const std::string& msg) : std::runtime_error{msg} {} + explicit Exception(sftp_session sftp); + + int err = 0; +}; + +class File { + public: + File() = default; + explicit File(sftp_file&& handle) : m_handle{handle} {} + ~File(); + + Attributes Stat() const; + + void SetNonblocking() { sftp_file_set_nonblocking(m_handle); } + void SetBlocking() { sftp_file_set_blocking(m_handle); } + + using AsyncId = uint32_t; + + size_t Read(void* buf, uint32_t count); + AsyncId AsyncReadBegin(uint32_t len) const; + size_t AsyncRead(void* data, uint32_t len, AsyncId id); + size_t Write(wpi::span data); + + void Seek(uint64_t offset); + uint64_t Tell() const; + void Rewind(); + + void Sync(); + + std::string_view GetName() const { return m_handle->name; } + uint64_t GetOffset() const { return m_handle->offset; } + bool IsEof() const { return m_handle->eof; } + bool IsNonblocking() const { return m_handle->nonblocking; } + + private: + sftp_file m_handle{nullptr}; +}; + +/** + * This class is a C++ implementation of the SshSessionController in + * wpilibsuite/deploy-utils. It handles connecting to an SSH server, running + * commands, and transferring files. + */ +class Session { + public: + /** + * Constructs a new session controller. + * + * @param host The hostname of the server to connect to. + * @param port The port that the sshd server is operating on. + * @param user The username to login as. + * @param pass The password for the given username. + */ + Session(std::string_view host, int port, std::string_view user, + std::string_view pass); + + /** + * Destroys the controller object. This also disconnects the session from the + * server. + */ + ~Session(); + + /** + * Opens the SSH connection to the given host. + */ + void Connect(); + + /** + * Disconnects the SSH connection. + */ + void Disconnect(); + + /** + * Reads directory entries + * + * @param path remote path + * @return vector of file attributes + */ + std::vector ReadDir(const std::string& path); + + /** + * Unlinks (deletes) a file. + * + * @param filename filename + */ + void Unlink(const std::string& filename); + + /** + * Opens a file. + * + * @param filename filename + * @param accesstype O_RDONLY, O_WRONLY, or O_RDWR, combined with O_CREAT, + * O_EXCL, or O_TRUNC + * @param mode permissions to use if a new file is created + * @return File + */ + File Open(const std::string& filename, int accesstype, mode_t mode); + + private: + ssh_session m_session{nullptr}; + sftp_session m_sftp{nullptr}; + std::string m_host; + + int m_port; + + std::string m_username; + std::string m_password; +}; + +} // namespace sftp diff --git a/datalogtool/src/main/native/cpp/main.cpp b/datalogtool/src/main/native/cpp/main.cpp new file mode 100644 index 00000000000..5f1261b00f4 --- /dev/null +++ b/datalogtool/src/main/native/cpp/main.cpp @@ -0,0 +1,25 @@ +// Copyright (c) FIRST and other WPILib contributors. +// Open Source Software; you can modify and/or share it under the terms of +// the WPILib BSD license file in the root directory of this project. + +#include + +void Application(std::string_view saveDir); + +#ifdef _WIN32 +int __stdcall WinMain(void* hInstance, void* hPrevInstance, char* pCmdLine, + int nCmdShow) { + int argc = __argc; + char** argv = __argv; +#else +int main(int argc, char** argv) { +#endif + std::string_view saveDir; + if (argc == 2) { + saveDir = argv[1]; + } + + Application(saveDir); + + return 0; +} diff --git a/datalogtool/src/main/native/mac/datalogtool.icns b/datalogtool/src/main/native/mac/datalogtool.icns new file mode 100644 index 00000000000..583eaabddb0 Binary files /dev/null and b/datalogtool/src/main/native/mac/datalogtool.icns differ diff --git a/datalogtool/src/main/native/resources/dlt-128.png b/datalogtool/src/main/native/resources/dlt-128.png new file mode 100644 index 00000000000..b2ebf519339 Binary files /dev/null and b/datalogtool/src/main/native/resources/dlt-128.png differ diff --git a/datalogtool/src/main/native/resources/dlt-16.png b/datalogtool/src/main/native/resources/dlt-16.png new file mode 100644 index 00000000000..f7439c60347 Binary files /dev/null and b/datalogtool/src/main/native/resources/dlt-16.png differ diff --git a/datalogtool/src/main/native/resources/dlt-256.png b/datalogtool/src/main/native/resources/dlt-256.png new file mode 100644 index 00000000000..c1a40d2b02c Binary files /dev/null and b/datalogtool/src/main/native/resources/dlt-256.png differ diff --git a/datalogtool/src/main/native/resources/dlt-32.png b/datalogtool/src/main/native/resources/dlt-32.png new file mode 100644 index 00000000000..1d9a2121ade Binary files /dev/null and b/datalogtool/src/main/native/resources/dlt-32.png differ diff --git a/datalogtool/src/main/native/resources/dlt-48.png b/datalogtool/src/main/native/resources/dlt-48.png new file mode 100644 index 00000000000..119d054e044 Binary files /dev/null and b/datalogtool/src/main/native/resources/dlt-48.png differ diff --git a/datalogtool/src/main/native/resources/dlt-512.png b/datalogtool/src/main/native/resources/dlt-512.png new file mode 100644 index 00000000000..6dfd8211be1 Binary files /dev/null and b/datalogtool/src/main/native/resources/dlt-512.png differ diff --git a/datalogtool/src/main/native/resources/dlt-64.png b/datalogtool/src/main/native/resources/dlt-64.png new file mode 100644 index 00000000000..4ad82c7ea60 Binary files /dev/null and b/datalogtool/src/main/native/resources/dlt-64.png differ diff --git a/datalogtool/src/main/native/win/datalogtool.ico b/datalogtool/src/main/native/win/datalogtool.ico new file mode 100644 index 00000000000..1b90647050d Binary files /dev/null and b/datalogtool/src/main/native/win/datalogtool.ico differ diff --git a/datalogtool/src/main/native/win/datalogtool.rc b/datalogtool/src/main/native/win/datalogtool.rc new file mode 100644 index 00000000000..d0a5fb4f411 --- /dev/null +++ b/datalogtool/src/main/native/win/datalogtool.rc @@ -0,0 +1 @@ +IDI_ICON1 ICON "datalogtool.ico" diff --git a/settings.gradle b/settings.gradle index 2d62f1000f9..6afbdea4434 100644 --- a/settings.gradle +++ b/settings.gradle @@ -33,6 +33,7 @@ include 'fieldImages' include 'glass' include 'outlineviewer' include 'roborioteamnumbersetter' +include 'datalogtool' include 'simulation:gz_msgs' include 'simulation:frc_gazebo_plugins' include 'simulation:halsim_gazebo'