From 40bf2681413ba7ea39c4fde7c4947b6d60263d71 Mon Sep 17 00:00:00 2001 From: PGZXB Date: Tue, 20 Dec 2022 18:24:24 +0800 Subject: [PATCH 1/8] [CLI] Add "ti ticache clean" command to clean ticache files --- python/taichi/_main.py | 28 ++++++++++++-- taichi/cache/gfx/cache_manager.cpp | 40 +++++++++++-------- taichi/cache/metal/cache_manager.cpp | 8 +++- taichi/python/export_misc.cpp | 4 ++ taichi/runtime/llvm/llvm_offline_cache.cpp | 29 +++++++------- taichi/util/offline_cache.cpp | 41 ++++++++++++++++++-- taichi/util/offline_cache.h | 15 +++++++- tests/python/test_cli.py | 45 ++++++++++++++++++++++ 8 files changed, 171 insertions(+), 39 deletions(-) diff --git a/python/taichi/_main.py b/python/taichi/_main.py index 66cd6ff64c6e7..798067dbe62bf 100644 --- a/python/taichi/_main.py +++ b/python/taichi/_main.py @@ -18,6 +18,7 @@ from rich.syntax import Syntax from taichi._lib import core as _ti_core from taichi._lib import utils +from taichi.lang import impl from taichi.tools import cc_compose, diagnose, video import taichi as ti @@ -248,9 +249,9 @@ def colormap(index, name): nrows += 1 names = sorted(choices.keys()) for k in range(nrows): - table.add_row( - * - [colormap(j, names[j]) for j in range(k, len(choices), nrows)]) + table.add_row(*[ + colormap(j, names[j]) for j in range(k, len(choices), nrows) + ]) parser = argparse.ArgumentParser(prog='ti example', description=f"{self.example.__doc__}") @@ -826,6 +827,27 @@ def lint(arguments: list = sys.argv[2:]): import pylint # pylint: disable=C0415 pylint.lint.Run(options) + @staticmethod + @register + def ticache(arguments: list = sys.argv[2:]): + """Manage the ticache files manually""" + if len(arguments) < 1: return + + subcmd = arguments[0] + arguments = arguments[1:] + if subcmd == 'clean': + parser = argparse.ArgumentParser( + prog='ti ticache', + description='Clean all ticache files in given path') + parser.add_argument( + '-p', + '--offline-cache-file-path', + dest='offline_cache_file_path', + default=impl.default_cfg().offline_cache_file_path) + args = parser.parse_args(arguments) + path = args.offline_cache_file_path + _ti_core.clean_offline_cache_files(path) + def main(): cli = TaichiMain() diff --git a/taichi/cache/gfx/cache_manager.cpp b/taichi/cache/gfx/cache_manager.cpp index b330aebbd1e97..b8b8c7726d9c7 100644 --- a/taichi/cache/gfx/cache_manager.cpp +++ b/taichi/cache/gfx/cache_manager.cpp @@ -79,7 +79,8 @@ struct CacheCleanerUtils { const KernelMetaData &kernel_meta) { std::vector result; for (std::size_t i = 0; i < kernel_meta.num_files; ++i) { - result.push_back(kernel_meta.kernel_key + std::to_string(i) + ".spv"); + result.push_back(kernel_meta.kernel_key + std::to_string(i) + "." + + kSpirvCacheFilenameExt); } return result; } @@ -95,7 +96,7 @@ struct CacheCleanerUtils { // To check if a file is cache file static bool is_valid_cache_file(const CacheCleanerConfig &config, const std::string &name) { - return filename_extension(name) == "spv"; + return filename_extension(name) == kSpirvCacheFilenameExt; } }; @@ -121,20 +122,27 @@ CacheManager::CacheManager(Params &&init_params) auto exists = taichi::path_exists(taichi::join_path(path_, kAotMetadataFilename)) && taichi::path_exists(taichi::join_path(path_, kGraphMetadataFilename)); - if (exists && lock_with_file(lock_path)) { - auto _ = make_cleanup([&lock_path]() { - if (!unlock_with_file(lock_path)) { - TI_WARN( - "Unlock {} failed. You can remove this .lock file manually and " - "try again.", - lock_path); - } - }); - gfx::AotModuleParams params; - params.module_path = path_; - params.runtime = runtime_; - params.enable_lazy_loading = true; - cached_module_ = gfx::make_aot_module(params, init_params.arch); + if (exists) { + if (lock_with_file(lock_path)) { + auto _ = make_cleanup([&lock_path]() { + if (!unlock_with_file(lock_path)) { + TI_WARN( + "Unlock {} failed. You can remove this .lock file manually " + "and try again.", + lock_path); + } + }); + gfx::AotModuleParams params; + params.module_path = path_; + params.runtime = runtime_; + params.enable_lazy_loading = true; + cached_module_ = gfx::make_aot_module(params, init_params.arch); + } else { + TI_WARN( + "Lock {} failed. You can run 'ti ticache clean -p {}' and try " + "again.", + lock_path, path_); + } } } } diff --git a/taichi/cache/metal/cache_manager.cpp b/taichi/cache/metal/cache_manager.cpp index ef2207821a3bb..3c3e40b189c1f 100644 --- a/taichi/cache/metal/cache_manager.cpp +++ b/taichi/cache/metal/cache_manager.cpp @@ -33,7 +33,7 @@ struct CacheCleanerUtils { static std::vector get_cache_files( const CacheCleanerConfig &config, const KernelMetaData &kernel_meta) { - std::string fn = kernel_meta.kernel_key + ".metal"; + std::string fn = kernel_meta.kernel_key + "." + kMetalCacheFilenameExt; return {fn}; } @@ -46,7 +46,7 @@ struct CacheCleanerUtils { // To check if a file is cache file static bool is_valid_cache_file(const CacheCleanerConfig &config, const std::string &name) { - return filename_extension(name) == "metal"; + return filename_extension(name) == kMetalCacheFilenameExt; } }; @@ -62,6 +62,10 @@ CacheManager::CacheManager(Params &&init_params) if (lock_with_file(lock_path)) { auto _ = make_unlocker(lock_path); offline_cache::load_metadata_with_checking(cached_data_, filepath); + } else { + TI_WARN( + "Lock {} failed. You can run 'ti ticache clean -p {}' and try again.", + lock_path, config_.cache_path); } } } diff --git a/taichi/python/export_misc.cpp b/taichi/python/export_misc.cpp index 04f5edb585fa9..d7fd3ada2165b 100644 --- a/taichi/python/export_misc.cpp +++ b/taichi/python/export_misc.cpp @@ -19,6 +19,7 @@ #include "taichi/system/dynamic_loader.h" #include "taichi/system/hacked_signal_handler.h" #include "taichi/system/profiler.h" +#include "taichi/util/offline_cache.h" #if defined(TI_WITH_CUDA) #include "taichi/rhi/cuda/cuda_driver.h" #endif @@ -178,6 +179,9 @@ void export_misc(py::module &m) { m.def("with_cc", []() { return false; }); #endif + m.def("clean_offline_cache_files", + lang::offline_cache::clean_offline_cache_files); + py::class_(m, "HackedSignalRegister").def(py::init<>()); } diff --git a/taichi/runtime/llvm/llvm_offline_cache.cpp b/taichi/runtime/llvm/llvm_offline_cache.cpp index 8a9b348b0a76b..c77acecef6c7d 100644 --- a/taichi/runtime/llvm/llvm_offline_cache.cpp +++ b/taichi/runtime/llvm/llvm_offline_cache.cpp @@ -38,8 +38,8 @@ static std::string get_llvm_cache_metadata_json_file_path( static std::vector get_possible_llvm_cache_filename_by_key( const std::string &key) { return { - key + ".ll", - key + ".bc", + key + "." + offline_cache::kLlvmCacheFilenameLLExt, + key + "." + offline_cache::kLlvmCacheFilenameBCExt, }; } @@ -90,7 +90,7 @@ struct CacheCleanerUtils { static bool is_valid_cache_file(const CacheCleanerConfig &config, const std::string &name) { std::string ext = filename_extension(name); - return ext == "ll" || ext == "bc"; + return ext == kLlvmCacheFilenameLLExt || ext == kLlvmCacheFilenameBCExt; } }; @@ -138,9 +138,8 @@ bool LlvmOfflineCacheFileReader::load_meta_data( }); return Error::kNoError == load_metadata_with_checking(data, tcb_path); } - TI_WARN( - "Lock {} failed. You can remove this .lock file manually and try again.", - lock_path); + TI_WARN("Lock {} failed. You can run 'ti ticache clean -p {}' and try again.", + lock_path, cache_file_path); return false; } @@ -224,12 +223,15 @@ std::unique_ptr LlvmOfflineCacheFileReader::load_module( TI_AUTO_PROF; if (format_ & Format::BC) { LlvmModuleBitcodeLoader loader; - return loader.set_bitcode_path(path_prefix + ".bc") + return loader + .set_bitcode_path(path_prefix + "." + + offline_cache::kLlvmCacheFilenameBCExt) .set_buffer_id(key) .set_inline_funcs(false) .load(&llvm_ctx); } else if (format_ & Format::LL) { - const std::string filename = path_prefix + ".ll"; + const std::string filename = + path_prefix + "." + offline_cache::kLlvmCacheFilenameLLExt; llvm::SMDiagnostic err; auto ret = llvm::parseAssemblyFile(filename, err, llvm_ctx); if (!ret) { // File not found or Parse failed @@ -270,7 +272,8 @@ void LlvmOfflineCacheFileWriter::dump(const std::string &path, auto *mod = data.module.get(); TI_ASSERT(mod != nullptr); if (format & Format::LL) { - std::string filename = filename_prefix + ".ll"; + std::string filename = + filename_prefix + "." + offline_cache::kLlvmCacheFilenameLLExt; if (!merge_with_old || try_lock_with_file(filename)) { size += write_llvm_module(filename, [mod](llvm::raw_os_ostream &os) { mod->print(os, /*AAW=*/nullptr); @@ -280,7 +283,8 @@ void LlvmOfflineCacheFileWriter::dump(const std::string &path, } } if (format & Format::BC) { - std::string filename = filename_prefix + ".bc"; + std::string filename = + filename_prefix + "." + offline_cache::kLlvmCacheFilenameBCExt; if (!merge_with_old || try_lock_with_file(filename)) { size += write_llvm_module(filename, [mod](llvm::raw_os_ostream &os) { llvm::WriteBitcodeToFile(*mod, os); @@ -319,9 +323,8 @@ void LlvmOfflineCacheFileWriter::dump(const std::string &path, std::string lock_path = taichi::join_path(path, kMetadataFileLockName); if (!lock_with_file(lock_path)) { TI_WARN( - "Lock {} failed. You can remove this .lock file manually and try " - "again.", - lock_path); + "Lock {} failed. You can run 'ti ticache clean -p {}' and try again.", + lock_path, path); return; } auto _ = make_cleanup([&lock_path]() { diff --git a/taichi/util/offline_cache.cpp b/taichi/util/offline_cache.cpp index e298da7fd6df2..67a5e3b523b36 100644 --- a/taichi/util/offline_cache.cpp +++ b/taichi/util/offline_cache.cpp @@ -21,11 +21,11 @@ void disable_offline_cache_if_needed(CompileConfig *config) { std::string get_cache_path_by_arch(const std::string &base_path, Arch arch) { std::string subdir; if (arch_uses_llvm(arch)) { - subdir = "llvm"; + subdir = kLlvmCachSubPath; } else if (arch == Arch::vulkan || arch == Arch::opengl) { - subdir = "gfx"; + subdir = kSpirvCacheSubPath; } else if (arch == Arch::metal) { - subdir = "metal"; + subdir = kMetalCacheSubPath; } else if (arch == Arch::dx12) { subdir = "dx12"; } else { @@ -87,4 +87,39 @@ bool try_demangle_name(const std::string &mangled_name, return true; } +void clean_offline_cache_files(const std::string &path) { + std::vector sub_dirs = {kLlvmCachSubPath, kSpirvCacheSubPath, + kMetalCacheSubPath}; + auto is_cache_filename = [](const std::string &name) { + const auto ext = taichi::filename_extension(name); + return ext == kLlvmCacheFilenameBCExt || ext == kLlvmCacheFilenameLLExt || + ext == kSpirvCacheFilenameExt || ext == kMetalCacheFilenameExt || + ext == "lock" || ext == "tcb"; + }; + // Temp implementation. We will refactor the offline cache + taichi::traverse_directory(path, [&sub_dirs, &is_cache_filename, &path]( + const std::string &name, bool is_dir) { + if (is_dir) { // ~/.cache/taichi/ticache/llvm ... + for (auto subdir : sub_dirs) { + auto subpath = taichi::join_path(path, subdir); + if (taichi::path_exists(subpath)) { + taichi::traverse_directory( + subpath, [&is_cache_filename, &subpath](const std::string &name, + bool is_dir) { + if (is_cache_filename(name) && !is_dir) { + const auto fpath = taichi::join_path(subpath, name); + TI_TRACE("Remove {}", fpath); + taichi::remove(fpath); + } + }); + } + } + } else if (is_cache_filename(name)) { + const auto fpath = taichi::join_path(path, name); + TI_TRACE("Remove {}", fpath); + taichi::remove(fpath); + } + }); +} + } // namespace taichi::lang::offline_cache diff --git a/taichi/util/offline_cache.h b/taichi/util/offline_cache.h index e96aa186f5e96..e1a77953f30f0 100644 --- a/taichi/util/offline_cache.h +++ b/taichi/util/offline_cache.h @@ -18,6 +18,14 @@ namespace taichi::lang { namespace offline_cache { +constexpr char kLlvmCacheFilenameLLExt[] = "ll"; +constexpr char kLlvmCacheFilenameBCExt[] = "bc"; +constexpr char kSpirvCacheFilenameExt[] = "spv"; +constexpr char kMetalCacheFilenameExt[] = "metal"; +constexpr char kLlvmCachSubPath[] = "llvm"; +constexpr char kSpirvCacheSubPath[] = "gfx"; +constexpr char kMetalCacheSubPath[] = "metal"; + using Version = std::uint16_t[3]; // {MAJOR, MINOR, PATCH} enum CleanCacheFlags { @@ -174,9 +182,9 @@ class CacheCleaner { taichi::join_path(path, config.metadata_lock_name); if (!lock_with_file(lock_path)) { TI_WARN( - "Lock {} failed. You can remove this .lock file manually and try " + "Lock {} failed. You can run 'ti ticache clean -p {}' and try " "again.", - lock_path); + lock_path, path); return; } auto _ = make_cleanup([&lock_path]() { @@ -293,5 +301,8 @@ bool try_demangle_name(const std::string &mangled_name, std::string &primal_name, std::string &key); +// utils to manage ticache files +void clean_offline_cache_files(const std::string &path); + } // namespace offline_cache } // namespace taichi::lang diff --git a/tests/python/test_cli.py b/tests/python/test_cli.py index 17dba3fa5f911..341f691a6c5fe 100644 --- a/tests/python/test_cli.py +++ b/tests/python/test_cli.py @@ -1,12 +1,16 @@ import argparse +import tempfile +import shutil import copy import sys +import os from contextlib import contextmanager from pathlib import Path from unittest.mock import patch import pytest from taichi._main import TaichiMain +from tests import test_utils import taichi as ti @@ -206,3 +210,44 @@ def test_cli_run(): cli = TaichiMain(test_mode=True) args = cli() assert args.filename == "a.py" + + +def test_cli_ticache(): + archs = {ti.cpu, ti.cuda, ti.opengl, ti.vulkan, ti.metal} + archs = {v for v in archs if v in test_utils.expected_archs()} + exts = ('ll', 'bc', 'spv', 'metal', 'tcb', 'lock') + tmp_path = tempfile.mkdtemp() + + @ti.kernel + def simple_kernel(a: ti.i32) -> ti.i32: + return -a + + def launch_kernel(arch): + ti.init(arch=arch, + offline_cache=True, + offline_cache_file_path=tmp_path) + simple_kernel(128) + ti.reset() + + for arch in archs: + launch_kernel(arch) + + found = False + for root, dirs, files in os.walk(tmp_path): + for file in files: + if file.endswith(exts): + found = True + break + if found: break + assert found + + with patch_sys_argv_helper(["ti", "ticache", "clean", "-p", + tmp_path]) as custom_argv: + cli = TaichiMain() + cli() + + for root, dirs, files in os.walk(tmp_path): + for file in files: + assert not file.endswith(exts) + + shutil.rmtree(tmp_path) From e2d34f0387e5e6541b2e2182f66335f4ced9ed73 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 Dec 2022 10:27:21 +0000 Subject: [PATCH 2/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- python/taichi/_main.py | 6 +++--- tests/python/test_cli.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/python/taichi/_main.py b/python/taichi/_main.py index 798067dbe62bf..f440cf70f646e 100644 --- a/python/taichi/_main.py +++ b/python/taichi/_main.py @@ -249,9 +249,9 @@ def colormap(index, name): nrows += 1 names = sorted(choices.keys()) for k in range(nrows): - table.add_row(*[ - colormap(j, names[j]) for j in range(k, len(choices), nrows) - ]) + table.add_row( + * + [colormap(j, names[j]) for j in range(k, len(choices), nrows)]) parser = argparse.ArgumentParser(prog='ti example', description=f"{self.example.__doc__}") diff --git a/tests/python/test_cli.py b/tests/python/test_cli.py index 341f691a6c5fe..51d3a8833ebbe 100644 --- a/tests/python/test_cli.py +++ b/tests/python/test_cli.py @@ -1,18 +1,18 @@ import argparse -import tempfile -import shutil import copy -import sys import os +import shutil +import sys +import tempfile from contextlib import contextmanager from pathlib import Path from unittest.mock import patch import pytest from taichi._main import TaichiMain -from tests import test_utils import taichi as ti +from tests import test_utils @contextmanager From 7bbbcb0d2d977d932cd9c5e90421c37e16de3c49 Mon Sep 17 00:00:00 2001 From: PGZXB Date: Tue, 20 Dec 2022 18:31:26 +0800 Subject: [PATCH 3/8] Code format --- python/taichi/_main.py | 3 ++- tests/python/test_cli.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/taichi/_main.py b/python/taichi/_main.py index f440cf70f646e..c115bad219824 100644 --- a/python/taichi/_main.py +++ b/python/taichi/_main.py @@ -831,7 +831,8 @@ def lint(arguments: list = sys.argv[2:]): @register def ticache(arguments: list = sys.argv[2:]): """Manage the ticache files manually""" - if len(arguments) < 1: return + if len(arguments) < 1: + return subcmd = arguments[0] arguments = arguments[1:] diff --git a/tests/python/test_cli.py b/tests/python/test_cli.py index 51d3a8833ebbe..d7562a60283b5 100644 --- a/tests/python/test_cli.py +++ b/tests/python/test_cli.py @@ -238,7 +238,8 @@ def launch_kernel(arch): if file.endswith(exts): found = True break - if found: break + if found: + break assert found with patch_sys_argv_helper(["ti", "ticache", "clean", "-p", From dfdddafc523c19e91c112bc0a6b7e6a9a4134949 Mon Sep 17 00:00:00 2001 From: PGZXB Date: Wed, 21 Dec 2022 16:55:50 +0800 Subject: [PATCH 4/8] 'ti ticache clean' -> 'ti cache clean' --- python/taichi/_main.py | 14 +++++++------- taichi/cache/gfx/cache_manager.cpp | 3 +-- taichi/cache/metal/cache_manager.cpp | 2 +- taichi/runtime/llvm/llvm_offline_cache.cpp | 2 +- taichi/util/offline_cache.h | 5 ++--- tests/python/test_cli.py | 2 +- 6 files changed, 13 insertions(+), 15 deletions(-) diff --git a/python/taichi/_main.py b/python/taichi/_main.py index c115bad219824..a9b39b9b4cd2d 100644 --- a/python/taichi/_main.py +++ b/python/taichi/_main.py @@ -249,9 +249,9 @@ def colormap(index, name): nrows += 1 names = sorted(choices.keys()) for k in range(nrows): - table.add_row( - * - [colormap(j, names[j]) for j in range(k, len(choices), nrows)]) + table.add_row(*[ + colormap(j, names[j]) for j in range(k, len(choices), nrows) + ]) parser = argparse.ArgumentParser(prog='ti example', description=f"{self.example.__doc__}") @@ -829,8 +829,8 @@ def lint(arguments: list = sys.argv[2:]): @staticmethod @register - def ticache(arguments: list = sys.argv[2:]): - """Manage the ticache files manually""" + def cache(arguments: list = sys.argv[2:]): + """Manage the offline cache files manually""" if len(arguments) < 1: return @@ -838,8 +838,8 @@ def ticache(arguments: list = sys.argv[2:]): arguments = arguments[1:] if subcmd == 'clean': parser = argparse.ArgumentParser( - prog='ti ticache', - description='Clean all ticache files in given path') + prog='ti cache clean', + description='Clean all offline cache files in given path') parser.add_argument( '-p', '--offline-cache-file-path', diff --git a/taichi/cache/gfx/cache_manager.cpp b/taichi/cache/gfx/cache_manager.cpp index b8b8c7726d9c7..d632440874af9 100644 --- a/taichi/cache/gfx/cache_manager.cpp +++ b/taichi/cache/gfx/cache_manager.cpp @@ -139,8 +139,7 @@ CacheManager::CacheManager(Params &&init_params) cached_module_ = gfx::make_aot_module(params, init_params.arch); } else { TI_WARN( - "Lock {} failed. You can run 'ti ticache clean -p {}' and try " - "again.", + "Lock {} failed. You can run 'ti cache clean -p {}' and try again.", lock_path, path_); } } diff --git a/taichi/cache/metal/cache_manager.cpp b/taichi/cache/metal/cache_manager.cpp index 3c3e40b189c1f..040a769a046da 100644 --- a/taichi/cache/metal/cache_manager.cpp +++ b/taichi/cache/metal/cache_manager.cpp @@ -64,7 +64,7 @@ CacheManager::CacheManager(Params &&init_params) offline_cache::load_metadata_with_checking(cached_data_, filepath); } else { TI_WARN( - "Lock {} failed. You can run 'ti ticache clean -p {}' and try again.", + "Lock {} failed. You can run 'ti cache clean -p {}' and try again.", lock_path, config_.cache_path); } } diff --git a/taichi/runtime/llvm/llvm_offline_cache.cpp b/taichi/runtime/llvm/llvm_offline_cache.cpp index c77acecef6c7d..e933235143e45 100644 --- a/taichi/runtime/llvm/llvm_offline_cache.cpp +++ b/taichi/runtime/llvm/llvm_offline_cache.cpp @@ -138,7 +138,7 @@ bool LlvmOfflineCacheFileReader::load_meta_data( }); return Error::kNoError == load_metadata_with_checking(data, tcb_path); } - TI_WARN("Lock {} failed. You can run 'ti ticache clean -p {}' and try again.", + TI_WARN("Lock {} failed. You can run 'ti cache clean -p {}' and try again.", lock_path, cache_file_path); return false; } diff --git a/taichi/util/offline_cache.h b/taichi/util/offline_cache.h index e1a77953f30f0..6594cc6e79013 100644 --- a/taichi/util/offline_cache.h +++ b/taichi/util/offline_cache.h @@ -182,8 +182,7 @@ class CacheCleaner { taichi::join_path(path, config.metadata_lock_name); if (!lock_with_file(lock_path)) { TI_WARN( - "Lock {} failed. You can run 'ti ticache clean -p {}' and try " - "again.", + "Lock {} failed. You can run 'ti cache clean -p {}' and try again.", lock_path, path); return; } @@ -301,7 +300,7 @@ bool try_demangle_name(const std::string &mangled_name, std::string &primal_name, std::string &key); -// utils to manage ticache files +// utils to manage the offline cache files void clean_offline_cache_files(const std::string &path); } // namespace offline_cache diff --git a/tests/python/test_cli.py b/tests/python/test_cli.py index d7562a60283b5..8d828f967bd8c 100644 --- a/tests/python/test_cli.py +++ b/tests/python/test_cli.py @@ -242,7 +242,7 @@ def launch_kernel(arch): break assert found - with patch_sys_argv_helper(["ti", "ticache", "clean", "-p", + with patch_sys_argv_helper(["ti", "cache", "clean", "-p", tmp_path]) as custom_argv: cli = TaichiMain() cli() From fd41763d083fe72059b8cba110375254e97f14b2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 21 Dec 2022 08:57:33 +0000 Subject: [PATCH 5/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- python/taichi/_main.py | 6 +++--- taichi/cache/gfx/cache_manager.cpp | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python/taichi/_main.py b/python/taichi/_main.py index a9b39b9b4cd2d..83b877bf6d3a5 100644 --- a/python/taichi/_main.py +++ b/python/taichi/_main.py @@ -249,9 +249,9 @@ def colormap(index, name): nrows += 1 names = sorted(choices.keys()) for k in range(nrows): - table.add_row(*[ - colormap(j, names[j]) for j in range(k, len(choices), nrows) - ]) + table.add_row( + * + [colormap(j, names[j]) for j in range(k, len(choices), nrows)]) parser = argparse.ArgumentParser(prog='ti example', description=f"{self.example.__doc__}") diff --git a/taichi/cache/gfx/cache_manager.cpp b/taichi/cache/gfx/cache_manager.cpp index d632440874af9..faa899f4cd170 100644 --- a/taichi/cache/gfx/cache_manager.cpp +++ b/taichi/cache/gfx/cache_manager.cpp @@ -139,7 +139,8 @@ CacheManager::CacheManager(Params &&init_params) cached_module_ = gfx::make_aot_module(params, init_params.arch); } else { TI_WARN( - "Lock {} failed. You can run 'ti cache clean -p {}' and try again.", + "Lock {} failed. You can run 'ti cache clean -p {}' and try " + "again.", lock_path, path_); } } From 84338b6cbea4e9a27d378d95dd00483049080e3c Mon Sep 17 00:00:00 2001 From: PGZXB Date: Wed, 21 Dec 2022 17:01:03 +0800 Subject: [PATCH 6/8] 'test_cli_ticache' -> 'test_cli_cache' --- tests/python/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/test_cli.py b/tests/python/test_cli.py index 8d828f967bd8c..2ab78624408d0 100644 --- a/tests/python/test_cli.py +++ b/tests/python/test_cli.py @@ -212,7 +212,7 @@ def test_cli_run(): assert args.filename == "a.py" -def test_cli_ticache(): +def test_cli_cache(): archs = {ti.cpu, ti.cuda, ti.opengl, ti.vulkan, ti.metal} archs = {v for v in archs if v in test_utils.expected_archs()} exts = ('ll', 'bc', 'spv', 'metal', 'tcb', 'lock') From 02cce8c68fb171757836c3e0ead667ac6a70a637 Mon Sep 17 00:00:00 2001 From: PGZXB Date: Wed, 21 Dec 2022 19:34:27 +0800 Subject: [PATCH 7/8] Update python/taichi/_main.py Co-authored-by: Lin Jiang --- python/taichi/_main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/taichi/_main.py b/python/taichi/_main.py index 83b877bf6d3a5..e271bdc34e12d 100644 --- a/python/taichi/_main.py +++ b/python/taichi/_main.py @@ -831,7 +831,7 @@ def lint(arguments: list = sys.argv[2:]): @register def cache(arguments: list = sys.argv[2:]): """Manage the offline cache files manually""" - if len(arguments) < 1: + if not arguments: return subcmd = arguments[0] From 2df3d8f5f7e7c09a252de9fefcea8e215b46402c Mon Sep 17 00:00:00 2001 From: PGZXB Date: Wed, 21 Dec 2022 21:14:33 +0800 Subject: [PATCH 8/8] Add some hint msgs --- python/taichi/_main.py | 43 ++++++++++++++++++-------- taichi/util/offline_cache.cpp | 58 +++++++++++++++++++++-------------- taichi/util/offline_cache.h | 2 +- 3 files changed, 67 insertions(+), 36 deletions(-) diff --git a/python/taichi/_main.py b/python/taichi/_main.py index e271bdc34e12d..46d8228bc304c 100644 --- a/python/taichi/_main.py +++ b/python/taichi/_main.py @@ -831,23 +831,42 @@ def lint(arguments: list = sys.argv[2:]): @register def cache(arguments: list = sys.argv[2:]): """Manage the offline cache files manually""" - if not arguments: - return - - subcmd = arguments[0] - arguments = arguments[1:] - if subcmd == 'clean': - parser = argparse.ArgumentParser( - prog='ti cache clean', - description='Clean all offline cache files in given path') + def clean(cmd_args, parser): parser.add_argument( '-p', '--offline-cache-file-path', dest='offline_cache_file_path', default=impl.default_cfg().offline_cache_file_path) - args = parser.parse_args(arguments) - path = args.offline_cache_file_path - _ti_core.clean_offline_cache_files(path) + args = parser.parse_args(cmd_args) + path = os.path.abspath(args.offline_cache_file_path) + count = _ti_core.clean_offline_cache_files(path) + print(f'Deleted {count} offline cache files in {path}') + + # TODO(PGZXB): Provide more tools to manage the offline cache files + subcmds_map = { + 'clean': (clean, 'Clean all offline cache files in given path') + } + + def print_help(): + print('usage: ti cache []') + for name, value in subcmds_map.items(): + _, description = value + print(f'\t{name}\t|-> {description}') + + if not arguments: + print_help() + return + + subcmd = arguments[0] + if subcmd not in subcmds_map: + print(f"'ti cache {subcmd}' is not a valid command!") + print_help() + return + + func, description = subcmds_map[subcmd] + parser = argparse.ArgumentParser(prog=f'ti cache {subcmd}', + description=description) + func(cmd_args=arguments[1:], parser=parser) def main(): diff --git a/taichi/util/offline_cache.cpp b/taichi/util/offline_cache.cpp index 67a5e3b523b36..2b9150a09ba69 100644 --- a/taichi/util/offline_cache.cpp +++ b/taichi/util/offline_cache.cpp @@ -87,39 +87,51 @@ bool try_demangle_name(const std::string &mangled_name, return true; } -void clean_offline_cache_files(const std::string &path) { +std::size_t clean_offline_cache_files(const std::string &path) { std::vector sub_dirs = {kLlvmCachSubPath, kSpirvCacheSubPath, kMetalCacheSubPath}; + auto is_cache_filename = [](const std::string &name) { const auto ext = taichi::filename_extension(name); return ext == kLlvmCacheFilenameBCExt || ext == kLlvmCacheFilenameLLExt || ext == kSpirvCacheFilenameExt || ext == kMetalCacheFilenameExt || ext == "lock" || ext == "tcb"; }; + + std::size_t count = 0; + // Temp implementation. We will refactor the offline cache - taichi::traverse_directory(path, [&sub_dirs, &is_cache_filename, &path]( - const std::string &name, bool is_dir) { - if (is_dir) { // ~/.cache/taichi/ticache/llvm ... - for (auto subdir : sub_dirs) { - auto subpath = taichi::join_path(path, subdir); - if (taichi::path_exists(subpath)) { - taichi::traverse_directory( - subpath, [&is_cache_filename, &subpath](const std::string &name, - bool is_dir) { - if (is_cache_filename(name) && !is_dir) { - const auto fpath = taichi::join_path(subpath, name); - TI_TRACE("Remove {}", fpath); - taichi::remove(fpath); - } - }); + taichi::traverse_directory( + path, [&count, &sub_dirs, &is_cache_filename, &path]( + const std::string &name, bool is_dir) { + if (is_dir) { // ~/.cache/taichi/ticache/llvm ... + for (auto subdir : sub_dirs) { + auto subpath = taichi::join_path(path, subdir); + + if (taichi::path_exists(subpath)) { + taichi::traverse_directory( + subpath, [&count, &is_cache_filename, &subpath]( + const std::string &name, bool is_dir) { + if (is_cache_filename(name) && !is_dir) { + const auto fpath = taichi::join_path(subpath, name); + TI_TRACE("Removing {}", fpath); + bool ok = taichi::remove(fpath); + count += ok ? 1 : 0; + TI_WARN_IF(!ok, "Remove {} failed", fpath); + } + }); + } + } + } else if (is_cache_filename(name)) { + const auto fpath = taichi::join_path(path, name); + TI_TRACE("Removing {}", fpath); + bool ok = taichi::remove(fpath); + count += ok ? 1 : 0; + TI_WARN_IF(!ok, "Remove {} failed", fpath); } - } - } else if (is_cache_filename(name)) { - const auto fpath = taichi::join_path(path, name); - TI_TRACE("Remove {}", fpath); - taichi::remove(fpath); - } - }); + }); + + return count; } } // namespace taichi::lang::offline_cache diff --git a/taichi/util/offline_cache.h b/taichi/util/offline_cache.h index 6594cc6e79013..cfb856644a1dc 100644 --- a/taichi/util/offline_cache.h +++ b/taichi/util/offline_cache.h @@ -301,7 +301,7 @@ bool try_demangle_name(const std::string &mangled_name, std::string &key); // utils to manage the offline cache files -void clean_offline_cache_files(const std::string &path); +std::size_t clean_offline_cache_files(const std::string &path); } // namespace offline_cache } // namespace taichi::lang