diff --git a/taichi/cache/gfx/cache_manager.h b/taichi/cache/gfx/cache_manager.h index 6d262250c4a02..da7ec5290f0ce 100644 --- a/taichi/cache/gfx/cache_manager.h +++ b/taichi/cache/gfx/cache_manager.h @@ -8,10 +8,14 @@ namespace taichi::lang { namespace gfx { -struct OfflineCacheKernelMetadata : public offline_cache::KernelMetadataBase { +struct OfflineCacheKernelMetadata { + std::string kernel_key; + std::size_t size{0}; // byte + std::time_t created_at{0}; // sec + std::time_t last_used_at{0}; // sec std::size_t num_files{0}; - TI_IO_DEF_WITH_BASECLASS(offline_cache::KernelMetadataBase, num_files); + TI_IO_DEF(kernel_key, size, created_at, last_used_at, num_files); }; class CacheManager { diff --git a/taichi/cache/metal/cache_manager.h b/taichi/cache/metal/cache_manager.h index 91e97b57c2f62..4bf2aa1b50915 100644 --- a/taichi/cache/metal/cache_manager.h +++ b/taichi/cache/metal/cache_manager.h @@ -9,11 +9,14 @@ namespace taichi::lang { namespace metal { -struct OfflineCacheKernelMetadata : public offline_cache::KernelMetadataBase { +struct OfflineCacheKernelMetadata { + std::string kernel_key; + std::size_t size{0}; // byte + std::time_t created_at{0}; // sec + std::time_t last_used_at{0}; // sec CompiledKernelData compiled_kernel_data; - TI_IO_DEF_WITH_BASECLASS(offline_cache::KernelMetadataBase, - compiled_kernel_data); + TI_IO_DEF(kernel_key, size, created_at, last_used_at, compiled_kernel_data); }; class CacheManager { diff --git a/taichi/common/CMakeLists.txt b/taichi/common/CMakeLists.txt index ff92d1e1da5f6..719a2dee9452a 100644 --- a/taichi/common/CMakeLists.txt +++ b/taichi/common/CMakeLists.txt @@ -5,6 +5,7 @@ target_sources(taichi_common PRIVATE cleanup.cpp core.cpp + json.cpp logging.cpp symbol_version.cpp ) diff --git a/taichi/common/json.cpp b/taichi/common/json.cpp new file mode 100644 index 0000000000000..ec9cb197392f6 --- /dev/null +++ b/taichi/common/json.cpp @@ -0,0 +1,423 @@ +// Adapted from https://github.com/PENGUINLIONG/graphi-t + +// Copyright (c) 2019 Rendong Liang +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +// JSON serialization/deserialization. +// @PENGUINLIONG +#include +//#include "gft/log.hpp" +#include "taichi/common/json.h" + +namespace liong { +namespace json { + +enum JsonTokenType { + L_JSON_TOKEN_UNDEFINED, + L_JSON_TOKEN_NULL, + L_JSON_TOKEN_TRUE, + L_JSON_TOKEN_FALSE, + L_JSON_TOKEN_STRING, + L_JSON_TOKEN_INT, + L_JSON_TOKEN_FLOAT, + L_JSON_TOKEN_COLON, + L_JSON_TOKEN_COMMA, + L_JSON_TOKEN_OPEN_BRACE, + L_JSON_TOKEN_CLOSE_BRACE, + L_JSON_TOKEN_OPEN_BRACKET, + L_JSON_TOKEN_CLOSE_BRACKET, +}; +struct JsonToken { + JsonTokenType ty; + int64_t num_int; + double num_float; + std::string str; +}; + +struct Tokenizer { + std::string lit; + std::string::const_iterator pos; + std::string::const_iterator end; + + explicit Tokenizer(const std::string &json) + : lit(json), pos(lit.cbegin()), end(lit.cend()) { + } + + // Check the range first before calling this method. + bool unsafe_starts_with(const char *head) { + auto i = 0; + while (*head != '\0') { + if (pos[i++] != *(head++)) { + return false; + } + } + return true; + } + bool next_token(JsonToken &out) { + std::stringstream ss; + while (pos != end) { + char c = *pos; + + // Ignore whitespaces. + if (c == ' ' || c == '\t' || c == '\r' || c == '\n') { + pos += 1; + continue; + } + + // Try parse scope punctuations. + switch (c) { + case ':': + out.ty = L_JSON_TOKEN_COLON; + pos += 1; + return true; + case ',': + out.ty = L_JSON_TOKEN_COMMA; + pos += 1; + return true; + case '{': + out.ty = L_JSON_TOKEN_OPEN_BRACE; + pos += 1; + return true; + case '}': + out.ty = L_JSON_TOKEN_CLOSE_BRACE; + pos += 1; + return true; + case '[': + out.ty = L_JSON_TOKEN_OPEN_BRACKET; + pos += 1; + return true; + case ']': + out.ty = L_JSON_TOKEN_CLOSE_BRACKET; + pos += 1; + return true; + } + + // Try parse numbers. + if (c == '+' || c == '-' || (c >= '0' && c <= '9')) { + out.ty = L_JSON_TOKEN_INT; + const int STATE_INTEGRAL = 0; + const int STATE_FRACTION = 1; + const int STATE_EXPONENT = 2; + int state = STATE_INTEGRAL; + do { + c = *pos; + if (state == STATE_INTEGRAL) { + if (c == '.') { + state = STATE_FRACTION; + ss.put(c); + continue; + } + if (c == 'e') { + state = STATE_EXPONENT; + ss.put(c); + continue; + } + if (c != '+' && c != '-' && (c < '0' || c > '9')) { + break; + } + } else if (state == STATE_FRACTION) { + out.ty = L_JSON_TOKEN_FLOAT; + if (c == 'e') { + state = STATE_EXPONENT; + ss.put(c); + continue; + } + if (c < '0' || c > '9') { + break; + } + } else if (state == STATE_EXPONENT) { + out.ty = L_JSON_TOKEN_FLOAT; + if (c != '+' && c != '-' && (c < '0' || c > '9')) { + break; + } + } + ss.put(c); + } while (++pos != end); + if (out.ty == L_JSON_TOKEN_INT) { + out.num_int = std::atoll(ss.str().c_str()); + } else if (out.ty == L_JSON_TOKEN_FLOAT) { + out.num_float = std::atof(ss.str().c_str()); + } + return true; + } + + // Try parse strings. + if (c == '"') { + out.ty = L_JSON_TOKEN_STRING; + bool escape = false; + while (++pos != end) { + c = *pos; + if (escape) { + switch (c) { + case '"': + case '/': + break; + case 'b': + c = '\b'; + break; + case 'f': + c = '\f'; + break; + case 'n': + c = '\n'; + break; + case 'r': + c = '\r'; + break; + case 't': + c = '\t'; + break; + case 'u': + throw JsonException("unicode escape is not supported"); + default: + throw JsonException("invalid escape charater"); + } + escape = false; + } else { + if (c == '\\') { + escape = true; + continue; + } else if (c == '"') { + if (escape != false) { + throw JsonException("incomplete escape sequence"); + } + out.str = ss.str(); + pos += 1; + return true; + } + } + ss.put(c); + } + throw JsonException("unexpected end of string"); + } + + // Try parse literals. + if (pos + 4 <= end) { + if (unsafe_starts_with("null")) { + out.ty = L_JSON_TOKEN_NULL; + pos += 4; + return true; + } + if (unsafe_starts_with("true")) { + out.ty = L_JSON_TOKEN_TRUE; + pos += 4; + return true; + } + } + if (pos + 5 <= end) { + if (unsafe_starts_with("false")) { + out.ty = L_JSON_TOKEN_FALSE; + pos += 5; + return true; + } + } + } + out.ty = L_JSON_TOKEN_UNDEFINED; + return false; + } +}; + +bool try_parse_impl(Tokenizer &tokenizer, JsonValue &out) { + JsonToken token; + while (tokenizer.next_token(token)) { + JsonValue val; + switch (token.ty) { + case L_JSON_TOKEN_TRUE: + out.ty = L_JSON_BOOLEAN; + out.b = true; + return true; + case L_JSON_TOKEN_FALSE: + out.ty = L_JSON_BOOLEAN; + out.b = false; + return true; + case L_JSON_TOKEN_NULL: + out.ty = L_JSON_NULL; + return true; + case L_JSON_TOKEN_STRING: + out.ty = L_JSON_STRING; + out.str = std::move(token.str); + return true; + case L_JSON_TOKEN_INT: + out.ty = L_JSON_INT; + out.num_int = token.num_int; + return true; + case L_JSON_TOKEN_FLOAT: + out.ty = L_JSON_FLOAT; + out.num_int = token.num_float; + return true; + case L_JSON_TOKEN_OPEN_BRACKET: + out.ty = L_JSON_ARRAY; + for (;;) { + if (!try_parse_impl(tokenizer, val)) { + // When the array has no element. + break; + } + out.arr.inner.emplace_back(std::move(val)); + if (tokenizer.next_token(token)) { + if (token.ty == L_JSON_TOKEN_COMMA) { + continue; + } else if (token.ty == L_JSON_TOKEN_CLOSE_BRACKET) { + break; + } else { + throw JsonException("unexpected token in array"); + } + } else { + throw JsonException("unexpected end of array"); + } + } + return true; + case L_JSON_TOKEN_OPEN_BRACE: + out.ty = L_JSON_OBJECT; + for (;;) { + // Match the key. + std::string key; + if (tokenizer.next_token(token)) { + if (token.ty == L_JSON_TOKEN_STRING) { + key = std::move(token.str); + } else if (token.ty == L_JSON_TOKEN_CLOSE_BRACE) { + // The object has no field. + break; + } else { + throw JsonException("unexpected object field key type"); + } + } else { + throw JsonException("unexpected end of object"); + } + // Match the colon. + if (!tokenizer.next_token(token)) { + throw JsonException("unexpected end of object"); + } + if (token.ty != L_JSON_TOKEN_COLON) { + throw JsonException("unexpected token in object"); + } + // Match the value. + if (!try_parse_impl(tokenizer, val)) { + throw JsonException("unexpected end of object"); + } + out.obj.inner[key] = std::move(val); + // Should we head for another round? + if (tokenizer.next_token(token)) { + if (token.ty == L_JSON_TOKEN_COMMA) { + continue; + } else if (token.ty == L_JSON_TOKEN_CLOSE_BRACE) { + break; + } else { + throw JsonException("unexpected token in object"); + } + } else { + throw JsonException("unexpected end of object"); + } + } + return true; + case L_JSON_TOKEN_CLOSE_BRACE: + case L_JSON_TOKEN_CLOSE_BRACKET: + return false; + default: + throw JsonException("unexpected token"); + } + } + throw JsonException("unexpected program state"); +} + +JsonValue parse(const std::string &json_lit) { + if (json_lit.empty()) { + throw JsonException("json text is empty"); + } + JsonValue rv; + Tokenizer tokenizer(json_lit); + if (!try_parse_impl(tokenizer, rv)) { + throw JsonException("unexpected close token"); + } + return rv; +} +bool try_parse(const std::string &json_lit, JsonValue &out) { + try { + out = parse(json_lit); + } catch (JsonException e) { + // log::error("failed to parse json: ", e.what()); + return true; + } + return false; +} + +void print_impl(const JsonValue &json, std::stringstream &out) { + switch (json.ty) { + case L_JSON_NULL: + out << "null"; + return; + case L_JSON_BOOLEAN: + out << (json.b ? "true" : "false"); + return; + case L_JSON_FLOAT: + out << json.num_float; + return; + case L_JSON_INT: + out << json.num_int; + return; + case L_JSON_STRING: + out << "\"" << json.str << "\""; + return; + case L_JSON_OBJECT: + out << "{"; + { + bool is_first_iter = true; + for (const auto &pair : json.obj.inner) { + if (is_first_iter) { + is_first_iter = false; + } else { + out << ","; + } + out << "\"" << pair.first << "\":"; + print_impl(pair.second, out); + } + } + out << "}"; + return; + case L_JSON_ARRAY: + out << "["; + { + bool is_first_iter = true; + for (const auto &elem : json.arr.inner) { + if (is_first_iter) { + is_first_iter = false; + } else { + out << ","; + } + print_impl(elem, out); + } + } + out << "]"; + return; + } +} +std::string print(const JsonValue &json) { + std::stringstream ss; + print_impl(json, ss); + return ss.str(); +} + +} // namespace json +} // namespace liong diff --git a/taichi/common/json.h b/taichi/common/json.h new file mode 100644 index 0000000000000..1261ce485b586 --- /dev/null +++ b/taichi/common/json.h @@ -0,0 +1,366 @@ +// Adapted from https://github.com/PENGUINLIONG/graphi-t + +// Copyright (c) 2019 Rendong Liang +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +// JSON serialization/deserialization. +// @PENGUINLIONG +#pragma once +#include +#include +#include +#include + +namespace liong { +namespace json { + +// Any error occured during JSON serialization/deserialization. +class JsonException : public std::exception { + private: + std::string msg_; + + public: + explicit JsonException(const char *msg) : msg_(msg) { + } + const char *what() const noexcept override { + return msg_.c_str(); + } +}; + +// Type of JSON value. +enum JsonType { + L_JSON_NULL, + L_JSON_BOOLEAN, + L_JSON_FLOAT, + L_JSON_INT, + L_JSON_STRING, + L_JSON_OBJECT, + L_JSON_ARRAY, +}; + +struct JsonValue; + +class JsonElementEnumerator { + std::vector::const_iterator beg_, end_; + + public: + explicit JsonElementEnumerator(const std::vector &arr) + : beg_(arr.cbegin()), end_(arr.cend()) { + } + + std::vector::const_iterator begin() const { + return beg_; + } + std::vector::const_iterator end() const { + return end_; + } +}; + +class JsonFieldEnumerator { + std::map::const_iterator beg_, end_; + + public: + explicit JsonFieldEnumerator(const std::map &obj) + : beg_(obj.cbegin()), end_(obj.cend()) { + } + + std::map::const_iterator begin() const { + return beg_; + } + std::map::const_iterator end() const { + return end_; + } +}; + +// JSON array builder. +struct JsonArray { + std::vector inner; + + inline JsonArray() = default; + inline explicit JsonArray(std::vector &&b) : inner(std::move(b)) { + } + inline JsonArray(std::initializer_list &&elems) : inner(elems) { + } +}; +// JSON object builder. +struct JsonObject { + std::map inner; + + inline JsonObject() = default; + inline explicit JsonObject(std::map &&b) + : inner(std::move(b)) { + } + inline JsonObject( + std::initializer_list> &&fields) + : inner(fields) { + } +}; + +// Represent a abstract value in JSON representation. +struct JsonValue { + JsonType ty; + bool b; + int64_t num_int; + double num_float; + std::string str; + JsonObject obj; + JsonArray arr; + + inline explicit JsonValue() : ty(L_JSON_NULL) { + } + inline explicit JsonValue(std::nullptr_t) : ty(L_JSON_NULL) { + } + inline explicit JsonValue(bool b) : ty(L_JSON_BOOLEAN), b(b) { + } + inline explicit JsonValue(double num) : ty(L_JSON_FLOAT), num_float(num) { + } + inline explicit JsonValue(float num) : ty(L_JSON_FLOAT), num_float(num) { + } + inline explicit JsonValue(char num) : ty(L_JSON_INT), num_int(num) { + } + inline explicit JsonValue(signed char num) : ty(L_JSON_INT), num_int(num) { + } + inline explicit JsonValue(unsigned char num) : ty(L_JSON_INT), num_int(num) { + } + inline explicit JsonValue(short num) : ty(L_JSON_INT), num_int(num) { + } + inline explicit JsonValue(unsigned short num) : ty(L_JSON_INT), num_int(num) { + } + inline explicit JsonValue(int num) : ty(L_JSON_INT), num_int(num) { + } + inline explicit JsonValue(unsigned int num) : ty(L_JSON_INT), num_int(num) { + } + inline explicit JsonValue(long num) : ty(L_JSON_INT), num_int(num) { + } + inline explicit JsonValue(unsigned long num) : ty(L_JSON_INT), num_int(num) { + } + inline explicit JsonValue(long long num) : ty(L_JSON_INT), num_int(num) { + } + inline explicit JsonValue(unsigned long long num) + : ty(L_JSON_INT), num_int(num) { + } + inline explicit JsonValue(const char *str) : ty(L_JSON_STRING), str(str) { + } + inline explicit JsonValue(const std::string &str) + : ty(L_JSON_STRING), str(str) { + } + inline explicit JsonValue(std::string &&str) + : ty(L_JSON_STRING), str(std::forward(str)) { + } + inline explicit JsonValue(JsonObject &&obj) + : ty(L_JSON_OBJECT), obj(std::move(obj.inner)) { + } + inline explicit JsonValue(JsonArray &&arr) + : ty(L_JSON_ARRAY), arr(move(arr.inner)) { + } + + inline JsonValue &operator[](const char *key) { + if (!is_obj()) { + throw JsonException("value is not an object"); + } + return obj.inner.at(key); + } + inline const JsonValue &operator[](const char *key) const { + if (!is_obj()) { + throw JsonException("value is not an object"); + } + return obj.inner.at(key); + } + inline JsonValue &operator[](const std::string &key) { + if (!is_obj()) { + throw JsonException("value is not an object"); + } + return obj.inner.at(key); + } + inline const JsonValue &operator[](const std::string &key) const { + if (!is_obj()) { + throw JsonException("value is not an object"); + } + return obj.inner.at(key); + } + inline JsonValue &operator[](size_t i) { + if (!is_arr()) { + throw JsonException("value is not an array"); + } + return arr.inner.at(i); + } + inline const JsonValue &operator[](size_t i) const { + if (!is_arr()) { + throw JsonException("value is not an array"); + } + return arr.inner.at(i); + } + inline explicit operator bool() const { + if (!is_bool()) { + throw JsonException("value is not a bool"); + } + return b; + } + inline explicit operator double() const { + if (!is_num()) { + throw JsonException("value is not a number"); + } + return num_float; + } + inline explicit operator float() const { + if (!is_num()) { + throw JsonException("value is not a number"); + } + return (float)num_float; + } + inline explicit operator char() const { + if (!is_num()) { + throw JsonException("value is not a number"); + } + return (char)num_int; + } + inline explicit operator signed char() const { + if (!is_num()) { + throw JsonException("value is not a number"); + } + return (signed char)num_int; + } + inline explicit operator unsigned char() const { + if (!is_num()) { + throw JsonException("value is not a number"); + } + return (unsigned char)num_int; + } + inline explicit operator short() const { + if (!is_num()) { + throw JsonException("value is not a number"); + } + return (short)num_int; + } + inline explicit operator unsigned short() const { + if (!is_num()) { + throw JsonException("value is not a number"); + } + return (unsigned short)num_int; + } + inline explicit operator int() const { + if (!is_num()) { + throw JsonException("value is not a number"); + } + return (int)num_int; + } + inline explicit operator unsigned int() const { + if (!is_num()) { + throw JsonException("value is not a number"); + } + return (unsigned int)num_int; + } + inline explicit operator long() const { + if (!is_num()) { + throw JsonException("value is not a number"); + } + return (long)num_int; + } + inline explicit operator unsigned long() const { + if (!is_num()) { + throw JsonException("value is not a number"); + } + return (unsigned long)num_int; + } + inline explicit operator long long() const { + if (!is_num()) { + throw JsonException("value is not a number"); + } + return (long long)num_int; + } + inline explicit operator unsigned long long() const { + if (!is_num()) { + throw JsonException("value is not a number"); + } + return (unsigned long long)num_int; + } + inline explicit operator const std::string &() const { + if (!is_str()) { + throw JsonException("value is not a string"); + } + return str; + } + inline explicit operator const JsonArray &() const { + if (!is_arr()) { + throw JsonException("value is not an array"); + } + return arr; + } + inline explicit operator const JsonObject &() const { + if (!is_obj()) { + throw JsonException("value is not an object"); + } + return obj; + } + + inline bool is_null() const { + return ty == L_JSON_NULL; + } + inline bool is_bool() const { + return ty == L_JSON_BOOLEAN; + } + inline bool is_num() const { + return ty == L_JSON_FLOAT || ty == L_JSON_INT; + } + inline bool is_str() const { + return ty == L_JSON_STRING; + } + inline bool is_obj() const { + return ty == L_JSON_OBJECT; + } + inline bool is_arr() const { + return ty == L_JSON_ARRAY; + } + + inline size_t size() const { + if (is_obj()) { + return obj.inner.size(); + } else if (is_arr()) { + return arr.inner.size(); + } else { + throw JsonException("only object and array can have size"); + } + } + inline JsonElementEnumerator elems() const { + return JsonElementEnumerator(arr.inner); + } + inline JsonFieldEnumerator fields() const { + return JsonFieldEnumerator(obj.inner); + } +}; + +// Parse JSON literal into and `JsonValue` object. If the JSON is invalid or +// unsupported, `JsonException` will be raised. +JsonValue parse(const std::string &json_lit); +// Returns true when JSON parsing successfully finished and parsed value is +// returned via `out`. Otherwise, false is returned and out contains incomplete +// result. +bool try_parse(const std::string &json_lit, JsonValue &out); + +std::string print(const JsonValue &json); + +} // namespace json +} // namespace liong diff --git a/taichi/common/json_serde.h b/taichi/common/json_serde.h new file mode 100644 index 0000000000000..b254c417234ac --- /dev/null +++ b/taichi/common/json_serde.h @@ -0,0 +1,432 @@ +// Adapted from https://github.com/PENGUINLIONG/graphi-t + +// Copyright (c) 2019 Rendong Liang +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +// JSON generated ser/de. +// @PENGUINLIONG +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include "taichi/common/json.h" + +namespace liong { +namespace json { + +namespace detail { + +struct FieldNameList { + const std::vector field_names; + + static std::vector split_field_names(const char *field_names) { + std::vector out{}; + std::string buf{}; + + const char *pos = field_names; + for (char c = *pos; c != '\0'; c = *(++pos)) { + bool is_lower = (c >= 'a' && c <= 'z'); + bool is_upper = (c >= 'A' && c <= 'Z'); + bool is_digit = (c >= '0' && c <= '9'); + bool is_underscore = c == '_'; + + if (is_lower || is_upper || is_digit || is_underscore) { + buf.push_back(c); + } else { + if (!buf.empty()) { + out.emplace_back(std::exchange(buf, std::string())); + } + } + } + if (!buf.empty()) { + out.emplace_back(std::move(buf)); + } + return out; + } + + explicit FieldNameList(const char *field_names) + : field_names(split_field_names(field_names)) { + } +}; + +template +struct JsonSerde { + // Numeric and boolean types (integers and floating-point numbers). + template ::type> + static JsonValue serialize( + typename std::enable_if_t::value, T> x) { + return JsonValue(x); + } + template ::type> + static void deserialize( + const JsonValue &j, + typename std::enable_if_t::value, T> &x) { + x = (T)j; + } + template ::type> + static JsonValue serialize( + typename std::enable_if_t::value, T> x) { + return JsonValue((typename std::underlying_type::type)x); + } + template ::type> + static void deserialize( + const JsonValue &j, + typename std::enable_if_t::value, T> &x) { + x = (T)(typename std::underlying_type::type)j; + } + + // String type. + template ::type> + static JsonValue serialize( + typename std::enable_if_t::value, T> x) { + return JsonValue(x); + } + template ::type> + static void deserialize( + const JsonValue &j, + typename std::enable_if_t::value, T> &x) { + x = (T)j; + } + + // Structure types (with a `FieldNameList` field provided). + template ::type> + static JsonValue serialize( + const typename std::enable_if_t< + std::is_same().json_serialize_fields()), + JsonValue>::value, + T> &x) { + return JsonValue(x.json_serialize_fields()); + } + template ::type> + static void deserialize( + const JsonValue &j, + typename std::enable_if_t< + std::is_same().json_deserialize_fields( + std::declval())), + void>::value, + T> &x) { + x.json_deserialize_fields((const JsonObject &)j); + } + + // Key-value pairs. + template ::type> + static JsonValue serialize(const typename std::enable_if_t< + std::is_same, + T>::value, + T> &x) { + JsonObject obj{}; + obj.inner.emplace(std::make_pair( + "key", JsonSerde::serialize(x.first))); + obj.inner.emplace(std::make_pair( + "value", JsonSerde::serialize(x.second))); + return JsonValue(std::move(obj)); + } + template ::type> + static void deserialize( + const JsonValue &j, + typename std::enable_if_t, + T>::value, + T> &x) { + JsonSerde::deserialize(j["key"], x.first); + JsonSerde::deserialize(j["value"], x.second); + } + + // Owned pointer (requires default constructable). + template ::type> + static JsonValue serialize( + const typename std::enable_if_t< + std::is_same, T>::value, + T> &x) { + if (x == nullptr) { + return JsonValue(nullptr); + } else { + return JsonSerde::serialize(*x); + } + } + template ::type> + static void deserialize( + const JsonValue &j, + typename std::enable_if_t< + std::is_same, T>::value, + T> &x) { + if (j.is_null()) { + x = nullptr; + } else { + x = std::make_unique(); + JsonSerde::deserialize(j, *x); + } + } + + // Array types (requires default + move constructable). + template ::type> + static JsonValue serialize( + const typename std::enable_if_t::value, T> &x) { + JsonArray arr{}; + for (const auto &xx : x) { + arr.inner.emplace_back( + JsonSerde>::serialize(xx)); + } + return JsonValue(std::move(arr)); + } + template ::type> + static JsonValue serialize(const typename std::enable_if_t< + std::is_same::value>, + T>::value, + T> &x) { + JsonArray arr{}; + for (const auto &xx : x) { + arr.inner.emplace_back(JsonSerde::serialize(xx)); + } + return JsonValue(std::move(arr)); + } + template ::type> + static JsonValue serialize( + const typename std::enable_if_t< + std::is_same, T>::value, + T> &x) { + JsonArray arr{}; + for (const auto &xx : x) { + arr.inner.emplace_back(JsonSerde::serialize(xx)); + } + return JsonValue(std::move(arr)); + } + template ::type> + static void deserialize( + const JsonValue &j, + typename std::enable_if_t::value, T> &x) { + for (size_t i = 0; i < std::extent::value; ++i) { + JsonSerde>::deserialize(j[i], x[i]); + } + } + template ::type> + static void deserialize( + const JsonValue &j, + typename std::enable_if_t< + std::is_same< + std::array::value>, + T>::value, + T> &x) { + for (size_t i = 0; i < x.size(); ++i) { + JsonSerde::deserialize(j[i], x.at(i)); + } + } + template ::type> + static void deserialize( + const JsonValue &j, + typename std::enable_if_t< + std::is_same, T>::value, + T> &x) { + x.clear(); + for (const auto &elem : j.elems()) { + typename T::value_type xx{}; + JsonSerde::deserialize(elem, xx); + x.emplace_back(std::move(xx)); + } + } + + // Dictionary types (requires default + move constructable). + template ::type> + static JsonValue serialize( + const typename std::enable_if_t< + std::is_same, + T>::value, + T> &x) { + JsonArray arr{}; + for (const auto &xx : x) { + arr.inner.emplace_back(JsonSerde::serialize(xx)); + } + return JsonValue(std::move(arr)); + } + template ::type> + static JsonValue serialize( + const typename std::enable_if_t< + std::is_same< + std::unordered_map, + T>::value, + T> &x) { + JsonArray arr{}; + for (const auto &xx : x) { + arr.inner.emplace_back(JsonSerde::serialize(xx)); + } + return JsonValue(std::move(arr)); + } + template ::type> + static void deserialize( + const JsonValue &j, + typename std::enable_if_t< + std::is_same, + T>::value, + T> &x) { + x.clear(); + for (const auto &elem : j.elems()) { + std::pair xx{}; + JsonSerde::deserialize(elem, xx); + x.emplace(std::move(*(std::pair *)&xx)); + } + } + template ::type> + static void deserialize( + const JsonValue &j, + typename std::enable_if_t< + std::is_same< + std::unordered_map, + T>::value, + T> &x) { + x.clear(); + for (const auto &elem : j.elems()) { + std::pair xx{}; + JsonSerde::deserialize(elem, xx); + x.emplace(std::move(*(std::pair *)&xx)); + } + } + + // Optional types (requires default + move constructable). + template ::type> + static JsonValue serialize( + const typename std::enable_if_t< + std::is_same, T>::value, + T> &x) { + if (x.has_value()) { + return JsonSerde::serialize(x.value()); + } else { + return JsonValue(nullptr); + } + } + template ::type> + static void deserialize( + const JsonValue &j, + typename std::enable_if_t< + std::is_same, T>::value, + T> &x) { + if (j.is_null()) { + x = std::nullopt; + } else { + typename T::value_type xx; + JsonSerde::deserialize(j, xx); + x = std::move(xx); + } + } +}; + +template +struct JsonSerdeFieldImpl {}; +template +struct JsonSerdeFieldImpl { + inline static void serialize(JsonObject &obj, + std::vector::const_iterator name, + const TFirst &first, + const TOthers &...others) { + obj.inner.emplace(std::make_pair( + std::string(*name), JsonSerde::serialize(first))); + JsonSerdeFieldImpl::serialize(obj, ++name, others...); + } + inline static void deserialize(const JsonObject &obj, + std::vector::const_iterator name, + TFirst &first, + TOthers &...others) { + JsonSerde::deserialize(obj.inner.at(*name), first); + JsonSerdeFieldImpl::deserialize(obj, ++name, others...); + } +}; +template <> +struct JsonSerdeFieldImpl<> { + inline static void serialize(JsonObject &obj, + std::vector::const_iterator name) { + } + inline static void deserialize( + const JsonObject &obj, + std::vector::const_iterator name) { + } +}; +template +inline void json_serialize_field_impl( + JsonObject &obj, + std::vector::const_iterator name, + const TArgs &...args) { + JsonSerdeFieldImpl::serialize(obj, name, args...); +} +template +inline void json_deserialize_field_impl( + const JsonObject &obj, + std::vector::const_iterator name, + TArgs &...args) { + JsonSerdeFieldImpl::deserialize(obj, name, args...); +} + +} // namespace detail + +// Serialize a JSON serde object, turning in-memory representations into JSON +// text. +template +JsonValue serialize(const T &x) { + return detail::JsonSerde::serialize(x); +} + +// Deserialize a JSON serde object, turning JSON text into in-memory +// representations. +template +void deserialize(const JsonValue &j, T &out) { + detail::JsonSerde::deserialize(j, out); +} + +// If you need to control the serialization process on your own, you might want +// to inherit from this. +struct CustomJsonSerdeBase { + public: + // Serialize the field values into a JSON object. + virtual JsonObject json_serialize_fields() const = 0; + // Deserialize the current object with JSON fields. + virtual void json_deserialize_fields(const JsonObject &j) = 0; +}; + +} // namespace json +} // namespace liong + +#define L_JSON_SERDE_FIELDS(...) \ + const std::vector &json_serde_field_names() const { \ + static ::liong::json::detail::FieldNameList JSON_SERDE_FIELD_NAMES{ \ + #__VA_ARGS__}; \ + return JSON_SERDE_FIELD_NAMES.field_names; \ + } \ + ::liong::json::JsonValue json_serialize_fields() const { \ + ::liong::json::JsonObject out{}; \ + ::liong::json::detail::json_serialize_field_impl( \ + out, json_serde_field_names().begin(), __VA_ARGS__); \ + return ::liong::json::JsonValue(std::move(out)); \ + } \ + void json_deserialize_fields(const ::liong::json::JsonObject &j) { \ + ::liong::json::detail::json_deserialize_field_impl( \ + j, json_serde_field_names().begin(), __VA_ARGS__); \ + } diff --git a/taichi/common/serialization.h b/taichi/common/serialization.h index 39145ae7f4e3e..1a106dc520e8b 100644 --- a/taichi/common/serialization.h +++ b/taichi/common/serialization.h @@ -18,6 +18,8 @@ #include #include #include +#include "taichi/common/json.h" +#include "taichi/common/json_serde.h" #ifdef TI_INCLUDED namespace taichi { @@ -137,17 +139,11 @@ serialize_kv_impl(SER &ser, template \ void io(S &serializer) const -#define TI_IO_DEF(...) \ - template \ - void io(S &serializer) const { \ - TI_IO(__VA_ARGS__); \ - } - -#define TI_IO_DEF_WITH_BASECLASS(BaseClass, ...) \ - template \ - void io(S &serializer) const { \ - this->BaseClass::io(serializer); \ - TI_IO(__VA_ARGS__); \ +#define TI_IO_DEF(...) \ + L_JSON_SERDE_FIELDS(__VA_ARGS__) \ + template \ + void io(S &serializer) const { \ + TI_IO(__VA_ARGS__); \ } // This macro serializes each field with its name by doing the following: diff --git a/taichi/ir/ir.h b/taichi/ir/ir.h index 477cc6fbd3b13..95481869700b3 100644 --- a/taichi/ir/ir.h +++ b/taichi/ir/ir.h @@ -354,7 +354,11 @@ class StmtFieldManager { bool equal(StmtFieldManager &other) const; }; -#define TI_STMT_DEF_FIELDS(...) TI_IO_DEF(__VA_ARGS__) +#define TI_STMT_DEF_FIELDS(...) \ + template \ + void io(S &serializer) const { \ + TI_IO(__VA_ARGS__); \ + } #define TI_STMT_REG_FIELDS \ mark_fields_registered(); \ io(field_manager) diff --git a/taichi/math/linalg.h b/taichi/math/linalg.h index b6d2708acfe8a..4d31bb5e7fb10 100644 --- a/taichi/math/linalg.h +++ b/taichi/math/linalg.h @@ -510,19 +510,7 @@ struct VectorND : public VectorNDBase { return ret; } - TI_IO_DECL { - if (TI_SERIALIZER_IS(TextSerializer)) { - std::string ret = "("; - for (int i = 0; i < dim - 1; i++) { - ret += fmt::format("{}, ", d[i]); - } - ret += fmt::format("{}", d[dim - 1]); - ret += ")"; - serializer("vec", ret); - } else { - TI_IO(d); - } - } + TI_IO_DEF(d); TI_FORCE_INLINE explicit operator std::array() const { std::array arr; @@ -875,20 +863,7 @@ struct MatrixND { return MatrixND(1.0_f); } - TI_IO_DECL { - if constexpr (TI_SERIALIZER_IS(TextSerializer)) { - for (int i = 0; i < dim; i++) { - std::string line = "["; - for (int j = 0; j < dim; j++) { - line += fmt::format("{} ", d[j][i]); - } - line += "]"; - serializer.add_line(line); - } - } else { - TI_IO(d); - } - } + TI_IO_DEF(d); }; template @@ -1413,10 +1388,6 @@ static_assert(Serializer::has_io::value, ""); static_assert(Serializer::has_io::value, ""); static_assert(Serializer::has_io::value, ""); static_assert(Serializer::has_io::value, ""); -static_assert( - TextSerializer::has_io< - const taichi::MatrixND<4, double, (taichi::InstSetExt)3>>::value, - ""); namespace type { template diff --git a/taichi/util/offline_cache.h b/taichi/util/offline_cache.h index 7576a46c1ae55..0eed139482385 100644 --- a/taichi/util/offline_cache.h +++ b/taichi/util/offline_cache.h @@ -47,15 +47,6 @@ inline CleanCachePolicy string_to_clean_cache_policy(const std::string &str) { return Never; } -struct KernelMetadataBase { - std::string kernel_key; - std::size_t size{0}; // byte - std::time_t created_at{0}; // sec - std::time_t last_used_at{0}; // sec - - TI_IO_DEF(kernel_key, size, created_at, last_used_at); -}; - template struct Metadata { using KernelMetadata = KernelMetadataType; diff --git a/tests/cpp/offline_cache/load_metadata_test.cpp b/tests/cpp/offline_cache/load_metadata_test.cpp index d17cde132e983..8362bf8d18b55 100644 --- a/tests/cpp/offline_cache/load_metadata_test.cpp +++ b/tests/cpp/offline_cache/load_metadata_test.cpp @@ -100,11 +100,23 @@ void load_metadata_test() { } // namespace +// FIXME: (penguinliong) This structure has a same prototype as the actual types +// including `OfflineCacheKernelMetadata`s. It's currently used only for the +// tests and should probably be removed in the future. +struct KernelMetadataBase { + std::string kernel_key; + std::size_t size{0}; // byte + std::time_t created_at{0}; // sec + std::time_t last_used_at{0}; // sec + + TI_IO_DEF(kernel_key, size, created_at, last_used_at); +}; + TEST(OfflineCache, LoadMetadata) { #ifdef TI_WITH_LLVM load_metadata_test(); #endif // TI_WITH_LLVM - load_metadata_test>(); + load_metadata_test>(); } } // namespace taichi::lang