From 007e7a33db924b2180f7f1a8126c7f6a58100636 Mon Sep 17 00:00:00 2001 From: Wenchao Hu Date: Wed, 4 Dec 2019 16:35:15 +0800 Subject: [PATCH] feat: metrics logger (#43) * add logger config * add logger.metrics example * Revert "add logger.metrics example" This reverts commit 7c04ff3e906ab5e155c8258b63195740f9d1f0f5. * add logger module instruction * chore: add newline to the end of text file * refactor: change serde to json crate to manipulate json * chore: remove patch version in Cargo.toml depencencies --- .gitignore | 1 + Cargo.lock | 152 ++++++++++++++++++++++++++++++++++--- common/logger/Cargo.toml | 3 +- common/logger/README.md | 67 ++++++++++++++++ common/logger/log.yml | 36 +++++++++ common/logger/src/lib.rs | 141 +++++++++++++++++++--------------- devtools/chain/config.toml | 8 ++ src/config.rs | 33 +++++++- src/main.rs | 10 ++- 9 files changed, 371 insertions(+), 80 deletions(-) create mode 100644 common/logger/README.md create mode 100644 common/logger/log.yml diff --git a/.gitignore b/.gitignore index 578d4054a..e7edaefc8 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ tests/e2e/node_modules/ # rocksdb **/rocksdb/ +logs/ diff --git a/Cargo.lock b/Cargo.lock index 1bdbb3959..15b0a1330 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,11 +52,21 @@ dependencies = [ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "antidote" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "anyhow" version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "arc-swap" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "arrayref" version = "0.3.5" @@ -379,8 +389,9 @@ dependencies = [ name = "common-logger" version = "0.1.0" dependencies = [ - "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "json 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log4rs 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -421,7 +432,7 @@ dependencies = [ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "publicsuffix 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", "try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -468,7 +479,7 @@ dependencies = [ "rlp 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "runtime 0.3.0-alpha.8 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1430,6 +1441,11 @@ dependencies = [ "wasm-bindgen 0.2.54 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "json" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "juliex" version = "0.3.0-alpha.8" @@ -1522,6 +1538,11 @@ dependencies = [ "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "linked-hash-map" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "lock_api" version = "0.2.0" @@ -1552,6 +1573,36 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log-mdc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "log4rs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "arc-swap 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", + "flate2 1.0.12 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "log-mdc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", + "serde-value 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_yaml 0.8.11 (registry+https://github.com/rust-lang/crates.io-index)", + "thread-id 3.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1714,7 +1765,7 @@ dependencies = [ "runtime-tokio 0.3.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1879,6 +1930,14 @@ dependencies = [ "secp256k1 0.15.5 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "ordered-float" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "overlord" version = "0.1.0" @@ -1900,7 +1959,7 @@ dependencies = [ "runtime 0.3.0-alpha.8 (registry+https://github.com/rust-lang/crates.io-index)", "runtime-tokio 0.3.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2200,7 +2259,7 @@ dependencies = [ "rlp 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2529,7 +2588,7 @@ dependencies = [ "mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", "serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2758,6 +2817,15 @@ dependencies = [ "serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serde-value" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ordered-float 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "serde_derive" version = "1.0.102" @@ -2770,7 +2838,7 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.41" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2811,6 +2879,17 @@ dependencies = [ "url 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serde_yaml" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", + "yaml-rust 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "sha-1" version = "0.8.1" @@ -3104,6 +3183,16 @@ dependencies = [ "unicode-width 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "thread-id" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "thread_local" version = "0.3.6" @@ -3173,7 +3262,7 @@ dependencies = [ "http-service 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "route-recognizer 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3454,6 +3543,11 @@ dependencies = [ "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "traitobject" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "try-lock" version = "0.2.2" @@ -3485,6 +3579,14 @@ dependencies = [ "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "typemap" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "typenum" version = "1.11.2" @@ -3553,6 +3655,14 @@ name = "unicode-xid" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unsafe-any" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "unsigned-varint" version = "0.2.3" @@ -3805,6 +3915,14 @@ dependencies = [ "xml-rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "yaml-rust" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [metadata] "checksum adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" "checksum aes-ctr 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2e5b0458ea3beae0d1d8c0f3946564f8e10f90646cf78c06b4351052058d1ee" @@ -3812,7 +3930,9 @@ dependencies = [ "checksum aesni 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2f70a6b5f971e473091ab7cfb5ffac6cde81666c4556751d8d5620ead8abf100" "checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5" "checksum anyhow 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)" = "57114fc2a6cc374bce195d3482057c846e706d252ff3604363449695684d7a0d" +"checksum arc-swap 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)" = "bc4662175ead9cd84451d5c35070517777949a2ed84551764129cedb88384841" "checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee" "checksum async-datagram 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ec4e1e7c4c16c964d7040b676d2eeb0ce4f94bed2e57e28f28a88768c8f934c" "checksum async-ready 3.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9470c79dd71fcd6090676d2ef90f45e9d357831234e8a1f70a1991093019bc27" @@ -3950,6 +4070,7 @@ dependencies = [ "checksum itertools 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "87fa75c9dea7b07be3138c49abbb83fd4bea199b5cdc76f9804458edc5da0d6e" "checksum itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501266b7edd0174f8530248f87f99c88fbe60ca4ef3dd486835b8d8d53136f7f" "checksum js-sys 0.3.31 (registry+https://github.com/rust-lang/crates.io-index)" = "d8657b7ca06a6044ece477f6900bf7670f8b5fd0cce177a1d7094eef51e0adf4" +"checksum json 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3ca41abbeb7615d56322a984e63be5e5d0a117dfaca86c14393e32a762ccac1" "checksum juliex 0.3.0-alpha.8 (registry+https://github.com/rust-lang/crates.io-index)" = "22dcfd5923ddd3e945884e66c89f92e7ce370e8fcabc593f6af1ce76e21099bc" "checksum juniper 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b570603cc5ea362590965be2d201f24c5e39f2675953d46813ea4c5f7a34e75c" "checksum juniper_codegen 0.13.2 (registry+https://github.com/rust-lang/crates.io-index)" = "230e99ba589c473bb53d33bf4bd9aedcb7069c50b336bb76dcff9f69a1b19418" @@ -3959,10 +4080,13 @@ dependencies = [ "checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8" "checksum libloading 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" "checksum librocksdb-sys 6.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b93b02499a16b46dfa35536a54b54ca14cc27275dd766aba4f4e267c97572994" +"checksum linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" "checksum lock_api 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ed946d4529956a20f2d63ebe1b69996d5a2137c91913fe3ebbeff957f5bca7ff" "checksum lock_api 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f8912e782533a93a167888781b836336a6ca5da6175c05944c86cf28c31104dc" "checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +"checksum log-mdc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a94d21414c1f4a51209ad204c1776a3d0765002c76c6abcb602a6f09f1e881c7" +"checksum log4rs 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "100052474df98158c0738a7d3f4249c99978490178b5f9f68cd835ac57adbd1b" "checksum mach_o_sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3e854583a83f20cf329bb9283366335387f7db59d640d1412167e05fedb98826" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" @@ -3994,6 +4118,7 @@ dependencies = [ "checksum ophelia-derive 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1b40038ab43ba5ae7399dae03ef13765d36e036fc4f9a400e91381d3f17e5196" "checksum ophelia-hasher 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b08ec8574c7c180763d5d3c26c31525db5981ceb2d4b62c47c4bc32555c776b9" "checksum ophelia-secp256k1 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d3a734015b125cef5a331105bd428be591d7874a362ed5d2c2252c046677386" +"checksum ordered-float 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "18869315e81473c951eb56ad5558bbc56978562d3ecfb87abb7a1e944cea4518" "checksum overlord 0.1.0 (git+https://github.com/nervosnetwork/overlord?branch=test)" = "" "checksum owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cdf84f41639e037b484f93433aa3897863b561ed65c6e59c7073d7c561710f37" "checksum pairing 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ceda21136251c6d5a422d3d798d8ac22515a6e8d3521bb60c59a8349d36d0d57" @@ -4083,11 +4208,13 @@ dependencies = [ "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" "checksum serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4b39bd9b0b087684013a792c59e3e07a46a01d2322518d8a1104641a0b1be0" +"checksum serde-value 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7a663f873dedc4eac1a559d4c6bc0d0b2c34dc5ac4702e105014b8281489e44f" "checksum serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)" = "ca13fc1a832f793322228923fbb3aba9f3f44444898f835d31ad1b74fa0a2bf8" -"checksum serde_json 1.0.41 (registry+https://github.com/rust-lang/crates.io-index)" = "2f72eb2a68a7dc3f9a691bfda9305a1c017a6215e5a4545c258500d2099a37c2" +"checksum serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)" = "1a3351dcbc1f067e2c92ab7c3c1f288ad1a4cffc470b5aaddb4c2e0a3ae80043" "checksum serde_qs 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b232c387059b62b17eb0487bf23de3ddf21b648ad2206fadc6ff3af9e2f3c07" "checksum serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "642dd69105886af2efd227f75a520ec9b44a820d65bc133a9131f7d229fd165a" "checksum serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +"checksum serde_yaml 0.8.11 (registry+https://github.com/rust-lang/crates.io-index)" = "691b17f19fc1ec9d94ec0b5864859290dff279dbd7b03f017afda54eb36c3c35" "checksum sha-1 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "23962131a91661d643c98940b20fcaffe62d776a823247be80a48fcb8b6fce68" "checksum sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b4d8bfd0e469f417657573d8451fb33d16cfe0989359b93baf3a1ffc639543d" "checksum sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd26bc0e7a2e3a7c959bc494caf58b72ee0c71d67704e9520f736ca7e4853ecf" @@ -4119,6 +4246,7 @@ dependencies = [ "checksum tentacle-secio 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f639e9adcc3f3d89ec67b65799651bb0e34db476ebd0562caabd960cfcd920b7" "checksum termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "96d6098003bde162e4277c70665bd87c326f5a0c3f3fbfb285787fa482d54e6e" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +"checksum thread-id 3.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7fbf4c9d56b320106cd64fd024dadfa0be7cb4706725fc44a7d7ce952d820c1" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" "checksum threshold_crypto 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "95be1032c63011f20b01c5edb64930e2b51512782b43b458b1e3449613d70f87" "checksum tide 0.2.0 (git+https://github.com/rustasync/tide?rev=af0b900)" = "" @@ -4147,10 +4275,12 @@ dependencies = [ "checksum tokio-uds 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "037ffc3ba0e12a0ab4aca92e5234e0dedeb48fddf6ccd260f1f150a36a9f2445" "checksum tokio-yamux 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "723c77f5e71061a1363d8402418ac87be5570fe548a638a4eb231fa01d6124b6" "checksum toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" +"checksum traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" "checksum try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "283d3b89e1368717881a9d51dad843cc435380d8109c9e47d38780a324698d8b" "checksum twofish 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712d261e83e727c8e2dbb75dacac67c36e35db36a958ee504f2164fc052434e1" "checksum twoway 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" +"checksum typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "653be63c80a3296da5551e1bfd2cca35227e13cdd08c6668903ae2f4f77aa1f6" "checksum typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" "checksum uint 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "754ba11732b9161b94c41798e5197e5e75388d012f760c42adb5000353e98646" "checksum unicase 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" @@ -4161,6 +4291,7 @@ dependencies = [ "checksum unicode-width 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7007dbd421b92cc6e28410fe7362e2e0a2503394908f417b68ec8d1c364c4e20" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +"checksum unsafe-any 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f30360d7979f5e9c6e6cea48af192ea8fab4afb3cf72597154b8f08935bc9c7f" "checksum unsigned-varint 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f0023a96687fe169081e8adce3f65e3874426b7886e9234d490af2dc077959" "checksum untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60369ef7a31de49bcb3f6ca728d4ba7300d9a1658f94c727d4cab8c8d9f4aece" "checksum url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" @@ -4192,3 +4323,4 @@ dependencies = [ "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" "checksum xml-rs 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3c1cb601d29fe2c2ac60a2b2e5e293994d87a1f6fa9687a31a15270f909be9c2" "checksum xmltree 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff8eaee9d17062850f1e6163b509947969242990ee59a35801af437abe041e70" +"checksum yaml-rust 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d" diff --git a/common/logger/Cargo.toml b/common/logger/Cargo.toml index 2c62f13dc..e1e9f46d7 100644 --- a/common/logger/Cargo.toml +++ b/common/logger/Cargo.toml @@ -7,5 +7,6 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -env_logger = "0.6" log = "0.4" +log4rs = "0.8" +json = "0.12" diff --git a/common/logger/README.md b/common/logger/README.md new file mode 100644 index 000000000..a6a0550b6 --- /dev/null +++ b/common/logger/README.md @@ -0,0 +1,67 @@ +# Logger Module Instruction + +## Logger Config + +The logger config in `config.toml` is listed below with default values. + +```toml +[logger] +filter = "info" +log_to_console = true +console_show_file_and_line = false +log_path = "logs/" +log_to_file = true +metrics = true +``` + +`filter` is the root logger filter, must be one of `off`, `trace`, `debug`, `info`, `warn` and `error`. + +If `log_to_console` is `true`, logs like below will be logged to console. + +``` +[2019-12-02T10:02:45.779337+08:00 INFO overlord::state::process] Overlord: state receive commit event epoch ID 11220, round 0 +``` + +If `console_show_file_and_line` is `true`, log file and line number will also be logged to console, pretty useful for debugging. + +``` +[2019-12-02T10:05:28.343228+08:00 INFO core_network::peer_manager core/network/src/peer_manager/mod.rs:1035] network: PeerId(QmYSZUy3G5Mf5GSTKfH7LXJeFJrVW59rX1qPPfapuH7AUw): connected peer_ip(s): [] +``` + +If `log_to_file` is true, logs like below will be logged to `{log_path}/muta.log`. +It is json format, good for machine understanding. + +``` +{"time":"2019-12-01T22:01:57.839042+08:00","message":"network: PeerId(QmYSZUy3G5Mf5GSTKfH7LXJeFJrVW59rX1qPPfapuH7AUw): connect addrs [\"/ip4/0.0.0.0/tcp/1888\"]","module_path":"core_network::peer_manager","file":"core/network/src/peer_manager/mod.rs","line":591,"level":"INFO","target":"core_network::peer_manager","thread":"tokio-runtime-worker-0","thread_id":123145432756224,"mdc":{}} +``` + +This crate uses `log4rs` to init the logger, but you don't need to add dependency for that. After invoking the `init` function in this crate, you can use `log` crate to log. + +## Metrics + +Metrics is an independent logger, it `metrics` is `true`, the metrics will be logged to `{log_path}/metrics.log`. + +``` +{"time":"2019-12-01T22:02:49.035084+08:00","message":"{\"epoch_id\":7943,\"name\":\"save_epoch\",\"ordered_tx_num\":0}","module_path":"common_logger","file":"common/logger/src/lib.rs","line":83,"level":"TRACE","target":"metrics","thread":"tokio-runtime-worker-3","thread_id":123145445486592,"mdc":{}} +``` + +If you want to use log metrics in a module, you need to add this crate as dependency and use the code below to add a metric. The `name` field is reserved, please avoid using this as a key in your metrics. + +```rust +common_logger::metrics("save_epoch", common_logger::object! { + "epoch_id" => epoch.header.epoch_id, + "ordered_tx_num" => epoch.ordered_tx_hashes.len(), +}); +``` + +This signature of the function is showed below. The `JsonValue` is a `enum` from [`json crate`](https://docs.rs/json/0.12.0/json/enum.JsonValue.html). + +```rust +pub fn metrics(name: &str, mut content: JsonValue) +``` + +## Yaml File + +The `log.yml` in this crate is the yaml style config of log4rs with default logger config. + +If you need more customized configurations, you can copy the file to some config path, edit the file, and replace the `init` function with `log4rs::init_file("/path/to/log.yml", Default::default()).unwrap();`. \ No newline at end of file diff --git a/common/logger/log.yml b/common/logger/log.yml new file mode 100644 index 000000000..27b36e252 --- /dev/null +++ b/common/logger/log.yml @@ -0,0 +1,36 @@ +# This file is yaml style config, can make testing the logger more easily. +# When you need to do some test, Add the code below to the `init` function. +# log4rs::init_file("common/logger/log.yml", Default::default()).unwrap(); +# reference: +appenders: + console: + kind: console + encoder: + # this pattern below contains file name and line, usefule for debugging + # pattern: "[{d} {h({l})} {t} {f}:{L}] {m}{n}" + pattern: "[{d} {h({l})} {t}] {m}{n}" + + file: + kind: file + path: logs/muta.log + encoder: + kind: json + + metrics: + kind: file + path: logs/metrics.log + encoder: + kind: json + +root: + level: info + appenders: + - console + - file + +loggers: + metrics: + level: trace + appenders: + - metrics + additive: false diff --git a/common/logger/src/lib.rs b/common/logger/src/lib.rs index c4e5377cf..c7b6f4518 100644 --- a/common/logger/src/lib.rs +++ b/common/logger/src/lib.rs @@ -1,73 +1,88 @@ -use std::env; +use std::path::PathBuf; -pub enum Flag { - Main, // Used in main function - Test, // Used in test function -} +use json::JsonValue; +use log::LevelFilter; +use log4rs::append::console::ConsoleAppender; +use log4rs::append::file::FileAppender; +use log4rs::config::{Appender, Config, Logger, Root}; +use log4rs::encode::json::JsonEncoder; +use log4rs::encode::pattern::PatternEncoder; -/// Function init should used in main or test function. -pub fn init(flag: Flag) { - match flag { - Flag::Main => { - env::var("RUST_LOG").unwrap_or_else(|_| { - env::set_var("RUST_LOG", "info"); - "info".into() - }); - env_logger::init(); - } - Flag::Test => { - let _ = env_logger::builder().is_test(true).try_init(); - } - } -} +pub use json::array; +pub use json::object; -#[cfg(test)] -mod tests { - use super::{init, Flag}; - use log::{debug, error, info, trace, warn}; +pub fn init( + filter: String, + log_to_console: bool, + console_show_file_and_line: bool, + log_to_file: bool, + metrics: bool, + log_path: PathBuf, +) { + let console = ConsoleAppender::builder() + .encoder(Box::new(PatternEncoder::new( + if console_show_file_and_line { + "[{d} {h({l})} {t} {f}:{L}] {m}{n}" + } else { + "[{d} {h({l})} {t}] {m}{n}" + }, + ))) + .build(); - #[test] - fn test_debug() { - init(Flag::Test); - debug!("Hello World!"); - debug!("Hello {}", "World!"); - debug!("Hell{} {}", "o", "World!"); - debug!("Hell{} World", 0); - } + let file = FileAppender::builder() + .encoder(Box::new(JsonEncoder::new())) + .build(log_path.join("muta.log")) + .unwrap(); - #[test] - fn test_error() { - init(Flag::Test); - error!("Hello World!"); - error!("Hello {}", "World!"); - error!("Hell{} {}", "o", "World!"); - error!("Hell{} World", 0); - } + let metrics_appender = FileAppender::builder() + .encoder(Box::new(JsonEncoder::new())) + .build(log_path.join("metrics.log")) + .unwrap(); - #[test] - fn test_info() { - init(Flag::Test); - info!("Hello World!"); - info!("Hello {}", "World!"); - info!("Hell{} {}", "o", "World!"); - info!("Hell{} World", 0); + let mut root_builder = Root::builder(); + if log_to_console { + root_builder = root_builder.appender("console"); } - - #[test] - fn test_warn() { - init(Flag::Test); - warn!("Hello World!"); - warn!("Hello {}", "World!"); - warn!("Hell{} {}", "o", "World!"); - warn!("Hell{} World", 0); + if log_to_file { + root_builder = root_builder.appender("file"); } + let level_filter = match filter.as_ref() { + "off" => LevelFilter::Off, + "error" => LevelFilter::Error, + "info" => LevelFilter::Info, + "warn" => LevelFilter::Warn, + "debug" => LevelFilter::Debug, + "trace" => LevelFilter::Trace, + f => { + println!("invalid logger.filter {}, use info", f); + LevelFilter::Info + } + }; + let root = root_builder.build(level_filter); - #[test] - fn test_trace() { - init(Flag::Test); - trace!("Hello World!"); - trace!("Hello {}", "World!"); - trace!("Hell{} {}", "o", "World!"); - trace!("Hell{} World", 0); - } + let metrics_logger = Logger::builder().additive(false).appender("metrics").build( + "metrics", + if metrics { + LevelFilter::Trace + } else { + LevelFilter::Off + }, + ); + + let config = Config::builder() + .appender(Appender::builder().build("console", Box::new(console))) + .appender(Appender::builder().build("file", Box::new(file))) + .appender(Appender::builder().build("metrics", Box::new(metrics_appender))) + .logger(metrics_logger) + .build(root) + .unwrap(); + + log4rs::init_config(config).unwrap(); +} + +pub fn metrics(name: &str, mut content: JsonValue) { + log::trace!(target: "metrics", "{}", { + content["name"] = name.into(); + content + }); } diff --git a/devtools/chain/config.toml b/devtools/chain/config.toml index b7325fe84..8fcd32742 100644 --- a/devtools/chain/config.toml +++ b/devtools/chain/config.toml @@ -41,3 +41,11 @@ precommit_denominator = 30 [executor] light = false + +[logger] +filter = "info" +log_to_console = true +console_show_file_and_line = false +log_path = "logs/" +log_to_file = true +metrics = true diff --git a/src/config.rs b/src/config.rs index 10fa6e73b..daaceb038 100644 --- a/src/config.rs +++ b/src/config.rs @@ -58,6 +58,29 @@ pub struct ConfigExecutor { pub light: bool, } +#[derive(Debug, Deserialize)] +pub struct ConfigLogger { + pub filter: String, + pub log_to_console: bool, + pub console_show_file_and_line: bool, + pub log_to_file: bool, + pub metrics: bool, + pub log_path: PathBuf, +} + +impl Default for ConfigLogger { + fn default() -> Self { + Self { + filter: "info".into(), + log_to_console: true, + console_show_file_and_line: false, + log_to_file: true, + metrics: true, + log_path: "logs/".into(), + } + } +} + #[derive(Debug, Deserialize)] pub struct Config { // chain id @@ -67,11 +90,13 @@ pub struct Config { // db config pub data_path: PathBuf, - pub graphql: ConfigGraphQL, - pub network: ConfigNetwork, - pub mempool: ConfigMempool, + pub graphql: ConfigGraphQL, + pub network: ConfigNetwork, + pub mempool: ConfigMempool, pub consensus: ConfigConsensus, - pub executor: ConfigExecutor, + pub executor: ConfigExecutor, + #[serde(default)] + pub logger: ConfigLogger, } impl Config { diff --git a/src/main.rs b/src/main.rs index 14d32ec3d..cd6af610a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -43,8 +43,6 @@ use crate::config::Config; #[runtime::main(runtime_tokio::Tokio)] async fn main() { - common_logger::init(common_logger::Flag::Main); - let matches = clap::App::new("Muta") .version("v0.1.0") .author("Muta Dev ") @@ -59,6 +57,14 @@ async fn main() { .get_matches(); let args_config = matches.value_of("config").unwrap(); let cfg: Config = common_config_parser::parse(args_config).unwrap(); + common_logger::init( + cfg.logger.filter.clone(), + cfg.logger.log_to_console, + cfg.logger.console_show_file_and_line, + cfg.logger.log_to_file, + cfg.logger.metrics, + cfg.logger.log_path.clone(), + ); log::info!("Go with config: {:?}", cfg); // init genesis