From 70e8b5fe55dc9c4767f2fc992e83ac3bb932b740 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 26 Sep 2023 14:15:41 -0400 Subject: [PATCH] chore(deps): Update VRL to 0.7.0 (#18672) * chore(deps): Update VRL to 0.7.0 * remove some examples and add word to allow.txt * rename Lua code (due to breaking changes) * updated ordered-float version (otherwise two different versions will be used and we will get many compilation errors) * enable mlua macros feature and opt-in to FromLua * more lua changes * document truncate * Update website/cue/reference/remap/functions/truncate.cue * regen licenses * remove to_timestamp link * docs fmt * fix test runner VRL test paths * edit example using deprecated argument * Update tests to use parse_timestamp Signed-off-by: Jesse Szwedko * Bump rust-version Signed-off-by: Jesse Szwedko --------- Signed-off-by: Jesse Szwedko Co-authored-by: Jesse Szwedko --- .github/actions/spelling/allow.txt | 2 + Cargo.lock | 272 ++++++++++-------- Cargo.toml | 10 +- LICENSE-3rdparty.csv | 3 + lib/codecs/Cargo.toml | 4 +- .../vector_generate_fixtures.patch | 4 +- lib/opentelemetry-proto/Cargo.toml | 2 +- lib/vector-common/Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 6 +- lib/vector-core/src/event/lua/event.rs | 12 +- lib/vector-core/src/event/lua/log.rs | 8 +- lib/vector-core/src/event/lua/metric.rs | 32 +-- lib/vector-vrl/tests/src/main.rs | 11 +- lib/vrl/value/Cargo.toml | 4 +- src/internal_events/lua.rs | 1 - src/transforms/lua/v1/mod.rs | 5 +- tests/behavior/transforms/remap.toml | 8 +- .../advanced/parsing-csv-logs-with-lua.md | 1 - .../remap/functions/community_id.cue | 59 ++++ .../reference/remap/functions/encode_json.cue | 6 + .../functions/parse_aws_vpc_flow_log.cue | 40 ++- .../remap/functions/to_timestamp.cue | 90 ------ .../reference/remap/functions/truncate.cue | 22 +- website/cue/reference/urls.cue | 3 +- 24 files changed, 340 insertions(+), 267 deletions(-) create mode 100644 website/cue/reference/remap/functions/community_id.cue delete mode 100644 website/cue/reference/remap/functions/to_timestamp.cue diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 360f67b173aed..14b08efb7cc0e 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -408,7 +408,9 @@ rpms rstest rsyslog rsyslogd +security-tab servlet +sublocation snyk socketaddr solarwinds diff --git a/Cargo.lock b/Cargo.lock index 43ef5fcc9015d..e74b6b2c2107c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -307,12 +307,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" - [[package]] name = "arrayvec" version = "0.7.2" @@ -1551,9 +1545,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.2" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "bitmask-enum" @@ -2069,9 +2063,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.3" +version = "4.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" +checksum = "824956d0dca8334758a5b7f7e50518d66ea319330cbceedcf76905c2f6ab30e3" dependencies = [ "clap_builder", "clap_derive", @@ -2083,21 +2077,21 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eef05769009513df2eb1c3b4613e7fad873a14c600ff025b08f250f59fee7de" dependencies = [ - "clap 4.4.3", + "clap 4.4.5", "log", ] [[package]] name = "clap_builder" -version = "4.4.2" +version = "4.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "122ec64120a49b4563ccaedcbea7818d069ed8e9aa6d829b82d8a4128936b2ab" dependencies = [ "anstream", "anstyle 1.0.0", "clap_lex", "strsim 0.10.0", - "terminal_size 0.2.2", + "terminal_size 0.3.0", ] [[package]] @@ -2106,7 +2100,7 @@ version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8baeccdb91cd69189985f87f3c7e453a3a451ab5746cf3be6acc92120bd16d24" dependencies = [ - "clap 4.4.3", + "clap 4.4.5", ] [[package]] @@ -2161,7 +2155,7 @@ dependencies = [ "indoc", "memchr", "once_cell", - "ordered-float 3.9.1", + "ordered-float 4.1.0", "prost 0.12.1", "prost-reflect", "regex", @@ -2237,6 +2231,20 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "community-id" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a97e158411823bf87634e2e47552f712e51fa4119cdb2255da799e7cb5c90a9" +dependencies = [ + "anyhow", + "base64 0.21.4", + "hex", + "lazy_static", + "num_enum 0.6.1", + "sha1", +] + [[package]] name = "concurrent-queue" version = "1.2.4" @@ -2411,7 +2419,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.4.3", + "clap 4.4.5", "criterion-plot", "futures 0.3.28", "is-terminal", @@ -2499,7 +2507,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.4.0", "crossterm_winapi", "futures-core", "libc", @@ -2980,9 +2988,9 @@ dependencies = [ [[package]] name = "dns-lookup" -version = "2.0.2" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f332aa79f9e9de741ac013237294ef42ce2e9c6394dc7d766725812f1238812" +checksum = "8d0fa3cd8dc96ada974e126a940d37d4079bbbe6a24aca15b1113c2f362441c5" dependencies = [ "cfg-if", "libc", @@ -3218,17 +3226,6 @@ dependencies = [ "serde", ] -[[package]] -name = "errno" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" -dependencies = [ - "errno-dragonfly", - "libc", - "winapi", -] - [[package]] name = "errno" version = "0.3.1" @@ -4459,12 +4456,6 @@ version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1be380c410bf0595e94992a648ea89db4dd3f3354ba54af206fd2a68cf5ac8e" -[[package]] -name = "io-lifetimes" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ce5ef949d49ee85593fc4d3f3f95ad61657076395cbbce23e2121fc5542074" - [[package]] name = "io-lifetimes" version = "1.0.11" @@ -4519,7 +4510,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi 0.3.1", - "io-lifetimes 1.0.11", + "io-lifetimes", "rustix 0.37.19", "windows-sys 0.48.0", ] @@ -4929,15 +4920,15 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.0.46" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" [[package]] name = "listenfd" @@ -5016,11 +5007,12 @@ dependencies = [ [[package]] name = "luajit-src" -version = "210.4.3+resty8384278" +version = "210.4.8+resty107baaf" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ee5d5afddf1ec76ffa55ca7c3001f2f8a703834beba53c56a38ea6641cef44" +checksum = "e05167e8b2a2185758d83ed23541e5bd8bce37072e4204e0ef2c9b322bc87c4e" dependencies = [ "cc", + "which", ] [[package]] @@ -5278,18 +5270,44 @@ dependencies = [ [[package]] name = "mlua" -version = "0.8.10" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb37b0ba91f017aa7ca2b98ef99496827770cd635b4a932a6047c5b4bbe678e" +checksum = "6c3a7a7ff4481ec91b951a733390211a8ace1caba57266ccb5f4d4966704e560" +dependencies = [ + "bstr 1.6.2", + "mlua-sys", + "mlua_derive", + "num-traits", + "once_cell", + "rustc-hash", +] + +[[package]] +name = "mlua-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ec8b54eddb76093069cce9eeffb4c7b3a1a0fe66962d7bd44c4867928149ca3" dependencies = [ - "bstr 0.2.17", "cc", + "cfg-if", "lua-src", "luajit-src", - "num-traits", - "once_cell", "pkg-config", - "rustc-hash", +] + +[[package]] +name = "mlua_derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f359220f24e6452dd82a3f50d7242d4aab822b5594798048e953d7a9e0314c6" +dependencies = [ + "itertools 0.11.0", + "once_cell", + "proc-macro-error", + "proc-macro2 1.0.67", + "quote 1.0.33", + "regex", + "syn 2.0.37", ] [[package]] @@ -5524,7 +5542,7 @@ version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.4.0", "filetime", "fsevent-sys", "inotify", @@ -5589,7 +5607,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" dependencies = [ - "arrayvec 0.7.2", + "arrayvec", "itoa", "num-bigint", ] @@ -5644,6 +5662,15 @@ dependencies = [ "num_enum_derive 0.5.11", ] +[[package]] +name = "num_enum" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a015b430d3c108a207fd776d2e2196aaf8b1cf8cf93253e3a097ff3085076a1" +dependencies = [ + "num_enum_derive 0.6.1", +] + [[package]] name = "num_enum" version = "0.7.0" @@ -5665,6 +5692,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "num_enum_derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" +dependencies = [ + "proc-macro-crate 1.2.1", + "proc-macro2 1.0.67", + "quote 1.0.33", + "syn 2.0.37", +] + [[package]] name = "num_enum_derive" version = "0.7.0" @@ -5840,7 +5879,7 @@ version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.4.0", "cfg-if", "foreign-types", "libc", @@ -5895,7 +5934,7 @@ dependencies = [ "bytes 1.5.0", "chrono", "hex", - "ordered-float 3.9.1", + "ordered-float 4.1.0", "prost 0.12.1", "prost-build 0.12.1", "tonic 0.10.1", @@ -5929,6 +5968,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a540f3e3b3d7929c884e46d093d344e4e5bdeed54d08bf007df50c93cc85d5" +dependencies = [ + "num-traits", +] + [[package]] name = "os_info" version = "3.7.0" @@ -6073,19 +6121,20 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.7.2" +version = "2.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a" +checksum = "d7a4d085fd991ac8d5b05a147b437791b4260b76326baf0fc60cf7c9c27ecd33" dependencies = [ + "memchr", "thiserror", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.7.2" +version = "2.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853" +checksum = "a2bee7be22ce7918f641a33f08e3f43388c7656772244e2bbb2477f44cc9021a" dependencies = [ "pest", "pest_generator", @@ -6093,9 +6142,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.2" +version = "2.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929" +checksum = "d1511785c5e98d79a05e8a6bc34b4ac2168a0e3e92161862030ad84daa223141" dependencies = [ "pest", "pest_meta", @@ -6106,9 +6155,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.2" +version = "2.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48" +checksum = "b42f0394d3123e33353ca5e1e89092e533d2cc490389f2bd6131c43c634ebc5f" dependencies = [ "once_cell", "pest", @@ -6946,7 +6995,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e2e4cd95294a85c3b4446e63ef054eea43e0205b1fd60120c16b74ff7ff96ad" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.4.0", "cassowary", "crossterm", "indoc", @@ -7365,7 +7414,7 @@ version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26bd36b60561ee1fb5ec2817f198b6fd09fa571c897a5e86d1487cfc2b096dfc" dependencies = [ - "arrayvec 0.7.2", + "arrayvec", "borsh", "bytecheck", "byteorder", @@ -7419,29 +7468,28 @@ dependencies = [ [[package]] name = "rustix" -version = "0.35.13" +version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727a1a6d65f786ec22df8a81ca3121107f235970dc1705ed681d3e6e8b9cd5f9" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ "bitflags 1.3.2", - "errno 0.2.8", - "io-lifetimes 0.7.5", + "errno", + "io-lifetimes", "libc", - "linux-raw-sys 0.0.46", - "windows-sys 0.42.0", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", ] [[package]] name = "rustix" -version = "0.37.19" +version = "0.38.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" dependencies = [ - "bitflags 1.3.2", - "errno 0.3.1", - "io-lifetimes 1.0.11", + "bitflags 2.4.0", + "errno", "libc", - "linux-raw-sys 0.3.8", + "linux-raw-sys 0.4.7", "windows-sys 0.48.0", ] @@ -7524,7 +7572,7 @@ version = "12.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "994eca4bca05c87e86e15d90fc7a91d1be64b4482b38cb2d27474568fe7c9db9" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.4.0", "cfg-if", "clipboard-win", "libc", @@ -8236,22 +8284,13 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "strip-ansi-escapes" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "011cbb39cf7c1f62871aea3cc46e5817b0937b49e9447370c93cacbe93a766d8" -dependencies = [ - "vte 0.10.1", -] - [[package]] name = "strip-ansi-escapes" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55ff8ef943b384c414f54aefa961dd2bd853add74ec75e7ac74cf91dba62bcfa" dependencies = [ - "vte 0.11.1", + "vte", ] [[package]] @@ -8465,12 +8504,12 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ca90c434fd12083d1a6bdcbe9f92a14f96c8a1ba600ba451734ac334521f7a" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" dependencies = [ - "rustix 0.35.13", - "windows-sys 0.42.0", + "rustix 0.38.14", + "windows-sys 0.48.0", ] [[package]] @@ -8971,7 +9010,7 @@ checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ "async-compression", "base64 0.21.4", - "bitflags 2.3.2", + "bitflags 2.4.0", "bytes 1.5.0", "futures-core", "futures-util", @@ -9536,7 +9575,7 @@ dependencies = [ "atty", "cached", "chrono", - "clap 4.4.3", + "clap 4.4.5", "clap-verbosity-flag", "clap_complete", "confy", @@ -9613,7 +9652,7 @@ dependencies = [ "bytesize", "chrono", "cidr-utils", - "clap 4.4.3", + "clap 4.4.5", "codecs", "colored", "console-subscriber", @@ -9682,7 +9721,7 @@ dependencies = [ "openssl-probe", "openssl-src", "opentelemetry-proto", - "ordered-float 3.9.1", + "ordered-float 4.1.0", "paste", "percent-encoding", "pin-project", @@ -9723,7 +9762,7 @@ dependencies = [ "snap", "socket2 0.5.4", "stream-cancel", - "strip-ansi-escapes 0.2.0", + "strip-ansi-escapes", "syslog", "tempfile", "test-generator", @@ -9774,7 +9813,7 @@ dependencies = [ "anyhow", "async-trait", "chrono", - "clap 4.4.3", + "clap 4.4.5", "futures 0.3.28", "graphql_client", "indoc", @@ -9797,7 +9836,7 @@ dependencies = [ "async-trait", "bytecheck", "bytes 1.5.0", - "clap 4.4.3", + "clap 4.4.5", "crc32fast", "criterion", "crossbeam-queue", @@ -9846,7 +9885,7 @@ dependencies = [ "indexmap 2.0.0", "metrics", "nom", - "ordered-float 3.9.1", + "ordered-float 4.1.0", "paste", "pin-project", "quickcheck", @@ -9956,7 +9995,7 @@ dependencies = [ "noisy_float", "once_cell", "openssl", - "ordered-float 3.9.1", + "ordered-float 4.1.0", "parking_lot", "pin-project", "proptest", @@ -10017,7 +10056,7 @@ dependencies = [ name = "vector-vrl-cli" version = "0.1.0" dependencies = [ - "clap 4.4.3", + "clap 4.4.5", "vector-vrl-functions", "vrl", ] @@ -10036,7 +10075,7 @@ dependencies = [ "ansi_term", "chrono", "chrono-tz", - "clap 4.4.3", + "clap 4.4.5", "enrichment", "glob", "prettydiff", @@ -10078,9 +10117,9 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf1c43c0a6834959dd38eb762b462d156df043f3dfd86550ff62a96bd572466c" +checksum = "10d3c3ffe5322a8a7f63cf422aebb5000774e4152955fca1cd024c262893db5d" dependencies = [ "aes", "ansi_term", @@ -10097,8 +10136,9 @@ dependencies = [ "chrono", "chrono-tz", "cidr-utils", - "clap 4.4.3", + "clap 4.4.5", "codespan-reporting", + "community-id", "crypto_secretbox", "csv", "ctr", @@ -10122,7 +10162,7 @@ dependencies = [ "ofb", "once_cell", "onig", - "ordered-float 3.9.1", + "ordered-float 4.1.0", "paste", "peeking_take_while", "percent-encoding", @@ -10144,7 +10184,7 @@ dependencies = [ "sha2", "sha3", "snafu", - "strip-ansi-escapes 0.1.1", + "strip-ansi-escapes", "syslog_loose", "termcolor", "thiserror", @@ -10158,17 +10198,6 @@ dependencies = [ "zstd 0.12.4", ] -[[package]] -name = "vte" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cbce692ab4ca2f1f3047fcf732430249c0e971bfdd2b234cf2c47ad93af5983" -dependencies = [ - "arrayvec 0.5.2", - "utf8parse", - "vte_generate_state_changes", -] - [[package]] name = "vte" version = "0.11.1" @@ -10409,13 +10438,14 @@ dependencies = [ [[package]] name = "which" -version = "4.3.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix 0.38.14", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9526d4897b938..ca67af613d752 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ publish = false default-run = "vector" autobenches = false # our benchmarks are not runnable on their own either way # Minimum supported rust version -rust-version = "1.70.0" +rust-version = "1.71.1" [[bin]] name = "vector" @@ -117,7 +117,7 @@ members = [ ] [workspace.dependencies] -vrl = { version = "0.6.0", default-features = false, features = ["cli", "test", "test_framework", "arbitrary", "compiler", "value", "diagnostic", "path", "parser", "stdlib", "datadog", "core"] } +vrl = { version = "0.7.0", default-features = false, features = ["cli", "test", "test_framework", "arbitrary", "compiler", "value", "diagnostic", "path", "parser", "stdlib", "datadog", "core"] } pin-project = { version = "1.1.3", default-features = false } @@ -298,7 +298,7 @@ notify = { version = "6.1.1", default-features = false, features = ["macos_fseve once_cell = { version = "1.18", default-features = false } openssl = { version = "0.10.57", default-features = false, features = ["vendored"] } openssl-probe = { version = "0.1.5", default-features = false } -ordered-float = { version = "3.9.1", default-features = false } +ordered-float = { version = "4.1.0", default-features = false } paste = "1.0.14" percent-encoding = { version = "2.3.0", default-features = false } postgres-openssl = { version = "0.5.0", default-features = false, features = ["runtime"], optional = true } @@ -336,7 +336,7 @@ arr_macro = { version = "0.2.1" } heim = { git = "https://github.com/vectordotdev/heim.git", branch = "update-nix", default-features = false, features = ["disk"] } # make sure to update the external docs when the Lua version changes -mlua = { version = "0.8.10", default-features = false, features = ["lua54", "send", "vendored"], optional = true } +mlua = { version = "0.9.1", default-features = false, features = ["lua54", "send", "vendored", "macros"], optional = true } [target.'cfg(windows)'.dependencies] windows-service = "0.6.0" @@ -373,7 +373,7 @@ tokio = { version = "1.32.0", features = ["test-util"] } tokio-test = "0.4.3" tower-test = "0.4.0" vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl", "test"] } -vrl = { version = "0.6.0", features = ["cli", "test", "test_framework", "arbitrary"] } +vrl = { version = "0.7.0", features = ["cli", "test", "test_framework", "arbitrary"] } wiremock = "0.5.19" zstd = { version = "0.12.4", default-features = false } diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 41564f0004042..3ebe8070ad400 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -132,6 +132,7 @@ codespan-reporting,https://github.com/brendanzab/codespan,Apache-2.0,Brendan Zab colorchoice,https://github.com/rust-cli/anstyle,MIT OR Apache-2.0,The colorchoice Authors colored,https://github.com/mackwic/colored,MPL-2.0,Thomas Wickham combine,https://github.com/Marwes/combine,MIT,Markus Westerlind +community-id,https://github.com/traceflight/rs-community-id,MIT OR Apache-2.0,Julian Wang concurrent-queue,https://github.com/smol-rs/concurrent-queue,Apache-2.0 OR MIT,Stjepan Glavina concurrent-queue,https://github.com/smol-rs/concurrent-queue,Apache-2.0 OR MIT,"Stjepan Glavina , Taiki Endo , John Nunley " const-oid,https://github.com/RustCrypto/formats/tree/master/const-oid,Apache-2.0 OR MIT,RustCrypto Developers @@ -336,6 +337,8 @@ minimal-lexical,https://github.com/Alexhuszagh/minimal-lexical,MIT OR Apache-2.0 miniz_oxide,https://github.com/Frommi/miniz_oxide/tree/master/miniz_oxide,MIT OR Zlib OR Apache-2.0,"Frommi , oyvindln " mio,https://github.com/tokio-rs/mio,MIT,"Carl Lerche , Thomas de Zeeuw , Tokio Contributors " mlua,https://github.com/khvzak/mlua,MIT,"Aleksandr Orlenko , kyren " +mlua-sys,https://github.com/khvzak/mlua,MIT,Aleksandr Orlenko +mlua_derive,https://github.com/khvzak/mlua,MIT,Aleksandr Orlenko mongodb,https://github.com/mongodb/mongo-rust-driver,Apache-2.0,"Saghm Rossi , Patrick Freed , Isabel Atkinson , Abraham Egnor , Kaitlin Mahar " multer,https://github.com/rousan/multer-rs,MIT,Rousan Ali native-tls,https://github.com/sfackler/rust-native-tls,MIT OR Apache-2.0,Steven Fackler diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 1c86f744d01a4..eaa299a2b3dc0 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -15,7 +15,7 @@ dyn-clone = { version = "1", default-features = false } lookup = { package = "vector-lookup", path = "../vector-lookup", default-features = false } memchr = { version = "2", default-features = false } once_cell = { version = "1.18", default-features = false } -ordered-float = { version = "3.9.1", default-features = false } +ordered-float = { version = "4.1.0", default-features = false } prost = { version = "0.12.1", default-features = false, features = ["std"] } prost-reflect = { version = "0.12", default-features = false, features = ["serde"] } regex = { version = "1.9.5", default-features = false, features = ["std", "perf"] } @@ -38,7 +38,7 @@ futures = { version = "0.3", default-features = false } indoc = { version = "2", default-features = false } tokio = { version = "1", features = ["test-util"] } similar-asserts = "1.5.0" -vrl = { version = "0.6.0", features = ["cli", "test", "test_framework", "arbitrary"] } +vrl = { version = "0.7.0", features = ["cli", "test", "test_framework", "arbitrary"] } vector-core = { path = "../vector-core", default-features = false, features = ["test"] } [features] diff --git a/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch b/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch index bc0821aae9c76..ac046ce0adf7d 100644 --- a/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch +++ b/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch @@ -6,8 +6,8 @@ index 0f51fc830..aadf99841 100644 rand_distr = "0.4.3" tracing-subscriber = { version = "0.3.17", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } vector-common = { path = "../vector-common", default-features = false, features = ["test"] } --vrl = { version = "0.6.0", features = ["cli", "test", "test_framework", "arbitrary"] } -+vrl = { version = "0.6.0", features = ["cli", "test", "test_framework", "arbitrary"], path = "../../../vrl" } +-vrl = { version = "0.7.0", features = ["cli", "test", "test_framework", "arbitrary"] } ++vrl = { version = "0.7.0", features = ["cli", "test", "test_framework", "arbitrary"], path = "../../../vrl" } [features] api = ["dep:async-graphql"] diff --git a/lib/opentelemetry-proto/Cargo.toml b/lib/opentelemetry-proto/Cargo.toml index 46dc425bdd939..66a0f377bf6e3 100644 --- a/lib/opentelemetry-proto/Cargo.toml +++ b/lib/opentelemetry-proto/Cargo.toml @@ -14,7 +14,7 @@ bytes = { version = "1.5.0", default-features = false, features = ["serde"] } chrono = { version = "0.4.19", default-features = false, features = ["serde"] } hex = { version = "0.4.3", default-features = false, features = ["std"] } lookup = { package = "vector-lookup", path = "../vector-lookup", default-features = false } -ordered-float = { version = "3.9.1", default-features = false } +ordered-float = { version = "4.1.0", default-features = false } prost = { version = "0.12", default-features = false, features = ["std"] } tonic = { version = "0.10", default-features = false, features = ["codegen", "gzip", "prost", "tls", "tls-roots", "transport"] } vrl.workspace = true diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 617b138bb7e20..a4c6dc4af3f68 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -51,7 +51,7 @@ futures = { version = "0.3.28", default-features = false, features = ["std"] } indexmap = { version = "~2.0.0", default-features = false, features = ["std"] } metrics = "0.21.1" nom = { version = "7", optional = true } -ordered-float = { version = "3.9.1", default-features = false } +ordered-float = { version = "4.1.0", default-features = false } paste = "1.0.14" pin-project.workspace = true ryu = { version = "1", default-features = false } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 849a6dca87c5d..5a8b9d5344d56 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -28,10 +28,10 @@ lookup = { package = "vector-lookup", path = "../vector-lookup" } metrics = "0.21.1" metrics-tracing-context = { version = "0.14.0", default-features = false } metrics-util = { version = "0.15.1", default-features = false, features = ["registry"] } -mlua = { version = "0.8.10", default-features = false, features = ["lua54", "send", "vendored"], optional = true } +mlua = { version = "0.9.1", default-features = false, features = ["lua54", "send", "vendored"], optional = true } no-proxy = { version = "0.3.4", default-features = false, features = ["serialize"] } once_cell = { version = "1.18", default-features = false } -ordered-float = { version = "3.9.1", default-features = false } +ordered-float = { version = "4.1.0", default-features = false } openssl = { version = "0.10.57", default-features = false, features = ["vendored"] } parking_lot = { version = "0.12.1", default-features = false } pin-project.workspace = true @@ -95,7 +95,7 @@ rand = "0.8.5" rand_distr = "0.4.3" tracing-subscriber = { version = "0.3.17", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } vector-common = { path = "../vector-common", default-features = false, features = ["test"] } -vrl = { version = "0.6.0", features = ["cli", "test", "test_framework", "arbitrary"] } +vrl = { version = "0.7.0", features = ["cli", "test", "test_framework", "arbitrary"] } [features] api = ["dep:async-graphql"] diff --git a/lib/vector-core/src/event/lua/event.rs b/lib/vector-core/src/event/lua/event.rs index 754bfb4d0e18e..ded7dab1728ec 100644 --- a/lib/vector-core/src/event/lua/event.rs +++ b/lib/vector-core/src/event/lua/event.rs @@ -8,19 +8,19 @@ pub struct LuaEvent { pub metric_multi_value_tags: bool, } -impl<'a> ToLua<'a> for LuaEvent { +impl<'a> IntoLua<'a> for LuaEvent { #![allow(clippy::wrong_self_convention)] // this trait is defined by mlua - fn to_lua(self, lua: &'a Lua) -> LuaResult { + fn into_lua(self, lua: &'a Lua) -> LuaResult { let table = lua.create_table()?; match self.event { - Event::Log(log) => table.raw_set("log", log.to_lua(lua)?)?, + Event::Log(log) => table.raw_set("log", log.into_lua(lua)?)?, Event::Metric(metric) => table.raw_set( "metric", LuaMetric { metric, multi_value_tags: self.metric_multi_value_tags, } - .to_lua(lua)?, + .into_lua(lua)?, )?, Event::Trace(_) => { return Err(LuaError::ToLuaConversionError { @@ -92,7 +92,7 @@ mod test { } #[test] - fn to_lua_log() { + fn into_lua_log() { let mut event = LogEvent::default(); event.insert("field", "value"); @@ -107,7 +107,7 @@ mod test { } #[test] - fn to_lua_metric() { + fn into_lua_metric() { let event = Event::Metric(Metric::new( "example counter", MetricKind::Absolute, diff --git a/lib/vector-core/src/event/lua/log.rs b/lib/vector-core/src/event/lua/log.rs index ef08a54cf62d8..8d05055ef1a3c 100644 --- a/lib/vector-core/src/event/lua/log.rs +++ b/lib/vector-core/src/event/lua/log.rs @@ -2,11 +2,11 @@ use mlua::prelude::*; use crate::event::{EventMetadata, LogEvent, Value}; -impl<'a> ToLua<'a> for LogEvent { +impl<'a> IntoLua<'a> for LogEvent { #![allow(clippy::wrong_self_convention)] // this trait is defined by mlua - fn to_lua(self, lua: &'a Lua) -> LuaResult { + fn into_lua(self, lua: &'a Lua) -> LuaResult { let (value, _metadata) = self.into_parts(); - value.to_lua(lua) + value.into_lua(lua) } } @@ -22,7 +22,7 @@ mod test { use super::*; #[test] - fn to_lua() { + fn into_lua() { let mut log = LogEvent::default(); log.insert("a", 1); log.insert("nested.field", "2"); diff --git a/lib/vector-core/src/event/lua/metric.rs b/lib/vector-core/src/event/lua/metric.rs index 057e1c01cd0bc..f1bdbb73e8344 100644 --- a/lib/vector-core/src/event/lua/metric.rs +++ b/lib/vector-core/src/event/lua/metric.rs @@ -22,9 +22,9 @@ pub struct LuaMetricTags { pub multi_value_tags: bool, } -impl<'a> ToLua<'a> for MetricKind { +impl<'a> IntoLua<'a> for MetricKind { #![allow(clippy::wrong_self_convention)] // this trait is defined by mlua - fn to_lua(self, lua: &'a Lua) -> LuaResult { + fn into_lua(self, lua: &'a Lua) -> LuaResult { let kind = match self { MetricKind::Absolute => "absolute", MetricKind::Incremental => "incremental", @@ -49,8 +49,8 @@ impl<'a> FromLua<'a> for MetricKind { } } -impl<'a> ToLua<'a> for StatisticKind { - fn to_lua(self, lua: &'a Lua) -> LuaResult { +impl<'a> IntoLua<'a> for StatisticKind { + fn into_lua(self, lua: &'a Lua) -> LuaResult { let kind = match self { StatisticKind::Summary => "summary", StatisticKind::Histogram => "histogram", @@ -105,14 +105,14 @@ impl<'a> FromLua<'a> for MetricTags { } } -impl<'a> ToLua<'a> for LuaMetricTags { - fn to_lua(self, lua: &'a Lua) -> LuaResult { +impl<'a> IntoLua<'a> for LuaMetricTags { + fn into_lua(self, lua: &'a Lua) -> LuaResult { if self.multi_value_tags { Ok(LuaValue::Table(lua.create_table_from( self.tags.0.into_iter().map(|(key, value)| { let value: Vec<_> = value .into_iter() - .filter_map(|tag_value| tag_value.into_option().to_lua(lua).ok()) + .filter_map(|tag_value| tag_value.into_option().into_lua(lua).ok()) .collect(); (key, value) }), @@ -125,9 +125,9 @@ impl<'a> ToLua<'a> for LuaMetricTags { } } -impl<'a> ToLua<'a> for LuaMetric { +impl<'a> IntoLua<'a> for LuaMetric { #![allow(clippy::wrong_self_convention)] // this trait is defined by mlua - fn to_lua(self, lua: &'a Lua) -> LuaResult { + fn into_lua(self, lua: &'a Lua) -> LuaResult { let tbl = lua.create_table()?; tbl.raw_set("name", self.metric.name())?; @@ -373,7 +373,7 @@ mod test { } #[test] - fn to_lua_counter_full() { + fn into_lua_counter_full() { let metric = Metric::new( "example counter", MetricKind::Incremental, @@ -459,7 +459,7 @@ mod test { } #[test] - fn to_lua_counter_minimal() { + fn into_lua_counter_minimal() { let metric = Metric::new( "example counter", MetricKind::Absolute, @@ -483,7 +483,7 @@ mod test { } #[test] - fn to_lua_gauge() { + fn into_lua_gauge() { let metric = Metric::new( "example gauge", MetricKind::Absolute, @@ -497,7 +497,7 @@ mod test { } #[test] - fn to_lua_set() { + fn into_lua_set() { let metric = Metric::new( "example set", MetricKind::Incremental, @@ -521,7 +521,7 @@ mod test { } #[test] - fn to_lua_distribution() { + fn into_lua_distribution() { let metric = Metric::new( "example distribution", MetricKind::Incremental, @@ -546,7 +546,7 @@ mod test { } #[test] - fn to_lua_aggregated_histogram() { + fn into_lua_aggregated_histogram() { let metric = Metric::new( "example histogram", MetricKind::Incremental, @@ -574,7 +574,7 @@ mod test { } #[test] - fn to_lua_aggregated_summary() { + fn into_lua_aggregated_summary() { let metric = Metric::new( "example summary", MetricKind::Incremental, diff --git a/lib/vector-vrl/tests/src/main.rs b/lib/vector-vrl/tests/src/main.rs index 9452c73b97ece..962f68b2ef296 100644 --- a/lib/vector-vrl/tests/src/main.rs +++ b/lib/vector-vrl/tests/src/main.rs @@ -4,6 +4,8 @@ mod docs; mod test_enrichment; +use std::env; +use std::path::PathBuf; use vrl::test::{get_tests_from_functions, run_tests, Test, TestConfig}; use chrono_tz::Tz; @@ -111,8 +113,15 @@ fn main() { ); } +pub fn test_dir() -> PathBuf { + PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()) +} + +fn test_glob_pattern() -> String { + test_dir().join("**/*.vrl").to_str().unwrap().to_string() +} fn get_tests(cmd: &Cmd) -> Vec { - glob("tests/**/*.vrl") + glob(test_glob_pattern().as_str()) .expect("valid pattern") .filter_map(|entry| { let path = entry.ok()?; diff --git a/lib/vrl/value/Cargo.toml b/lib/vrl/value/Cargo.toml index d9e51dbe25ba5..11854a880617f 100644 --- a/lib/vrl/value/Cargo.toml +++ b/lib/vrl/value/Cargo.toml @@ -18,7 +18,7 @@ tracing = { version = "0.1.34", default-features = false, features = ["attribute # Optional async-graphql = { version = "5.0.7", default-features = false, optional = true } -mlua = { version = "0.8.8", default-features = false, features = ["lua54", "send", "vendored"], optional = true} +mlua = { version = "0.9.1", default-features = false, features = ["lua54", "send", "vendored"], optional = true} serde = { version = "1.0.158", default-features = false, features = ["derive", "rc"], optional = true } serde_json = { version = "1.0.94", optional = true } toml = { version = "0.7.3", default-features = false, optional = true } @@ -39,4 +39,4 @@ lookup = { path = "../lookup", default-features = false, features = ["arbitrary" serde = { version = "1.0.158", default-features = false, features = ["derive", "rc"]} serde_json = { version = "1.0.94"} toml = { version = "0.7.3", default-features = false } -mlua = { version = "0.8.8", default-features = false, features = ["lua54", "send", "vendored"]} +mlua = { version = "0.9.1", default-features = false, features = ["lua54", "send", "vendored"]} diff --git a/src/internal_events/lua.rs b/src/internal_events/lua.rs index 302e908d490c1..a18665d87b419 100644 --- a/src/internal_events/lua.rs +++ b/src/internal_events/lua.rs @@ -82,7 +82,6 @@ const fn mlua_error_code(err: &mlua::Error) -> &'static str { MemoryError(_) => "memory_error", SafetyError(_) => "memory_safety_error", MemoryLimitNotAvailable => "memory_limit_not_available", - MainThreadNotAvailable => "main_thread_not_available", RecursiveMutCallback => "mutable_callback_called_recursively", CallbackDestructed => "callback_destructed", StackError => "out_of_stack", diff --git a/src/transforms/lua/v1/mod.rs b/src/transforms/lua/v1/mod.rs index 4da72b6c8e9bd..5fb82cf9610a3 100644 --- a/src/transforms/lua/v1/mod.rs +++ b/src/transforms/lua/v1/mod.rs @@ -2,6 +2,7 @@ use std::{future::ready, pin::Pin}; use futures::{stream, Stream, StreamExt}; use mlua::ExternalError; +use mlua::FromLua; use ordered_float::NotNan; use snafu::{ResultExt, Snafu}; use vector_config::configurable_component; @@ -104,7 +105,7 @@ impl Clone for Lua { // This wrapping structure is added in order to make it possible to have independent implementations // of `mlua::UserData` trait for event in version 1 and version 2 of the transform. -#[derive(Clone)] +#[derive(Clone, FromLua)] struct LuaEvent { inner: Event, } @@ -225,7 +226,7 @@ impl mlua::UserData for LuaEvent { methods.add_meta_method_mut( mlua::MetaMethod::NewIndex, |_lua, this, (key, value): (String, Option>)| { - let key_path = parse_target_path(key.as_str()).map_err(|e| e.to_lua_err())?; + let key_path = parse_target_path(key.as_str()).map_err(|e| e.into_lua_err())?; match value { Some(mlua::Value::String(string)) => { this.inner.as_mut_log().insert( diff --git a/tests/behavior/transforms/remap.toml b/tests/behavior/transforms/remap.toml index 2fdd1f7fb243e..baf162137300e 100644 --- a/tests/behavior/transforms/remap.toml +++ b/tests/behavior/transforms/remap.toml @@ -256,7 +256,7 @@ .bar = to_int!(.bar) .baz = to_float!(.baz) .bev = to_bool!(.bev) - .a = to_string(to_timestamp!(.a)) + .a = to_string(parse_timestamp!(.a, "%+")) """ [[tests]] name = "remap_coercion" @@ -643,7 +643,7 @@ inputs = [] type = "remap" source = """ - .a = format_timestamp!(to_timestamp!(.foo), format: "%+") + .a = format_timestamp!(from_unix_timestamp!(.foo), format: "%+") """ [[tests]] name = "remap_function_format_timestamp" @@ -1166,7 +1166,7 @@ .boolean = tag_types_externally(true) .map = tag_types_externally({ "foo": "bar" }) .array = tag_types_externally(["foo"]) - .timestamp = tag_types_externally(to_timestamp!("2021-01-01T00:00:00.00Z")) + .timestamp = tag_types_externally(parse_timestamp!("2021-01-01T00:00:00.00Z", "%+")) .regex = tag_types_externally(r'.*') .null = tag_types_externally(null) """ @@ -1187,7 +1187,7 @@ .boolean == { "boolean": true } && .map == { "foo": { "string": "bar" } } && .array == [{ "string": "foo" }] && - .timestamp == { "timestamp": to_timestamp!("2021-01-01T00:00:00.00Z") } && + .timestamp == { "timestamp": parse_timestamp!("2021-01-01T00:00:00.00Z", "%+") } && .regex == { "regex": r'.*' } && .null == null ''' diff --git a/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md b/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md index d59ad8e452baa..6c67bbdfb40ac 100644 --- a/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md +++ b/website/content/en/guides/advanced/parsing-csv-logs-with-lua.md @@ -241,7 +241,6 @@ built-in functions, such as [`tonumber`][urls.lua_tonumber]. Alternatively, it i [docs.sinks.console]: /docs/reference/configuration/sinks/console/ [docs.sources.file#multiline]: /docs/reference/configuration/sources/file/#multiline [docs.sources.file]: /docs/reference/configuration/sources/file/ -[docs.transforms.coercer#timestamps]: /docs/reference/vrl/functions/#to_timestamp [docs.transforms.coercer]: /docs/reference/vrl/functions/#coerce-functions [docs.transforms.lua#data-types]: /docs/reference/configuration/transforms/lua/#event-data-model [docs.transforms.lua#process]: /docs/reference/configuration/transforms/lua/#hooks.process diff --git a/website/cue/reference/remap/functions/community_id.cue b/website/cue/reference/remap/functions/community_id.cue new file mode 100644 index 0000000000000..f8aa79164a821 --- /dev/null +++ b/website/cue/reference/remap/functions/community_id.cue @@ -0,0 +1,59 @@ +package metadata + +remap: functions: community_id: { + category: "String" + description: """ + Used to generate an id based on the [Community ID Spec](\(urls.community_id_spec)). + """ + + arguments: [ + { + name: "source_ip" + description: "The source IP address." + required: true + type: ["string"] + }, + { + name: "destination_ip" + description: "The destination IP address." + required: true + type: ["string"] + }, + { + name: "protocol" + description: "The protocol number." + required: true + type: ["integer"] + }, + { + name: "source_port" + description: "The source port." + required: false + type: ["integer"] + }, + { + name: "destination_port" + description: "The destination port." + required: false + type: ["integer"] + }, + { + name: "seed" + description: "The custom seed number." + required: false + type: ["integer"] + }, + ] + internal_failure_reasons: [] + return: types: ["string"] + + examples: [ + { + title: "TCP" + source: #""" + community_id!(source_ip: "1.2.3.4", destination_ip: "5.6.7.8", source_port: 1122, destination_port: 3344, protocol: 6) + """# + return: "1:wCb3OG7yAFWelaUydu0D+125CLM=" + }, + ] +} diff --git a/website/cue/reference/remap/functions/encode_json.cue b/website/cue/reference/remap/functions/encode_json.cue index f2fa58df05130..1757905cf7ee6 100644 --- a/website/cue/reference/remap/functions/encode_json.cue +++ b/website/cue/reference/remap/functions/encode_json.cue @@ -13,6 +13,12 @@ remap: functions: encode_json: { required: true type: ["any"] }, + { + name: "pretty" + description: "Whether to pretty print the JSON string or not." + required: false + type: ["boolean"] + }, ] internal_failure_reasons: [] return: types: ["string"] diff --git a/website/cue/reference/remap/functions/parse_aws_vpc_flow_log.cue b/website/cue/reference/remap/functions/parse_aws_vpc_flow_log.cue index 0bbf752a562da..8fcd5a8e29abc 100644 --- a/website/cue/reference/remap/functions/parse_aws_vpc_flow_log.cue +++ b/website/cue/reference/remap/functions/parse_aws_vpc_flow_log.cue @@ -33,7 +33,7 @@ remap: functions: parse_aws_vpc_flow_log: { """# return: { "version": 2 - "account_id": 123456789010 + "account_id": "123456789010" "interface_id": "eni-1235b8ca123456789" "srcaddr": null "dstaddr": null @@ -65,5 +65,43 @@ remap: functions: parse_aws_vpc_flow_log: { "pkt_dstaddr": "203.0.113.5" } }, + { + title: "Parse AWS VPC Flow including v5 fields" + source: #""" + parse_aws_vpc_flow_log!("5 52.95.128.179 10.0.0.71 80 34210 6 1616729292 1616729349 IPv4 14 15044 123456789012 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-0c50d5961bcb2d47b eni-1235b8ca123456789 ap-southeast-2 apse2-az3 - - ACCEPT 19 52.95.128.179 10.0.0.71 S3 - - ingress OK", + format: "version srcaddr dstaddr srcport dstport protocol start end type packets bytes account_id vpc_id subnet_id instance_id interface_id region az_id sublocation_type sublocation_id action tcp_flags pkt_srcaddr pkt_dstaddr pkt_src_aws_service pkt_dst_aws_service traffic_path flow_direction log_status") + """# + return: { + "account_id": "123456789012" + "action": "ACCEPT" + "az_id": "apse2-az3" + "bytes": 15044 + "dstaddr": "10.0.0.71" + "dstport": 34210 + "end": 1616729349 + "flow_direction": "ingress" + "instance_id": "i-0c50d5961bcb2d47b" + "interface_id": "eni-1235b8ca123456789" + "log_status": "OK" + "packets": 14 + "pkt_dst_aws_service": null + "pkt_dstaddr": "10.0.0.71" + "pkt_src_aws_service": "S3" + "pkt_srcaddr": "52.95.128.179" + "protocol": 6 + "region": "ap-southeast-2" + "srcaddr": "52.95.128.179" + "srcport": 80 + "start": 1616729292 + "sublocation_id": null + "sublocation_type": null + "subnet_id": "subnet-aaaaaaaa012345678" + "tcp_flags": 19 + "traffic_path": null + "type": "IPv4" + "version": 5 + "vpc_id": "vpc-abcdefab012345678" + } + }, ] } diff --git a/website/cue/reference/remap/functions/to_timestamp.cue b/website/cue/reference/remap/functions/to_timestamp.cue deleted file mode 100644 index f46e3a37d23b1..0000000000000 --- a/website/cue/reference/remap/functions/to_timestamp.cue +++ /dev/null @@ -1,90 +0,0 @@ -package metadata - -remap: functions: to_timestamp: { - category: "Coerce" - description: """ - This function is deprecated. - For integer values, use `from_unix_timestamp`, otherwise use `parse_timestamp`. - Coerces the `value` into a timestamp. - """ - notices: ["There is the possibility of precision loss due to float arithmetic when coercing floats."] - - deprecated: true - - arguments: [ - { - name: "value" - description: "The value that is to be converted to a timestamp. If a string, must be a valid representation of a `timestamp` otherwise an `ArgumentError` will be raised." - required: true - type: ["string", "float", "integer", "timestamp"] - }, - { - name: "unit" - description: "The time unit." - type: ["string"] - required: false - enum: { - seconds: "Express Unix time in seconds" - milliseconds: "Express Unix time in milliseconds" - nanoseconds: "Express Unix time in nanoseconds" - } - default: "seconds" - }, - ] - internal_failure_reasons: [ - "When `value` is a `string`, it is not a valid timestamp format", - "When `value` is an `int`, it is not within the Unix timestamp range", - "When `value` is a `float`, it is not within the Unix timestamp range", - ] - return: { - types: ["timestamp"] - rules: [ - "If `value` is a `string`, the timestamp is parsed in these formats.", - "If `value` is an `integer`, it is assumed to be a Unix representation of the timestamp (the number of seconds after January 1st, 1970).", - "If `value` is a `float`, it s assumed to be a Unix representation of the timestamp (the number of seconds after January 1st, 1970) with fractional seconds.", - ] - } - - examples: [ - { - title: "Coerce a string to a timestamp" - source: """ - to_timestamp!("2020-10-21T16:00:00Z") - """ - return: "2020-10-21T16:00:00Z" - skip_test: true - }, - { - title: "Coerce a unix timestamp (integer) to a timestamp" - source: """ - to_timestamp!(1675968923) - """ - return: "2023-02-09T18:55:23Z" - skip_test: true - }, - { - title: "Coerce a unix timestamp (float) to a timestamp" - source: """ - to_timestamp!(1675968923.567) - """ - return: "2023-02-09T18:55:23.566999912Z" - skip_test: true - }, - { - title: "Coerce a unix timestamp, in milliseconds, to a timestamp" - source: """ - to_timestamp!(1676478566639, unit: "milliseconds") - """ - return: "2023-02-15T16:29:26.639Z" - skip_test: true - }, - { - title: "Coerce a unix timestamp, in nanoseconds, to a timestamp" - source: """ - to_timestamp!(1675968923012312311, unit: "nanoseconds") - """ - return: "2023-02-09T18:55:23.012312311Z" - skip_test: true - }, - ] -} diff --git a/website/cue/reference/remap/functions/truncate.cue b/website/cue/reference/remap/functions/truncate.cue index c0961b9ef7449..f67e444084a78 100644 --- a/website/cue/reference/remap/functions/truncate.cue +++ b/website/cue/reference/remap/functions/truncate.cue @@ -22,8 +22,17 @@ remap: functions: truncate: { { name: "ellipsis" description: """ - An ellipsis (`...`) is appended if this is set to `true` _and_ the `value` string ends up being - truncated because it's exceeded the `limit`. + This argument is deprecated. An ellipsis (`...`) is appended if this is set to `true` _and_ the `value` string + ends up being truncated because it's exceeded the `limit`. + """ + required: true + type: ["boolean"] + }, + { + name: "suffix" + description: """ + A custom suffix (`...`) will be appended to truncated strings. + This is ignored if "ellipsis" is set to true for backwards compatibility. """ required: true type: ["boolean"] @@ -42,9 +51,16 @@ remap: functions: truncate: { { title: "Truncate a string" source: #""" - truncate("A rather long sentence.", limit: 11, ellipsis: true) + truncate("A rather long sentence.", limit: 11, suffix: "...") """# return: "A rather lo..." }, + { + title: "Truncate a string" + source: #""" + truncate("A rather long sentence.", limit: 11, suffix: "[TRUNCATED]") + """# + return: "A rather lo[TRUNCATED]" + }, ] } diff --git a/website/cue/reference/urls.cue b/website/cue/reference/urls.cue index 485352e891fd5..836b1050114ad 100644 --- a/website/cue/reference/urls.cue +++ b/website/cue/reference/urls.cue @@ -112,9 +112,10 @@ urls: { cloudsmith: "https://cloudsmith.io/~timber/repos/vector/packages/" cloudsmith_apt: "https://cloudsmith.io/~timber/repos/vector/setup/#formats-deb" cloudsmith_yum: "https://cloudsmith.io/~timber/repos/vector/setup/#formats-rpm" + community_id_spec: "https://github.com/corelight/community-id-spec" console: "\(wikipedia)/wiki/System_console" - conventional_commits: "https://www.conventionalcommits.org" contributing: "\(vector_repo)/blob/master/CONTRIBUTING.md#setup" + conventional_commits: "https://www.conventionalcommits.org" crc: "\(wikipedia)/wiki/Cyclic_redundancy_check" ctime: "https://www.cplusplus.com/reference/ctime" cue: "https://cuelang.org/"