Clean up and update dependencies for 0.14 dev cycle (#823)

Cargo.toml

@@ -65,6 +65,9 @@ prost-build = {version = "0.13" }
 prost-types = "0.13"
 aws-config = "1.5.13"
 reqwest = "0.12"
+rusqlite = "0.31"
+apache-avro = "0.17.0"
+k8s-openapi = "0.24.0"
 
 [profile.release]
 debug = 1

crates/arroyo-api/Cargo.toml

@@ -15,79 +15,48 @@ arroyo-server-common = { path = "../arroyo-server-common" }
 arroyo-operator = { path = "../arroyo-operator" }
 arroyo-connectors = { path = "../arroyo-connectors" }
 arroyo-datastream = { path = "../arroyo-datastream" }
-arroyo-state = { path = "../arroyo-state" }
 arroyo-formats = { path = "../arroyo-formats" }
 arroyo-udf-host = { path = "../arroyo-udf/arroyo-udf-host" }
 arroyo-udf-python = { path = "../arroyo-udf/arroyo-udf-python" }
 
 tonic = { workspace = true }
-tonic-reflection = { workspace = true }
-tonic-web = { workspace = true }
 prost = {workspace = true}
-prost-reflect = {workspace = true}
 tokio = { version = "1", features = ["full"] }
 tokio-stream = "0.1.12"
 tower = {workspace = true}
-rand = "0.8"
-rand_chacha = "0.3"
-async-trait = "0.1"
-chrono = "0.4"
-once_cell = "1"
 
 arrow = { workspace = true }
 arrow-schema = {workspace = true, features = ["serde"]}
 
-bincode = { version = "2.0.0-rc.3", features = ["serde"]}
-petgraph = {version = "0.6", features = ["serde-1"]}
+petgraph = {version = "0.7", features = ["serde-1"]}
 
 http = "1"
 tower-http = { workspace = true, features = ["trace", "fs", "cors", "validate-request", "auth", "compression-zstd"]}
 axum = {workspace = true, features = ["tokio", "macros"]}
 axum-extra = { version = "0.9", features = ["typed-header"] }
-h2 = "0.4"
 thiserror = "1.0.40"
 utoipa = { workspace = true }
 utoipa-swagger-ui = { version = "7", features = ["axum"] }
 
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 
-argon2 = "0.5"
-
 # logging
 tracing = "0.1"
 anyhow = "1.0.70"
 
-# json-schema support
-typify = "0.0.13"
-schemars = "0.8"
-
-# metric querying
-reqwest = {workspace = true}
-base64 = '0.21'
-
-# codegen
-syn = {version = "2", features = ["full"]}
-quote = "1"
-proc-macro2 = "1"
-
 # postgres
 postgres-types = { version = "*", features = ["derive"] }
 tokio-postgres = { version = "*", features = ["with-serde_json-1", "with-time-0_3", "with-uuid-1"] }
-deadpool-postgres = { workspace = true }
 
 # sqlite
-rusqlite = { version = "0.31.0", features = ["bundled", "serde_json", "time"] }
-deadpool-sqlite = { version = "0.8.0", features = ["tracing", "serde"]}
+rusqlite = { workspace = true, features = ["bundled", "serde_json", "time"] }
 
 futures = "0.3"
 futures-util = "0.3.28"
 time = "0.3"
 cornucopia_async = { workspace = true, features = ["with-serde_json-1"]}
-jwt-simple = "0.11.4"
-uuid = "1.3.3"
-regress = "0.10"
-apache-avro = "0.16.0"
+apache-avro = { workspace = true }
 toml = "0.8"
 rust-embed = { version = "8", features = ["axum"] }
 mime_guess = "2.0.4"
@@ -97,5 +66,5 @@ cornucopia = { workspace = true }
 postgres = "0.19.5"
 arroyo-types = { path = "../arroyo-types" }
 utoipa = { workspace = true }
-rusqlite = "0.31.0"
+rusqlite = {workspace =  true}
 refinery = { version = "0.8.14", features = ["rusqlite"] }

crates/arroyo-connectors/Cargo.toml

@@ -13,7 +13,6 @@ default = []
 arroyo-types = { path = "../arroyo-types" }
 arroyo-storage = { path = "../arroyo-storage" }
 arroyo-rpc = { path = "../arroyo-rpc" }
-arroyo-datastream = { path = "../arroyo-datastream" }
 arroyo-formats = { path = "../arroyo-formats" }
 arroyo-operator = { path = "../arroyo-operator" }
 arroyo-state = { path = "../arroyo-state" }
@@ -27,34 +26,29 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 tokio = { version = "1", features = ["full"] }
 tokio-stream = "0.1"
-once_cell = "1.17.1"
 typify = "0.0.13"
-schemars = "0.8"
 prost = {workspace = true}
 tonic = {workspace = true}
-governor = "0.7.0"
+governor = "0.8.0"
 anyhow = "1.0.71"
 tracing = "0.1.37"
 regress = "0.10.0"
 futures = "0.3.28"
-axum = {workspace = true}
 rand = "0.8.5"
-base64 = "0.13.1"
 bytes = "1.5.0"
 url = "2.5.0"
-itertools = "0.11.0"
+itertools = "0.14.0"
 regex = "1"
 
 ##########################
 # connector dependencies #
 ##########################
 
 # Kafka
-aws-sdk-kafka = { version = "1.44" }
 aws-msk-iam-sasl-signer = "1.0.0"
-rdkafka = { version = "0.36", features = ["cmake-build", "tracing", "sasl", "ssl-vendored", "zstd"] }
-rdkafka-sys = "4.7.0"
-sasl2-sys = { version = "0.1.6", features = ["vendored"] }
+rdkafka = { version = "0.37", features = ["cmake-build", "tracing", "sasl", "ssl-vendored", "zstd"] }
+rdkafka-sys = "4.8.0"
+sasl2-sys = { version = "0.1.22", features = ["vendored"] }
 
 # SSE
 eventsource-client = "0.13.0"
@@ -69,11 +63,10 @@ reqwest = { workspace = true, features = ["stream"] }
 rabbitmq-stream-client = "0.7"
 
 # Redis
-redis = { version = "0.27", features = ["default", "tokio-rustls-comp", "cluster-async", "connection-manager"] }
+redis = { version = "0.28", features = ["default", "tokio-rustls-comp", "cluster-async", "connection-manager"] }
 
 # Fluvio
-fluvio = {version = "0.23", features = ["openssl"]}
-fluvio-future = "0.7"
+fluvio = {version = "0.24", features = ["openssl"]}
 
 # Kinesis
 aws-sdk-kinesis = { version = "1.44" }
@@ -94,7 +87,7 @@ tokio-rustls = "0.25"
 rustls = "0.22"
 
 # NATS
-async-nats = "0.37.0"
+async-nats = "0.38.0"
 
 [build-dependencies]
 glob = "0.3"

crates/arroyo-controller/Cargo.toml

@@ -11,39 +11,27 @@ arroyo-types = { path = "../arroyo-types" }
 arroyo-datastream = { path = "../arroyo-datastream" }
 arroyo-rpc = { path = "../arroyo-rpc" }
 arroyo-state = { path = "../arroyo-state" }
-arroyo-storage = { path = "../arroyo-storage" }
 arroyo-server-common = { path = "../arroyo-server-common" }
 arroyo-worker = { path = "../arroyo-worker" }
 
 tonic = {workspace = true}
-tonic-reflection = {workspace = true}
 
 prost = {workspace = true}
 tokio = { version = "1", features = ["full"] }
 tokio-stream = "0.1.12"
 rand = "0.8"
-bincode = { version = "2.0.0-rc.3", features = ["serde"]}
-petgraph = {version = "0.6", features = ["serde-1"]}
-quote = "1"
-proc-macro2 = "1"
-prettyplease = "0.2.4"
-syn = {version = "2", features = ["full"]}
+petgraph = {version = "0.7", features = ["serde-1"]}
 prometheus = "0.13"
 async-trait = "0.1"
 lazy_static = "1.4.0"
-chrono = "0.4"
-zstd = "0.13"
-
-arrow-schema = {workspace = true}
 
 serde = "1"
 
 anyhow = "1.0.70"
 
 # Kubernetes
-kube = { version = "0.96", features = ["runtime", "derive"] }
-k8s-openapi = { version = "0.23.0", features = ["v1_30"] }
-serde_yaml = {version = "0.9"}
+kube = { version = "0.98", features = ["runtime", "derive"] }
+k8s-openapi = { workspace = true, features = ["v1_30"] }
 shlex = "1.3"
 
 # json-schema support
@@ -55,22 +43,16 @@ tracing = "0.1"
 # SQL
 postgres-types = { version = "*", features = ["derive"] }
 tokio-postgres = { version = "*", features = ["with-serde_json-1", "with-time-0_3", "with-uuid-1"] }
-deadpool-postgres = { workspace = true }
 futures = "0.3"
 time = "0.3"
 cornucopia_async = { workspace = true, features = ["with-serde_json-1"] }
 thiserror = "1.0.40"
 regex = "1.7.3"
-reqwest = { workspace = true, features = ["json"] }
-uuid = "1.3.3"
-async-stream = "0.3.5"
-base64 = "0.22"
-rusqlite = { version = "0.31.0", features = ["serde_json", "time"] }
-log = "0.4.22"
+rusqlite = { workspace = true, features = ["serde_json", "time"] }
 
 [build-dependencies]
 cornucopia = { workspace = true }
 postgres = "0.19.5"
 arroyo-types = { path = "../arroyo-types" }
-rusqlite = "0.31.0"
+rusqlite = { workspace = true }
 refinery = { version = "0.8.14", features = ["rusqlite"] }

crates/arroyo-datastream/Cargo.toml

@@ -5,29 +5,17 @@ edition = "2021"
 
 
 [dependencies]
-arroyo-types = { path = "../arroyo-types" }
 arroyo-rpc = { path = "../arroyo-rpc" }
 arrow-schema = { workspace = true, features = ["serde"] }
 
-ahash = {workspace = true}
-dyn-clone = "1.0.11"
-petgraph = {version = "0.6", features = ["serde-1"]}
+petgraph = {version = "0.7", features = ["serde-1"]}
 serde = {version = "1", features = ["derive"]}
 syn = {version = "2", features = ["full"]}
-quote = "1"
-proc-macro2 = "1"
 bincode = { version = "2.0.0-rc.3", features = ["serde"]}
 rand = "0"
-toml = "0.8"
-bytes = "1.4"
-hex = "0.4"
-
-tokio = "1"
-tonic = {workspace = true}
 anyhow = "1.0.70"
 prost = {workspace = true}
-regex = "1.9.5"
 serde_json = "1.0.108"
-strum = { version = "0.25.0", features = ["derive"] }
+strum = { version = "0.26.0", features = ["derive"] }
 datafusion-proto = { workspace = true }
-itertools = "0.13.0"
+itertools = "0.14"

crates/arroyo-formats/Cargo.toml

@@ -7,26 +7,22 @@ edition = "2021"
 arroyo-types = { path = "../arroyo-types" }
 arroyo-rpc = { path = "../arroyo-rpc" }
 
-apache-avro = "0.16.0"
+apache-avro = {workspace = true}
 serde = {version = "1.0", features = ["derive"]}
 serde_json = "1.0"
-utoipa = { workspace = true }
 arrow = { workspace = true }
 arrow-schema = { workspace = true }
 arrow-array = { workspace = true}
 arrow-json = { workspace = true }
 tokio = { version = "1", features = ["full"] }
 tracing = "0.1"
 anyhow = "1"
-chrono = "0.4"
-bincode = "2.0.0-rc.3"
 memchr = "2"
 typify = "0.0.13"
 schemars = "0.8"
 prost = { workspace = true}
 prost-reflect = { workspace = true}
 prost-build = { workspace = true }
-prost-types = { workspace = true}
 base64 = "0.22.1"
 uuid = { version = "1.10.0", features = ["v4"] }
 regex = "1.10.6"

crates/arroyo-formats/src/avro/de.rs

@@ -98,7 +98,10 @@ pub(crate) fn avro_to_json(value: AvroValue) -> JsonValue {
         | Value::TimestampMillis(i)
         | Value::TimestampMicros(i)
         | Value::LocalTimestampMillis(i)
-        | Value::LocalTimestampMicros(i) => JsonValue::Number(serde_json::Number::from(i)),
+        | Value::LocalTimestampMicros(i)
+        | Value::TimestampNanos(i)
+        | Value::LocalTimestampNanos(i) => JsonValue::Number(serde_json::Number::from(i)),
+        Value::BigDecimal(d) => JsonValue::String(d.to_string()),
         Value::Float(f) => float_to_json(f as f64),
         Value::Double(f) => float_to_json(f),
         Value::String(s) | Value::Enum(_, s) => JsonValue::String(s),

crates/arroyo-formats/src/avro/ser.rs

@@ -172,7 +172,7 @@ fn serialize_column<T: SerializeTarget>(
             {
                 if let Some(v) = &mut v {
                     serialize_column(
-                        item_schema,
+                        &item_schema.items,
                         v,
                         "",
                         &column.expect("unmasked null in list"),

crates/arroyo-node/Cargo.toml

@@ -9,15 +9,10 @@ arroyo-rpc = { path = "../arroyo-rpc" }
 arroyo-server-common = { path = "../arroyo-server-common" }
 
 tonic = { workspace = true }
-tonic-reflection = { workspace = true }
-prost = {workspace = true}
 tokio = { version = "1", features = ["full"] }
 tracing = "0.1"
-fork = "0.1"
 rand = "0.8"
 local-ip-address = "0.5"
 lazy_static = "1.4.0"
 prometheus = "0.13.3"
-tokio-stream = "0.1.14"
-anyhow = "1.0.72"
-base64 = "0.22.0"
+anyhow = "1.0.72"

crates/arroyo-operator/Cargo.toml

@@ -18,7 +18,6 @@ arroyo-udf-python = { path = "../arroyo-udf/arroyo-udf-python" }
 
 anyhow = "1.0.71"
 arrow = { workspace = true, features = ["ffi"] }
-ahash = { workspace = true }
 async-trait = "0.1.68"
 bincode = "2.0.0-rc.3"
 datafusion = { workspace = true }
@@ -32,4 +31,3 @@ async-stream = "0.3.5"
 serde_json = "1.0.111"
 serde = "1.0.195"
 dlopen2 = "0.7.0"
-async-ffi = "0.5.0"