diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0cbbd5739..4339d6233 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,7 @@ jobs: - Windows - driver only - gateway only + - simd json include: - name: beta @@ -55,6 +56,10 @@ jobs: - name: gateway only features: serenity-rustls dont-test: true + - name: simd json + features: simd-json serenity-rustls driver gateway serenity/simd_json + rustflags: -C target-cpu=native + dont-test: true steps: - name: Checkout sources @@ -94,6 +99,10 @@ jobs: target key: ${{ runner.os }}-test-${{ steps.tc.outputs.rustc_hash }}-${{ hashFiles('**/Cargo.toml') }} + - name: Set RUSTFLAGS + if: runner.os != 'Windows' + run: echo "RUSTFLAGS=${{ matrix.rustflags || '' }}" >> $GITHUB_ENV + - name: Build all features if: matrix.features == '' run: cargo build --features full-doc diff --git a/Cargo.toml b/Cargo.toml index 56e2fba71..3448d6e91 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,6 +104,11 @@ features = ["voice", "gateway"] optional = true version = "0.1" +[dependencies.simd-json] +optional = true +features = ["serde_impl"] +version = "0.6.0" + [dependencies.streamcatcher] optional = true version = "1" @@ -225,6 +230,8 @@ zlib-simd = ["twilight-gateway/zlib-simd"] zlib-stock = ["twilight-gateway/zlib-stock"] serenity-deps = ["async-trait"] +simdjson = [] + rustls-marker = [] native-marker = [] diff --git a/Makefile.toml b/Makefile.toml index 7e0e9581f..881881917 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -8,6 +8,12 @@ args = ["fmt", "--all"] args = ["build", "--features", "full-doc"] dependencies = ["format"] +[tasks.build-simd] +args = ["build", "--features", "full-doc,simd-json,serenity/simd_json,twilight-gateway/simd-json"] +command = "cargo" +dependencies = ["format"] +env = { "RUSTFLAGS" = "-C target-cpu=native" } + [tasks.build-examples] args = ["build", "--manifest-path", "./examples/Cargo.toml", "--workspace"] command = "cargo" @@ -24,7 +30,7 @@ command = "cargo" dependencies = ["format"] [tasks.build-variants] -dependencies = ["build", "build-gateway", "build-driver"] +dependencies = ["build", "build-gateway", "build-driver", "build-simd"] [tasks.check] args = ["check", "--features", "full-doc"] @@ -37,6 +43,11 @@ dependencies = ["format"] [tasks.test] args = ["test", "--features", "full-doc"] +[tasks.test-simd] +args = ["test", "--features", "full-doc,simd-json,serenity/simd_json,twilight-gateway/simd-json"] +command = "cargo" +env = { "RUSTFLAGS" = "-C target-cpu=native" } + [tasks.bench] description = "Runs performance benchmarks." category = "Test" diff --git a/README.md b/README.md index bd54340cd..fab1c349e 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ The library offers: * And, by default, a fully featured voice system featuring events, queues, RT(C)P packet handling, seeking on compatible streams, shared multithreaded audio stream caches, and direct Opus data passthrough from DCA files. + * To be able to use `simd-json` from serenity, you will need to enable the `simdjson` + feature on both songbird and serenity. ## Intents Songbird's gateway functionality requires you to specify the `GUILD_VOICE_STATES` intent. diff --git a/src/error.rs b/src/error.rs index 2691f4e93..17769224a 100644 --- a/src/error.rs +++ b/src/error.rs @@ -2,8 +2,12 @@ #[cfg(feature = "serenity")] use futures::channel::mpsc::TrySendError; +#[cfg(not(feature = "simd-json"))] +pub use serde_json::Error as JsonError; #[cfg(feature = "serenity")] use serenity::gateway::InterMessage; +#[cfg(feature = "simd-json")] +pub use simd_json::Error as JsonError; #[cfg(feature = "gateway")] use std::{error::Error, fmt}; #[cfg(feature = "twilight")] diff --git a/src/input/adapters/cached/compressed.rs b/src/input/adapters/cached/compressed.rs index c15073cc9..ac8b8996f 100644 --- a/src/input/adapters/cached/compressed.rs +++ b/src/input/adapters/cached/compressed.rs @@ -199,7 +199,7 @@ impl Compressed { )?; let mut metabytes = b"DCA1\0\0\0\0".to_vec(); let orig_len = metabytes.len(); - serde_json::to_writer(&mut metabytes, &metadata)?; + crate::json::to_writer(&mut metabytes, &metadata)?; let meta_len = (metabytes.len() - orig_len) .try_into() .map_err(|_| CodecCacheError::MetadataTooLarge)?; diff --git a/src/input/adapters/cached/error.rs b/src/input/adapters/cached/error.rs index e5892fb3c..4cc63bbe9 100644 --- a/src/input/adapters/cached/error.rs +++ b/src/input/adapters/cached/error.rs @@ -1,6 +1,5 @@ -use crate::input::AudioStreamError; +use crate::{error::JsonError, input::AudioStreamError}; use audiopus::error::Error as OpusError; -use serde_json::Error as JsonError; use std::{ error::Error as StdError, fmt::{Display, Formatter, Result as FmtResult}, diff --git a/src/input/codecs/dca/mod.rs b/src/input/codecs/dca/mod.rs index 826875098..45fcf0757 100644 --- a/src/input/codecs/dca/mod.rs +++ b/src/input/codecs/dca/mod.rs @@ -110,9 +110,11 @@ impl FormatReader for DcaReader { return symph_err::decode_error("missing DCA1 metadata block"); } - let raw_json = source.read_boxed_slice_exact(size as usize)?; + let mut raw_json = source.read_boxed_slice_exact(size as usize)?; - let metadata: DcaMetadata = serde_json::from_slice::(&raw_json) + // NOTE: must be mut for simd-json. + #[allow(clippy::unnecessary_mut_passed)] + let metadata: DcaMetadata = crate::json::from_slice::(&mut raw_json) .map_err(|_| SymphError::DecodeError("malformed DCA1 metadata block"))?; let mut revision = MetadataBuilder::new(); diff --git a/src/input/metadata/mod.rs b/src/input/metadata/mod.rs index 916ac610f..e27fd1a25 100644 --- a/src/input/metadata/mod.rs +++ b/src/input/metadata/mod.rs @@ -1,3 +1,4 @@ +use crate::error::JsonError; use std::time::Duration; use symphonia_core::{meta::Metadata as ContainerMetadata, probe::ProbedMetadata}; @@ -47,8 +48,8 @@ pub struct AuxMetadata { impl AuxMetadata { /// Extract metadata and details from the output of `ffprobe -of json`. - pub fn from_ffprobe_json(value: &[u8]) -> Result { - let output: ffprobe::Output = serde_json::from_slice(value)?; + pub fn from_ffprobe_json(value: &mut [u8]) -> Result { + let output: ffprobe::Output = crate::json::from_slice(value)?; Ok(output.into_aux_metadata()) } diff --git a/src/input/sources/file.rs b/src/input/sources/file.rs index 756a81688..5597f1f5c 100644 --- a/src/input/sources/file.rs +++ b/src/input/sources/file.rs @@ -68,13 +68,13 @@ impl + Send + Sync> Compose for File

{ "-i", ]; - let output = Command::new("ffprobe") + let mut output = Command::new("ffprobe") .args(&args) .output() .await .map_err(|e| AudioStreamError::Fail(Box::new(e)))?; - AuxMetadata::from_ffprobe_json(&output.stdout[..]) + AuxMetadata::from_ffprobe_json(&mut output.stdout[..]) .map_err(|e| AudioStreamError::Fail(Box::new(e))) } } diff --git a/src/input/sources/ytdl.rs b/src/input/sources/ytdl.rs index c79623567..6f032ffde 100644 --- a/src/input/sources/ytdl.rs +++ b/src/input/sources/ytdl.rs @@ -59,13 +59,15 @@ impl YoutubeDl { async fn query(&mut self) -> Result { let ytdl_args = ["-j", &self.url, "-f", "ba[abr>0][vcodec=none]/best"]; - let output = Command::new(self.program) + let mut output = Command::new(self.program) .args(&ytdl_args) .output() .await .map_err(|e| AudioStreamError::Fail(Box::new(e)))?; - let stdout: Output = serde_json::from_slice(&output.stdout[..]) + // NOTE: must be mut for simd-json. + #[allow(clippy::unnecessary_mut_passed)] + let stdout: Output = crate::json::from_slice(&mut output.stdout[..]) .map_err(|e| AudioStreamError::Fail(Box::new(e)))?; self.metadata = Some(stdout.as_aux_metadata()); diff --git a/src/lib.rs b/src/lib.rs index 50fc45ec8..18461bcf4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -107,6 +107,12 @@ pub use serenity_voice_model as model; #[cfg(feature = "driver")] pub use typemap_rev as typemap; +// Re-export serde-json APIs locally to minimise conditional config elsewhere. +#[cfg(not(feature = "simd-json"))] +pub(crate) use serde_json as json; +#[cfg(feature = "simd-json")] +pub(crate) use simd_json::serde as json; + #[cfg(feature = "driver")] pub use crate::{ driver::Driver, diff --git a/src/ws.rs b/src/ws.rs index 88e47be4b..f47e1a95f 100644 --- a/src/ws.rs +++ b/src/ws.rs @@ -1,4 +1,4 @@ -use crate::model::Event; +use crate::{error::JsonError, model::Event}; use async_trait::async_trait; use async_tungstenite::{ @@ -8,7 +8,6 @@ use async_tungstenite::{ WebSocketStream, }; use futures::{SinkExt, StreamExt, TryStreamExt}; -use serde_json::Error as JsonError; use tokio::time::{timeout, Duration}; use tracing::instrument; @@ -92,7 +91,7 @@ impl ReceiverExt for WsStream { #[async_trait] impl SenderExt for SplitSink { async fn send_json(&mut self, value: &Event) -> Result<()> { - Ok(serde_json::to_string(value) + Ok(crate::json::to_string(value) .map(Message::Text) .map_err(Error::from) .map(|m| self.send(m))? @@ -103,7 +102,7 @@ impl SenderExt for SplitSink { #[async_trait] impl SenderExt for WsStream { async fn send_json(&mut self, value: &Event) -> Result<()> { - Ok(serde_json::to_string(value) + Ok(crate::json::to_string(value) .map(Message::Text) .map_err(Error::from) .map(|m| self.send(m))? @@ -114,7 +113,8 @@ impl SenderExt for WsStream { #[inline] pub(crate) fn convert_ws_message(message: Option) -> Result> { Ok(match message { - Some(Message::Text(payload)) => serde_json::from_str(&payload).map(Some)?, + Some(Message::Text(mut payload)) => + crate::json::from_str(payload.as_mut_str()).map(Some)?, Some(Message::Binary(bytes)) => { return Err(Error::UnexpectedBinaryMessage(bytes)); },