From 7745768b9a8ca9cb03e2618c0655fd4a28492099 Mon Sep 17 00:00:00 2001 From: getnamo Date: Tue, 24 Jul 2018 21:29:19 +0100 Subject: [PATCH] update TFAudioCapture - adds support for native c++ delegates - should capture the whole audio better when it finishes - returns average volume (as a fraction of int16) with each data chunk --- .../Private/AudioCaptureData.cpp | 8 +-- .../Private/AudioCapturePrivatePCH.h | 4 -- .../Private/FTFAudioCapture.cpp | 45 ++++++++----- .../Private/FWindowsAudioCapture.cpp | 65 ++++++++++++++++--- .../Private/FWindowsAudioCapture.h | 4 +- .../Private/IAudioCaptureInterface.h | 2 +- .../Private/IAudioDataInterface.h | 14 ++++ .../Private/TFAudioCaptureComponent.cpp | 10 ++- .../TFAudioCapture/Public/ITFAudioCapture.h | 7 +- .../Public/TFAudioCaptureComponent.h | 11 +++- Source/TFAudioCapture/TFAudioCapture.Build.cs | 3 +- 11 files changed, 132 insertions(+), 41 deletions(-) delete mode 100644 Source/TFAudioCapture/Private/AudioCapturePrivatePCH.h create mode 100644 Source/TFAudioCapture/Private/IAudioDataInterface.h diff --git a/Source/TFAudioCapture/Private/AudioCaptureData.cpp b/Source/TFAudioCapture/Private/AudioCaptureData.cpp index cf3402e..baee4fb 100644 --- a/Source/TFAudioCapture/Private/AudioCaptureData.cpp +++ b/Source/TFAudioCapture/Private/AudioCaptureData.cpp @@ -1,9 +1,9 @@ -#include "AudioCapturePrivatePCH.h" #include "AudioCaptureData.h" FAudioCaptureOptions::FAudioCaptureOptions() { - SampleRate = 22050; + //this is deep-speech default + SampleRate = 16000; BitsPerSample = 16; Channels = 1; } @@ -17,8 +17,8 @@ FWavHeader::FWavHeader() memcpy((char*)Subchunk2ID, "data", 4); Subchunk1Size = 16; - //Default to 44khz 16 bit PCM Mono - SetHeaderSampleData(44100, 16); + //Default to 16khz 16 bit PCM Mono + SetHeaderSampleData(16000, 16); AudioFormat = 1; //PCM } diff --git a/Source/TFAudioCapture/Private/AudioCapturePrivatePCH.h b/Source/TFAudioCapture/Private/AudioCapturePrivatePCH.h deleted file mode 100644 index dcc74a6..0000000 --- a/Source/TFAudioCapture/Private/AudioCapturePrivatePCH.h +++ /dev/null @@ -1,4 +0,0 @@ -#pragma once - -#include "CoreUObject.h" -#include "Engine.h" \ No newline at end of file diff --git a/Source/TFAudioCapture/Private/FTFAudioCapture.cpp b/Source/TFAudioCapture/Private/FTFAudioCapture.cpp index e1921e4..4370f67 100644 --- a/Source/TFAudioCapture/Private/FTFAudioCapture.cpp +++ b/Source/TFAudioCapture/Private/FTFAudioCapture.cpp @@ -1,4 +1,3 @@ -#include "AudioCapturePrivatePCH.h" #include "ITFAudioCapture.h" #include "TFAudioCaptureComponent.h" #include "LambdaRunnable.h" @@ -8,12 +7,14 @@ class FTFAudioCapture : public ITFAudioCapture { public: - virtual void StartCapture(TFunction&)> OnAudioData = nullptr, TFunction&)> OnCaptureFinished = nullptr) override; + virtual void StartCapture(TFunction&, float)> OnAudioData = nullptr, TFunction&, float)> OnCaptureFinished = nullptr) override; virtual void StopCapture() override; virtual void SetOptions(const FAudioCaptureOptions& Options) override; virtual void AddAudioComponent(const UTFAudioCaptureComponent* Component) override; virtual void RemoveAudioComponent(const UTFAudioCaptureComponent* Component) override; + virtual void AddAudioDelegate(IAudioDataInterface* Delegate) override; + virtual void RemoveAudioDelegate(IAudioDataInterface* Delegate) override; /** IModuleInterface implementation */ virtual void StartupModule() override; @@ -21,7 +22,7 @@ class FTFAudioCapture : public ITFAudioCapture private: TSharedPtr WindowsCapture; - TArray Components; + TArray Delegates; }; void FTFAudioCapture::StartupModule() @@ -37,42 +38,41 @@ void FTFAudioCapture::ShutdownModule() } -void FTFAudioCapture::StartCapture(TFunction&)> OnAudioData, TFunction&)> OnCaptureFinished) +void FTFAudioCapture::StartCapture(TFunction&, float)> OnAudioData, TFunction&, float)> OnCaptureFinished) { - TFunction&)> OnDataDelegate = [this, OnAudioData] (const TArray& AudioData) + TFunction&, float)> OnDataDelegate = [this, OnAudioData] (const TArray& AudioData, float AudioMaxLevel) { //Call each added component function inside game thread - FLambdaRunnable::RunShortLambdaOnGameThread([this, AudioData, OnAudioData] + FLambdaRunnable::RunShortLambdaOnGameThread([this, AudioData, OnAudioData, AudioMaxLevel] { - for (auto Component : Components) + for (auto Delegate : Delegates) { - Component->OnAudioData.Broadcast(AudioData); + Delegate->OnAudioDataEvent(AudioData, AudioMaxLevel); } //Also if valid pass it to the new delegate if (OnAudioData != nullptr) { - OnAudioData(AudioData); + OnAudioData(AudioData, AudioMaxLevel); } }); }; - TFunction&)> OnFinishedDelegate = [this, OnCaptureFinished](const TArray& AudioData) + TFunction&, float)> OnFinishedDelegate = [this, OnCaptureFinished](const TArray& AudioData, float AudioMaxLevel) { //Call each added component function inside game thread - FLambdaRunnable::RunShortLambdaOnGameThread([this, AudioData, OnCaptureFinished] + FLambdaRunnable::RunShortLambdaOnGameThread([this, AudioData, OnCaptureFinished, AudioMaxLevel] { - for (auto Component : Components) + for (auto Delegate : Delegates) { - Component->OnCaptureFinished.Broadcast(AudioData); + Delegate->OnCaptureFinishedEvent(AudioData, AudioMaxLevel); } //Also if valid pass it to the new delegate if (OnCaptureFinished != nullptr) { - OnCaptureFinished(AudioData); + OnCaptureFinished(AudioData, AudioMaxLevel); } - }); }; @@ -94,14 +94,25 @@ void FTFAudioCapture::SetOptions(const FAudioCaptureOptions& Options) void FTFAudioCapture::AddAudioComponent(const UTFAudioCaptureComponent* Component) { - Components.Add((UTFAudioCaptureComponent*)Component); + AddAudioDelegate((IAudioDataInterface*)Component); } void FTFAudioCapture::RemoveAudioComponent(const UTFAudioCaptureComponent* Component) { - Components.Remove((UTFAudioCaptureComponent*)Component); + RemoveAudioDelegate((IAudioDataInterface*)Component); +} + +void FTFAudioCapture::AddAudioDelegate(IAudioDataInterface* Delegate) +{ + Delegates.Add(Delegate); } +void FTFAudioCapture::RemoveAudioDelegate(IAudioDataInterface* Delegate) +{ + Delegates.Remove(Delegate); +} + + IMPLEMENT_MODULE(FTFAudioCapture, TFAudioCapture) diff --git a/Source/TFAudioCapture/Private/FWindowsAudioCapture.cpp b/Source/TFAudioCapture/Private/FWindowsAudioCapture.cpp index 90e3279..63380bd 100644 --- a/Source/TFAudioCapture/Private/FWindowsAudioCapture.cpp +++ b/Source/TFAudioCapture/Private/FWindowsAudioCapture.cpp @@ -1,6 +1,6 @@ -#include "AudioCapturePrivatePCH.h" -#include "LambdaRunnable.h" #include "FWindowsAudioCapture.h" +#include "LambdaRunnable.h" +#include "CoreMinimal.h" #include "AllowWindowsPlatformTypes.h" #include @@ -15,7 +15,7 @@ FWindowsAudioCapture::FWindowsAudioCapture() bRunLoopActive = false; } -void FWindowsAudioCapture::StartCapture(TFunction&)> OnAudioData /*= nullptr*/, TFunction&)> OnCaptureFinished /*= nullptr*/) +void FWindowsAudioCapture::StartCapture(TFunction&, float)> OnAudioData /*= nullptr*/, TFunction&, float)> OnCaptureFinished /*= nullptr*/) { //Only attempt to start capture once. If it's active return. if (bRunLoopActive) @@ -58,21 +58,24 @@ void FWindowsAudioCapture::StartCapture(TFunction&)> On // Insert a wave input buffer result = waveInAddBuffer(hWaveIn, &hWaveInHdr, sizeof(WAVEHDR)); - result = waveInStart(hWaveIn); + float MaxLevel = 0.f; + //The headers will now get filled and we should check them periodically for new data while (*bShouldRunPtr) { if (hWaveInHdr.dwFlags & WHDR_DONE) { TArray OutData; - OutData.SetNum(AudioBuffer.Num()); - FMemory::Memcpy(OutData.GetData(), AudioBuffer.GetData(), AudioBuffer.Num()); + OutData.SetNum(hWaveInHdr.dwBytesRecorded); + FMemory::Memcpy(OutData.GetData(), AudioBuffer.GetData(), hWaveInHdr.dwBytesRecorded); + + MaxLevel = CalculateMaxAudioLevel(OutData, pFormat.wBitsPerSample); if (OnAudioData != nullptr) { - OnAudioData(OutData); + OnAudioData(OutData, MaxLevel); } //Clear flags @@ -85,14 +88,34 @@ void FWindowsAudioCapture::StartCapture(TFunction&)> On } } + //Stop waveInStop(hWaveIn); + + //Clear flags + hWaveInHdr.dwFlags = 0; + hWaveInHdr.dwBytesRecorded = 0; + + //Flush last data + waveInPrepareHeader(hWaveIn, &hWaveInHdr, sizeof(WAVEHDR)); + waveInAddBuffer(hWaveIn, &hWaveInHdr, sizeof(WAVEHDR)); + + TArray OutData; + OutData.SetNum(hWaveInHdr.dwBytesRecorded); + FMemory::Memcpy(OutData.GetData(), AudioBuffer.GetData(), hWaveInHdr.dwBytesRecorded); + MaxLevel = CalculateMaxAudioLevel(OutData, pFormat.wBitsPerSample); + waveInUnprepareHeader(hWaveIn, &hWaveInHdr, sizeof(WAVEHDR)); waveInClose(hWaveIn); + if (OnAudioData != nullptr) + { + OnAudioData(OutData, MaxLevel); + } + + //flush whatever is left of the buffer, emit both in data and on finished if (OnCaptureFinished != nullptr) { - //flush whatever is left of the buffer - OnCaptureFinished(AudioBuffer); + OnCaptureFinished(OutData, MaxLevel); } }); } @@ -107,4 +130,28 @@ void FWindowsAudioCapture::SetOptions(const FAudioCaptureOptions& InOptions) Options = InOptions; } +float FWindowsAudioCapture::CalculateMaxAudioLevel(TArray& Buffer, int32 BitsPerSample) +{ + //Can't handle non-16 sample + if(BitsPerSample != 16) + { + return 0.5f; + } + + int32 Num = (Buffer.Num() / BitsPerSample) * 8; + int16* Buffer16 = (int16*)Buffer.GetData(); + int16 MaxValue = 0; + for (int i = 0; i < Num; i++) + { + int16 Value = FMath::Abs(Buffer16[i]); + if (Value > MaxValue) + { + MaxValue = Value; + } + } + //UE_LOG(LogTemp, Log, TEXT("max value: %d"), MaxValue); + + return (float)MaxValue / (float)INT16_MAX; +} + #include "HideWindowsPlatformTypes.h" \ No newline at end of file diff --git a/Source/TFAudioCapture/Private/FWindowsAudioCapture.h b/Source/TFAudioCapture/Private/FWindowsAudioCapture.h index 39f57dd..cf8433e 100644 --- a/Source/TFAudioCapture/Private/FWindowsAudioCapture.h +++ b/Source/TFAudioCapture/Private/FWindowsAudioCapture.h @@ -7,11 +7,13 @@ class FWindowsAudioCapture : public IAudioCaptureInterface public: FWindowsAudioCapture(); - virtual void StartCapture(TFunction&)> OnAudioData = nullptr, TFunction&)> OnCaptureFinished = nullptr) override; + virtual void StartCapture(TFunction&, float)> OnAudioData = nullptr, TFunction&, float)> OnCaptureFinished = nullptr) override; virtual void StopCapture() override; virtual void SetOptions(const FAudioCaptureOptions& InOptions) override; FAudioCaptureOptions Options; private: + + float CalculateMaxAudioLevel(TArray& Buffer, int32 BitsPerSample); FThreadSafeBool bRunLoopActive; }; \ No newline at end of file diff --git a/Source/TFAudioCapture/Private/IAudioCaptureInterface.h b/Source/TFAudioCapture/Private/IAudioCaptureInterface.h index 5a85eef..2342345 100644 --- a/Source/TFAudioCapture/Private/IAudioCaptureInterface.h +++ b/Source/TFAudioCapture/Private/IAudioCaptureInterface.h @@ -7,7 +7,7 @@ class IAudioCaptureInterface /** Required API */ /** Start capturing audio, whenever there is audio data, call passed-in lambda */ - virtual void StartCapture(TFunction&)> OnAudioData = nullptr, TFunction&)> OnCaptureFinished = nullptr) {}; + virtual void StartCapture(TFunction&, float)> OnAudioData = nullptr, TFunction&, float)> OnCaptureFinished = nullptr) {}; /** Stop the audio capture and cleanup */ virtual void StopCapture() {}; diff --git a/Source/TFAudioCapture/Private/IAudioDataInterface.h b/Source/TFAudioCapture/Private/IAudioDataInterface.h new file mode 100644 index 0000000..52437f8 --- /dev/null +++ b/Source/TFAudioCapture/Private/IAudioDataInterface.h @@ -0,0 +1,14 @@ +#pragma once + +//Native interface to receive callbacks from audio capture +class IAudioDataInterface +{ +public: + /** Required API */ + + /** When data gets streamed */ + virtual void OnAudioDataEvent(const TArray& Bytes, float MaxLevel) {}; + + /** When capture is complete */ + virtual void OnCaptureFinishedEvent(const TArray& Bytes, float MaxLevel) {}; +}; \ No newline at end of file diff --git a/Source/TFAudioCapture/Private/TFAudioCaptureComponent.cpp b/Source/TFAudioCapture/Private/TFAudioCaptureComponent.cpp index 9840a58..26791c4 100644 --- a/Source/TFAudioCapture/Private/TFAudioCaptureComponent.cpp +++ b/Source/TFAudioCapture/Private/TFAudioCaptureComponent.cpp @@ -1,4 +1,3 @@ -#include "AudioCapturePrivatePCH.h" #include "TFAudioCaptureComponent.h" #include "ITFAudioCapture.h" @@ -67,3 +66,12 @@ void UTFAudioCaptureComponent::UninitializeComponent() Super::UninitializeComponent(); } + +void UTFAudioCaptureComponent::OnAudioDataEvent(const TArray& Bytes, float MaxLevel) +{ + OnAudioData.Broadcast(Bytes, MaxLevel); +} +void UTFAudioCaptureComponent::OnCaptureFinishedEvent(const TArray& Bytes, float MaxLevel) +{ + OnCaptureFinished.Broadcast(Bytes, MaxLevel); +} \ No newline at end of file diff --git a/Source/TFAudioCapture/Public/ITFAudioCapture.h b/Source/TFAudioCapture/Public/ITFAudioCapture.h index be40631..92a598c 100644 --- a/Source/TFAudioCapture/Public/ITFAudioCapture.h +++ b/Source/TFAudioCapture/Public/ITFAudioCapture.h @@ -1,6 +1,7 @@ #pragma once #include "IAudioCaptureInterface.h" +#include "IAudioDataInterface.h" #include "ModuleManager.h" class UTFAudioCaptureComponent; @@ -30,7 +31,11 @@ class TFAUDIOCAPTURE_API ITFAudioCapture : public IModuleInterface, public IAudi return FModuleManager::Get().IsModuleLoaded("TFAudioCapture"); } + //For components virtual void AddAudioComponent(const UTFAudioCaptureComponent* Component) {}; - virtual void RemoveAudioComponent(const UTFAudioCaptureComponent* Component) {}; + + //for native delegates + virtual void AddAudioDelegate(IAudioDataInterface* Delegate) {}; + virtual void RemoveAudioDelegate(IAudioDataInterface* Delegate) {}; }; \ No newline at end of file diff --git a/Source/TFAudioCapture/Public/TFAudioCaptureComponent.h b/Source/TFAudioCapture/Public/TFAudioCaptureComponent.h index a9f02a9..2979110 100644 --- a/Source/TFAudioCapture/Public/TFAudioCaptureComponent.h +++ b/Source/TFAudioCapture/Public/TFAudioCaptureComponent.h @@ -2,15 +2,16 @@ #include "Components/ActorComponent.h" #include "AudioCaptureData.h" +#include "IAudioDataInterface.h" #include "TFAudioCaptureComponent.generated.h" -DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FAudioDataSignature, const TArray&, Bytes); +DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FAudioDataSignature, const TArray&, Bytes, float, MaxLevel); /** * Component used to capture microphone audio and emit bytes as the data streams in */ UCLASS(ClassGroup = "Sound", meta = (BlueprintSpawnableComponent)) -class TFAUDIOCAPTURE_API UTFAudioCaptureComponent : public UActorComponent +class TFAUDIOCAPTURE_API UTFAudioCaptureComponent : public UActorComponent, public IAudioDataInterface { GENERATED_UCLASS_BODY() @@ -44,6 +45,12 @@ class TFAUDIOCAPTURE_API UTFAudioCaptureComponent : public UActorComponent UFUNCTION(BlueprintCallable, Category = "Utilities|Audio Capture") void ConvertWavToRaw(const TArray& InBytes, TArray& OutBytes, FAudioCaptureOptions& OutOptions); + //Start IAudioDataInterface + virtual void OnAudioDataEvent(const TArray& Bytes, float MaxLevel) override; + virtual void OnCaptureFinishedEvent(const TArray& Bytes, float MaxLevel) override; + //End IAudioDataInterface + + virtual void InitializeComponent() override; virtual void UninitializeComponent() override; diff --git a/Source/TFAudioCapture/TFAudioCapture.Build.cs b/Source/TFAudioCapture/TFAudioCapture.Build.cs index 8f06682..f18d562 100644 --- a/Source/TFAudioCapture/TFAudioCapture.Build.cs +++ b/Source/TFAudioCapture/TFAudioCapture.Build.cs @@ -4,8 +4,9 @@ public class TFAudioCapture : ModuleRules { public TFAudioCapture(ReadOnlyTargetRules Target) : base(Target) { + PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs; - PublicIncludePaths.AddRange( + PublicIncludePaths.AddRange( new string[] { "TFAudioCapture/Public" }