diff --git a/include/algorithms/public/Envelope.hpp b/include/algorithms/public/Envelope.hpp new file mode 100644 index 000000000..c625ef15f --- /dev/null +++ b/include/algorithms/public/Envelope.hpp @@ -0,0 +1,83 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright 2017-2019 University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ + +#pragma once + +#include "../util/ButterworthHPFilter.hpp" +#include "../util/FluidEigenMappings.hpp" +#include "../util/SlideUDFilter.hpp" +#include "../../data/FluidIndex.hpp" +#include "../../data/TensorTypes.hpp" +#include +#include + +namespace fluid { +namespace algorithm { + +class Envelope +{ + + using ArrayXd = Eigen::ArrayXd; + +public: + void init(double floor, double hiPassFreq) + { + mFastSlide.init(floor); + mSlowSlide.init(floor); + initFilters(hiPassFreq); + mHiPassFreq = hiPassFreq; + mInitialized = true; + } + + double processSample(const double in, + double floor, index fastRampUpTime, index slowRampUpTime, + index fastRampDownTime, index slowRampDownTime, + double hiPassFreq) + { + using namespace std; + assert(mInitialized); + mFastSlide.updateCoeffs(fastRampUpTime, fastRampDownTime); + mSlowSlide.updateCoeffs(slowRampUpTime, slowRampDownTime); + double filtered = in; + if (hiPassFreq != mHiPassFreq) + { + initFilters(hiPassFreq); + mHiPassFreq = hiPassFreq; + } + if (mHiPassFreq > 0){ + filtered = mHiPass2.processSample(mHiPass1.processSample(in)); + } + double rectified = abs(filtered); + double dB = 20 * log10(rectified); + double clipped = max(dB, floor); + double fast = mFastSlide.processSample(clipped); + double slow = mSlowSlide.processSample(clipped); + return fast - slow; + } + + bool initialized() { return mInitialized; } + +private: + void initFilters(double cutoff) + { + mHiPass1.init(cutoff); + mHiPass2.init(cutoff); + } + + double mHiPassFreq{0}; + bool mInitialized{false}; + + ButterworthHPFilter mHiPass1; + ButterworthHPFilter mHiPass2; + SlideUDFilter mFastSlide; + SlideUDFilter mSlowSlide; +}; +} // namespace algorithm +} // namespace fluid diff --git a/include/algorithms/public/EnvelopeSegmentation.hpp b/include/algorithms/public/EnvelopeSegmentation.hpp index 50c50c513..60495008a 100644 --- a/include/algorithms/public/EnvelopeSegmentation.hpp +++ b/include/algorithms/public/EnvelopeSegmentation.hpp @@ -10,6 +10,7 @@ under the European Union’s Horizon 2020 research and innovation programme #pragma once +#include "Envelope.hpp" #include "../util/ButterworthHPFilter.hpp" #include "../util/FluidEigenMappings.hpp" #include "../util/SlideUDFilter.hpp" @@ -29,14 +30,15 @@ class EnvelopeSegmentation public: void init(double floor, double hiPassFreq) { - mFastSlide.init(floor); - mSlowSlide.init(floor); + // mFastSlide.init(floor); + // mSlowSlide.init(floor); + mEnvelope.init(floor,hiPassFreq); mDebounceCount = 1; - initFilters(hiPassFreq); - mHiPassFreq = hiPassFreq; + // initFilters(hiPassFreq); + // mHiPassFreq = hiPassFreq; mPrevValue = 0; mState = false; - mInitialized = true; + // mInitialized = true; } double processSample(const double in, double onThreshold, double offThreshold, @@ -44,25 +46,30 @@ class EnvelopeSegmentation index fastRampDownTime, index slowRampDownTime, double hiPassFreq, index debounce) { - using namespace std; - assert(mInitialized); - mFastSlide.updateCoeffs(fastRampUpTime, fastRampDownTime); - mSlowSlide.updateCoeffs(slowRampUpTime, slowRampDownTime); - double filtered = in; - if (hiPassFreq != mHiPassFreq) - { - initFilters(hiPassFreq); - mHiPassFreq = hiPassFreq; - } - if (mHiPassFreq > 0){ - filtered = mHiPass2.processSample(mHiPass1.processSample(in)); - } - double rectified = abs(filtered); - double dB = 20 * log10(rectified); - double clipped = max(dB, floor); - double fast = mFastSlide.processSample(clipped); - double slow = mSlowSlide.processSample(clipped); - double value = fast - slow; + // using namespace std; + // assert(mInitialized); + // mFastSlide.updateCoeffs(fastRampUpTime, fastRampDownTime); + // mSlowSlide.updateCoeffs(slowRampUpTime, slowRampDownTime); + // double filtered = in; + // if (hiPassFreq != mHiPassFreq) + // { + // initFilters(hiPassFreq); + // mHiPassFreq = hiPassFreq; + // } + // if (mHiPassFreq > 0){ + // filtered = mHiPass2.processSample(mHiPass1.processSample(in)); + // } + // double rectified = abs(filtered); + // double dB = 20 * log10(rectified); + // double clipped = max(dB, floor); + // double fast = mFastSlide.processSample(clipped); + // double slow = mSlowSlide.processSample(clipped); + // double value = fast - slow; + + + double value = + mEnvelope.processSample(in, floor, fastRampUpTime, slowRampUpTime, + fastRampDownTime, slowRampUpTime, hiPassFreq); double detected = 0; if (!mState && value > onThreshold && mPrevValue < onThreshold && @@ -81,25 +88,26 @@ class EnvelopeSegmentation return detected; } - bool initialized() { return mInitialized; } + bool initialized() { return mEnvelope.initialized(); } private: - void initFilters(double cutoff) - { - mHiPass1.init(cutoff); - mHiPass2.init(cutoff); - } - - double mHiPassFreq{0}; +// void initFilters(double cutoff) +// { +// mHiPass1.init(cutoff); +// mHiPass2.init(cutoff); +// } + + Envelope mEnvelope; + // double mHiPassFreq{0}; index mDebounceCount{1}; double mPrevValue{0}; - bool mInitialized{false}; + // bool mInitialized{false}; bool mState{false}; - ButterworthHPFilter mHiPass1; - ButterworthHPFilter mHiPass2; - SlideUDFilter mFastSlide; - SlideUDFilter mSlowSlide; + // ButterworthHPFilter mHiPass1; + // ButterworthHPFilter mHiPass2; + // SlideUDFilter mFastSlide; + // SlideUDFilter mSlowSlide; }; } // namespace algorithm } // namespace fluid diff --git a/include/algorithms/public/NoveltyFeature.hpp b/include/algorithms/public/NoveltyFeature.hpp new file mode 100644 index 000000000..6590c9631 --- /dev/null +++ b/include/algorithms/public/NoveltyFeature.hpp @@ -0,0 +1,62 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright 2017-2019 University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ + +#pragma once + +#include "../util/FluidEigenMappings.hpp" +#include "../util/Novelty.hpp" +#include "../../data/FluidIndex.hpp" +#include "../../data/TensorTypes.hpp" +#include + +namespace fluid { +namespace algorithm { + +class NoveltyFeature +{ + +public: + using ArrayXd = Eigen::ArrayXd; + + NoveltyFeature(index maxKernelSize, index maxFilterSize) + : mFilterBufferStorage(maxFilterSize), mNovelty(maxKernelSize) + {} + + void init(index kernelSize, index filterSize, index nDims) + { + assert(kernelSize % 2); + mNovelty.init(kernelSize, nDims); + mFilterBuffer = mFilterBufferStorage.segment(0, filterSize); + mFilterBuffer.setZero(); + } + + double processFrame(const RealVectorView input) + { + double novelty = mNovelty.processFrame(_impl::asEigen(input)); + index filterSize = mFilterBuffer.size(); + + if (filterSize > 1) + { + mFilterBuffer.segment(0, filterSize - 1) = + mFilterBuffer.segment(1, filterSize - 1); + } + + mFilterBuffer(filterSize - 1) = novelty; + + return mFilterBuffer.mean(); + } + +private: + ArrayXd mFilterBuffer; + ArrayXd mFilterBufferStorage; + Novelty mNovelty; +}; +} // namespace algorithm +} // namespace fluid diff --git a/include/algorithms/public/NoveltySegmentation.hpp b/include/algorithms/public/NoveltySegmentation.hpp index 52457e31a..904c0c6cf 100644 --- a/include/algorithms/public/NoveltySegmentation.hpp +++ b/include/algorithms/public/NoveltySegmentation.hpp @@ -10,8 +10,8 @@ under the European Union’s Horizon 2020 research and innovation programme #pragma once +#include "NoveltyFeature.hpp" #include "../util/FluidEigenMappings.hpp" -#include "../util/Novelty.hpp" #include "../../data/FluidIndex.hpp" #include "../../data/TensorTypes.hpp" #include @@ -26,32 +26,23 @@ class NoveltySegmentation using ArrayXd = Eigen::ArrayXd; NoveltySegmentation(index maxKernelSize, index maxFilterSize) - : mFilterBufferStorage(maxFilterSize), mNovelty(maxKernelSize) + : mNovelty(maxFilterSize, maxKernelSize) {} void init(index kernelSize, index filterSize, index nDims) { - assert(kernelSize % 2); - mNovelty.init(kernelSize, nDims); - mFilterBuffer = mFilterBufferStorage.segment(0, filterSize); - mFilterBuffer.setZero(); + mNovelty.init(kernelSize, filterSize, nDims); mDebounceCount = 1; } double processFrame(const RealVectorView input, double threshold, index minSliceLength) { - double novelty = mNovelty.processFrame(_impl::asEigen(input)); double detected = 0.; - index filterSize = mFilterBuffer.size(); - if (filterSize > 1) - { - mFilterBuffer.segment(0, filterSize - 1) = - mFilterBuffer.segment(1, filterSize - 1); - } + mPeakBuffer.segment(0, 2) = mPeakBuffer.segment(1, 2); - mFilterBuffer(filterSize - 1) = novelty; - mPeakBuffer(2) = mFilterBuffer.mean(); + mPeakBuffer(2) = mNovelty.processFrame(input); + if (mPeakBuffer(1) > mPeakBuffer(0) && mPeakBuffer(1) > mPeakBuffer(2) && mPeakBuffer(1) > threshold && mDebounceCount == 0) { @@ -66,11 +57,9 @@ class NoveltySegmentation } private: - ArrayXd mFilterBuffer; - ArrayXd mFilterBufferStorage; - ArrayXd mPeakBuffer{3}; - Novelty mNovelty; - index mDebounceCount{1}; + NoveltyFeature mNovelty; + ArrayXd mPeakBuffer{3}; + index mDebounceCount{1}; }; } // namespace algorithm } // namespace fluid diff --git a/include/algorithms/public/OnsetDetectionFunctions.hpp b/include/algorithms/public/OnsetDetectionFunctions.hpp new file mode 100644 index 000000000..5ad85422e --- /dev/null +++ b/include/algorithms/public/OnsetDetectionFunctions.hpp @@ -0,0 +1,109 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright 2017-2019 University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ + +#pragma once + +#include "WindowFuncs.hpp" +#include "../util/FFT.hpp" +#include "../util/FluidEigenMappings.hpp" +#include "../util/MedianFilter.hpp" +#include "../util/OnsetDetectionFuncs.hpp" +#include "../../data/FluidIndex.hpp" +#include "../../data/TensorTypes.hpp" +#include +#include +#include + +namespace fluid { +namespace algorithm { + +class OnsetDetectionFunctions +{ + using ArrayXd = Eigen::ArrayXd; + using ArrayXcd = Eigen::ArrayXcd; + using WindowTypes = WindowFuncs::WindowTypes; + +public: + OnsetDetectionFunctions(index maxSize) + : mFFT(maxSize), mWindowStorage(maxSize) + {} + + void init(index windowSize, index fftSize, index filterSize) + { + makeWindow(windowSize); + prevFrame = ArrayXcd::Zero(fftSize / 2 + 1); + prevPrevFrame = ArrayXcd::Zero(fftSize / 2 + 1); + mFilter.init(filterSize); + mFFT.resize(fftSize); + mDebounceCount = 1; + mPrevFuncVal = 0; + mInitialized = true; + } + + void makeWindow(index windowSize) + { + mWindowStorage.setZero(); + WindowFuncs::map()[mWindowType](windowSize, mWindowStorage); + mWindow = mWindowStorage.segment(0, windowSize); + mWindowSize = windowSize; + } + + double processFrame(RealVectorView input, index function, index filterSize, + index frameDelta = 0) + { + assert(mInitialized); + ArrayXd in = _impl::asEigen(input); + double funcVal = 0; + double filteredFuncVal = 0; + double detected = 0.; + if (filterSize >= 3 && + (!mFilter.initialized() || filterSize != mFilter.size())) + mFilter.init(filterSize); + + ArrayXcd frame = mFFT.process(in.segment(0, mWindowSize) * mWindow); + auto odf = static_cast(function); + if (function > 1 && function < 5 && frameDelta != 0) + { + ArrayXcd frame2 = + mFFT.process(in.segment(frameDelta, mWindowSize) * mWindow); + funcVal = OnsetDetectionFuncs::map()[odf](frame2, frame, frame); + } + else + { + funcVal = + OnsetDetectionFuncs::map()[odf](frame, prevFrame, prevPrevFrame); + } + if (filterSize >= 3) + filteredFuncVal = funcVal - mFilter.processSample(funcVal); + else + filteredFuncVal = funcVal - mPrevFuncVal; + + prevPrevFrame = prevFrame; + prevFrame = frame; + + return filteredFuncVal; + } + +private: + FFT mFFT{1024}; + ArrayXd mWindowStorage; + ArrayXd mWindow; + index mWindowSize{1024}; + index mDebounceCount{1}; + ArrayXcd prevFrame; + ArrayXcd prevPrevFrame; + double mPrevFuncVal{0.0}; + WindowTypes mWindowType{WindowTypes::kHann}; + MedianFilter mFilter; + bool mInitialized{false}; +}; + +} // namespace algorithm +} // namespace fluid diff --git a/include/algorithms/public/OnsetSegmentation.hpp b/include/algorithms/public/OnsetSegmentation.hpp index b9f7ad61b..8492ba1da 100644 --- a/include/algorithms/public/OnsetSegmentation.hpp +++ b/include/algorithms/public/OnsetSegmentation.hpp @@ -10,11 +10,11 @@ under the European Union’s Horizon 2020 research and innovation programme #pragma once +#include "OnsetDetectionFunctions.hpp" #include "WindowFuncs.hpp" #include "../util/FFT.hpp" #include "../util/FluidEigenMappings.hpp" #include "../util/MedianFilter.hpp" -#include "../util/OnsetDetectionFuncs.hpp" #include "../../data/FluidIndex.hpp" #include "../../data/TensorTypes.hpp" #include @@ -31,61 +31,21 @@ class OnsetSegmentation using ArrayXd = Eigen::ArrayXd; using ArrayXcd = Eigen::ArrayXcd; - OnsetSegmentation(index maxSize) : mFFT(maxSize), mWindowStorage(maxSize) {} + OnsetSegmentation(index maxSize) : mODF{maxSize} {} void init(index windowSize, index fftSize, index filterSize) { - makeWindow(windowSize); - prevFrame = ArrayXcd::Zero(fftSize / 2 + 1); - prevPrevFrame = ArrayXcd::Zero(fftSize / 2 + 1); - mFilter.init(filterSize); - mFFT.resize(fftSize); + mODF.init(windowSize, fftSize, filterSize); mDebounceCount = 1; - mPrevFuncVal = 0; - mInitialized = true; - } - - void makeWindow(index windowSize) - { - mWindowStorage.setZero(); - WindowFuncs::map()[mWindowType](windowSize, mWindowStorage); - mWindow = mWindowStorage.segment(0, windowSize); - mWindowSize = windowSize; } double processFrame(RealVectorView input, index function, index filterSize, double threshold, index debounce = 0, index frameDelta = 0) { - assert(mInitialized); - ArrayXd in = _impl::asEigen(input); - double funcVal = 0; - double filteredFuncVal = 0; - double detected = 0.; - if (filterSize >= 3 && - (!mFilter.initialized() || filterSize != mFilter.size())) - mFilter.init(filterSize); - - ArrayXcd frame = mFFT.process(in.segment(0, mWindowSize) * mWindow); - auto odf = static_cast(function); - if (function > 1 && function < 5 && frameDelta != 0) - { - ArrayXcd frame2 = - mFFT.process(in.segment(frameDelta, mWindowSize) * mWindow); - funcVal = OnsetDetectionFuncs::map()[odf](frame2, frame, frame); - } - else - { - funcVal = - OnsetDetectionFuncs::map()[odf](frame, prevFrame, prevPrevFrame); - } - if (filterSize >= 3) - filteredFuncVal = funcVal - mFilter.processSample(funcVal); - else - filteredFuncVal = funcVal - mPrevFuncVal; - - prevPrevFrame = prevFrame; - prevFrame = frame; + double filteredFuncVal = + mODF.processFrame(input, function, filterSize, frameDelta); + double detected{0}; if (filteredFuncVal > threshold && mPrevFuncVal < threshold && mDebounceCount == 0) @@ -102,18 +62,9 @@ class OnsetSegmentation } private: - using WindowTypes = WindowFuncs::WindowTypes; - FFT mFFT{1024}; - ArrayXd mWindowStorage; - ArrayXd mWindow; - index mWindowSize{1024}; - index mDebounceCount{1}; - ArrayXcd prevFrame; - ArrayXcd prevPrevFrame; - double mPrevFuncVal{0.0}; - WindowTypes mWindowType{WindowTypes::kHann}; - MedianFilter mFilter; - bool mInitialized{false}; + index mDebounceCount{1}; + OnsetDetectionFunctions mODF; + double mPrevFuncVal{0.0}; }; } // namespace algorithm diff --git a/include/clients/common/FluidBaseClient.hpp b/include/clients/common/FluidBaseClient.hpp index f1d1d8de9..5a839b821 100644 --- a/include/clients/common/FluidBaseClient.hpp +++ b/include/clients/common/FluidBaseClient.hpp @@ -107,6 +107,12 @@ class FluidBaseClient std::vector mOutputLabels; }; +struct AnalysisSize +{ + index window; + index hop; +}; + template class ClientWrapper { @@ -206,7 +212,7 @@ class ClientWrapper mClient, std::forward(args)...); } - index controlRate() { return mClient.controlRate(); } + AnalysisSize analysisSettings() { return mClient.analysisSettings(); } template void set(T&& x, Result* reportage) noexcept diff --git a/include/clients/common/FluidNRTClientWrapper.hpp b/include/clients/common/FluidNRTClientWrapper.hpp index ef7ddcbff..814daf612 100644 --- a/include/clients/common/FluidNRTClientWrapper.hpp +++ b/include/clients/common/FluidNRTClientWrapper.hpp @@ -538,7 +538,7 @@ struct StreamingControl index startPadding = client.latency() + userPadding.first; index totalPadding = startPadding + userPadding.first; - index controlRate = client.controlRate(); + index controlRate = client.analysisSettings().hop; index paddedLength = nFrames + totalPadding; @@ -548,10 +548,9 @@ struct StreamingControl paddedLength = static_cast(std::ceil(double(paddedLength) / controlRate) * controlRate); - // Fix me. This assumes that client.latency() is always the window size of - // whatever buffered process we're wrapping, which seems well dodgy - index nHops = - static_cast(1 + std::floor((paddedLength - client.latency()) / controlRate)); + index windowSize = client.analysisSettings().window; + index nHops = static_cast( + 1 + std::floor((paddedLength - windowSize) / controlRate)); // in contrast to the plain streaming case, we're going to call process() // iteratively with a vector size = the control vector size, so we get KR @@ -608,7 +607,7 @@ struct StreamingControl BufferAdaptor::Access thisOutput(outputBuffers[0]); - index latencyHops = client.latency() / client.controlRate(); + index latencyHops = client.latency() / controlRate; index keepHops = nHops - latencyHops; Result resizeResult = thisOutput.resize(keepHops, nChans * nFeatures, diff --git a/include/clients/rt/AmpFeatureClient.hpp b/include/clients/rt/AmpFeatureClient.hpp new file mode 100644 index 000000000..76f8bc85e --- /dev/null +++ b/include/clients/rt/AmpFeatureClient.hpp @@ -0,0 +1,119 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright 2017-2019 University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ +#pragma once + +#include "../common/AudioClient.hpp" +#include "../common/BufferedProcess.hpp" +#include "../common/FluidBaseClient.hpp" +#include "../common/FluidNRTClientWrapper.hpp" +#include "../common/ParameterConstraints.hpp" +#include "../common/ParameterSet.hpp" +#include "../common/ParameterTrackChanges.hpp" +#include "../common/ParameterTypes.hpp" +#include "../../algorithms/public/Envelope.hpp" +#include + +namespace fluid { +namespace client { +namespace ampfeature { + +enum AmpFeatureParamIndex { + kFastRampUpTime, + kFastRampDownTime, + kSlowRampUpTime, + kSlowRampDownTime, + kSilenceThreshold, + kHiPassFreq, +}; + +constexpr auto AmpFeatureParams = defineParameters( + LongParam("fastRampUp", "Fast Envelope Ramp Up Length", 1, Min(1)), + LongParam("fastRampDown", "Fast Envelope Ramp Down Length", 1, Min(1)), + LongParam("slowRampUp", "Slow Envelope Ramp Up Length", 100, Min(1)), + LongParam("slowRampDown", "Slow Envelope Ramp Down Length", 100, Min(1)), + FloatParam("floor", "Floor value (dB)", -145, Min(-144), Max(144)), + FloatParam("highPassFreq", "High-Pass Filter Cutoff", 85, Min(0))); + +class AmpFeatureClient : public FluidBaseClient, public AudioIn, public AudioOut +{ + +public: + using ParamDescType = decltype(AmpFeatureParams); + + using ParamSetViewType = ParameterSetView; + std::reference_wrapper mParams; + + void setParams(ParamSetViewType& p) { mParams = p; } + + template + auto& get() const + { + return mParams.get().template get(); + } + + static constexpr auto& getParameterDescriptors() { return AmpFeatureParams; } + + AmpFeatureClient(ParamSetViewType& p) : mParams(p) + { + audioChannelsIn(1); + audioChannelsOut(1); + FluidBaseClient::setInputLabels({"audio input"}); + FluidBaseClient::setOutputLabels({"1 when slice detected, 0 otherwise"}); + } + + template + void process(std::vector>& input, + std::vector>& output, FluidContext&) + { + + if (!input[0].data() || !output[0].data()) return; + + double hiPassFreq = std::min(get() / sampleRate(), 0.5); + + if (!mAlgorithm.initialized()) + { + mAlgorithm.init(get(), hiPassFreq); + } + + for (index i = 0; i < input[0].size(); i++) + { + output[0](i) = static_cast(mAlgorithm.processSample( + input[0](i), get(), get(), + get(), get(), + get(), hiPassFreq)); + } + } + index latency() { return 0; } + + void reset() + { + double hiPassFreq = std::min(get() / sampleRate(), 0.5); + mAlgorithm.init(get(), hiPassFreq); + } + +private: + algorithm::Envelope mAlgorithm; +}; +} // namespace ampfeature + +using RTAmpFeatureClient = ClientWrapper; + +auto constexpr NRTAmpFeatureParams = makeNRTParams( + InputBufferParam("source", "Source Buffer"), + BufferParam("features", "Feature Buffer")); + +using NRTAmpFeatureClient = + NRTStreamAdaptor; + +using NRTThreadedAmpFeatureClient = NRTThreadingAdaptor; + +} // namespace client +} // namespace fluid diff --git a/include/clients/rt/ChromaClient.hpp b/include/clients/rt/ChromaClient.hpp index f34de86a9..b1c453c8d 100644 --- a/include/clients/rt/ChromaClient.hpp +++ b/include/clients/rt/ChromaClient.hpp @@ -114,16 +114,19 @@ class ChromaClient : public FluidBaseClient, public AudioIn, public ControlOut sampleRate()); } - index controlRate() { return get().hopSize(); } + AnalysisSize analysisSettings() + { + return { get().winSize(), get().hopSize() }; + } -private: - ParameterTrackChanges mTracker; - STFTBufferedProcess mSTFTBufferedProcess; + private: + ParameterTrackChanges mTracker; + STFTBufferedProcess mSTFTBufferedProcess; - algorithm::ChromaFilterBank mAlgorithm; - FluidTensor mMagnitude; - FluidTensor mChroma; -}; + algorithm::ChromaFilterBank mAlgorithm; + FluidTensor mMagnitude; + FluidTensor mChroma; + }; } // namespace chroma using RTChromaClient = ClientWrapper; diff --git a/include/clients/rt/LoudnessClient.hpp b/include/clients/rt/LoudnessClient.hpp index c71f810a1..a7bb7b196 100644 --- a/include/clients/rt/LoudnessClient.hpp +++ b/include/clients/rt/LoudnessClient.hpp @@ -111,7 +111,11 @@ class LoudnessClient : public FluidBaseClient, public AudioIn, public ControlOut mAlgorithm.init(get(), sampleRate()); } - index controlRate() { return get(); } + AnalysisSize analysisSettings() + { + return { get(), get() }; + } + private: ParameterTrackChanges mBufferParamsTracker; diff --git a/include/clients/rt/MFCCClient.hpp b/include/clients/rt/MFCCClient.hpp index 0cab8c696..60e4acd4a 100644 --- a/include/clients/rt/MFCCClient.hpp +++ b/include/clients/rt/MFCCClient.hpp @@ -135,7 +135,11 @@ class MFCCClient : public FluidBaseClient, public AudioIn, public ControlOut mDCT.init(get(), get() + get()); } - index controlRate() { return get().hopSize(); } + AnalysisSize analysisSettings() + { + return { get().winSize(), get().hopSize() }; + } + private: ParameterTrackChanges mTracker; diff --git a/include/clients/rt/MelBandsClient.hpp b/include/clients/rt/MelBandsClient.hpp index 4af861c4d..866e9891d 100644 --- a/include/clients/rt/MelBandsClient.hpp +++ b/include/clients/rt/MelBandsClient.hpp @@ -120,7 +120,11 @@ class MelBandsClient : public FluidBaseClient, public AudioIn, public ControlOut get().winSize()); } - index controlRate() { return get().hopSize(); } + AnalysisSize analysisSettings() + { + return { get().winSize(), get().hopSize() }; + } + private: ParameterTrackChanges diff --git a/include/clients/rt/NoveltyFeatureClient.hpp b/include/clients/rt/NoveltyFeatureClient.hpp new file mode 100644 index 000000000..3d8268c99 --- /dev/null +++ b/include/clients/rt/NoveltyFeatureClient.hpp @@ -0,0 +1,241 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright 2017-2019 University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ +#pragma once + +#include "../common/AudioClient.hpp" +#include "../common/BufferedProcess.hpp" +#include "../common/FluidBaseClient.hpp" +#include "../common/FluidNRTClientWrapper.hpp" +#include "../common/ParameterConstraints.hpp" +#include "../common/ParameterSet.hpp" +#include "../common/ParameterTypes.hpp" +#include "../../algorithms/public/ChromaFilterBank.hpp" +#include "../../algorithms/public/DCT.hpp" +#include "../../algorithms/public/Loudness.hpp" +#include "../../algorithms/public/MelBands.hpp" +#include "../../algorithms/public/NoveltyFeature.hpp" +#include "../../algorithms/public/STFT.hpp" +#include "../../algorithms/public/YINFFT.hpp" +#include "../../algorithms/util/TruePeak.hpp" +#include "../../data/TensorTypes.hpp" +#include + +namespace fluid { +namespace client { +namespace noveltyfeature { + +enum NoveltyParamIndex { + kFeature, + kKernelSize, + kFilterSize, + kFFT, + kMaxFFTSize, + kMaxKernelSize, + kMaxFilterSize, +}; + +constexpr auto NoveltyFeatureParams = defineParameters( + EnumParam("algorithm", "Feature", 0, "Spectrum", "MFCC", "Chroma", "Pitch", + "Loudness"), + LongParam("kernelSize", "KernelSize", 3, Min(3), Odd(), + UpperLimit()), + LongParam("filterSize", "Smoothing Filter Size", 1, Min(1), + UpperLimit()), + FFTParam("fftSettings", "FFT Settings", 1024, -1, -1), + LongParam>("maxFFTSize", "Maxiumm FFT Size", 16384, Min(4), + PowerOfTwo{}), + LongParam>("maxKernelSize", "Maxiumm Kernel Size", 101, Min(3), + Odd()), + LongParam>("maxFilterSize", "Maxiumm Filter Size", 100, + Min(1))); + +class NoveltyFeatureClient : public FluidBaseClient, + public AudioIn, + public ControlOut +{ +public: + using ParamDescType = decltype(NoveltyFeatureParams); + + using ParamSetViewType = ParameterSetView; + std::reference_wrapper mParams; + + void setParams(ParamSetViewType& p) { mParams = p; } + + template + auto& get() const + { + return mParams.get().template get(); + } + + static constexpr auto& getParameterDescriptors() + { + return NoveltyFeatureParams; + } + + NoveltyFeatureClient(ParamSetViewType& p) + : mParams{p}, mNovelty{get(), get()}, + mSTFT{get().winSize(), get().fftSize(), + get().hopSize()}, + mMelBands{40, get()}, + mChroma(12, get()), mLoudness{get()} + { + audioChannelsIn(1); + controlChannelsOut({1,1}); + setInputLabels({"audio input"}); + setOutputLabels({"estimated novelty"}); + } + + + void initAlgorithms(index feature, index windowSize) + { + index nDims = 2; + if (feature < 4) + { + mSpectrum.resize(get().frameSize()); + mMagnitude.resize(get().frameSize()); + mSTFT = algorithm::STFT(get().winSize(), get().fftSize(), + get().hopSize()); + } + if (feature == 0) { nDims = get().frameSize(); } + else if (feature == 1) + { + mBands.resize(40); + mMelBands.init(20, 20e3, 40, get().frameSize(), sampleRate(), + get().winSize()); + mDCT.init(40, 13); + nDims = 13; + } + else if (feature == 2) + { + mChroma.init(12, get().frameSize(), 440, sampleRate()); + nDims = 12; + } + else if (feature == 4) + { + mLoudness.init(windowSize, sampleRate()); + } + mFeature.resize(nDims); + mNovelty.init(get(), get(), nDims); + } + + template + void process(std::vector>& input, + std::vector>& output, FluidContext& c) + { + using algorithm::NoveltyFeature; + + + if (!input[0].data() || !output[0].data()) return; + + index hostVecSize = input[0].size(); + index windowSize = get().winSize(); + index feature = get(); + if (mParamsTracker.changed(hostVecSize, get(), get(), + get(), windowSize, sampleRate())) + { + mBufferedProcess.hostSize(hostVecSize); + mBufferedProcess.maxSize(windowSize, windowSize, + FluidBaseClient::audioChannelsIn(), + FluidBaseClient::audioChannelsOut()); + initAlgorithms(feature, windowSize); + } + RealMatrix in(1, hostVecSize); + in.row(0) = input[0]; + + mBufferedProcess.push(RealMatrixView(in)); + mBufferedProcess.processInput( + windowSize, get().hopSize(), c, + [&, this](RealMatrixView in) { + switch (feature) + { + case 0: + mSTFT.processFrame(in.row(0), mSpectrum); + mSTFT.magnitude(mSpectrum, mFeature); + break; + case 1: + mSTFT.processFrame(in.row(0), mSpectrum); + mSTFT.magnitude(mSpectrum, mMagnitude); + mMelBands.processFrame(mMagnitude, mBands, false, false, true); + mDCT.processFrame(mBands, mFeature); + break; + case 2: + mSTFT.processFrame(in.row(0), mSpectrum); + mSTFT.magnitude(mSpectrum, mMagnitude); + mChroma.processFrame(mMagnitude, mFeature, 20, 5000); + break; + case 3: + mSTFT.processFrame(in.row(0), mSpectrum); + mSTFT.magnitude(mSpectrum, mMagnitude); + mYinFFT.processFrame(mMagnitude, mFeature, 20, 5000, sampleRate()); + break; + case 4: + mLoudness.processFrame(in.row(0), mFeature, true, true); + break; + } +// if (mFrameOffset < out.row(0).size()) + mDescriptor = mNovelty.processFrame(mFeature); + }); + + output[0](0) = mDescriptor; + } + + index latency() + { + index filterSize = get(); + if (filterSize % 2) filterSize++; + return get().hopSize() * + (1 + ((get() + 1) >> 1) + (filterSize >> 1)); + } + + AnalysisSize analysisSettings() + { + return {get().winSize(), get().hopSize()}; + } + + void reset() + { + mBufferedProcess.reset(); + initAlgorithms(get(), get().winSize()); + } + +private: + algorithm::NoveltyFeature mNovelty; + ParameterTrackChanges + mParamsTracker; + BufferedProcess mBufferedProcess; + algorithm::STFT mSTFT; + FluidTensor, 1> mSpectrum; + FluidTensor mMagnitude; + FluidTensor mBands; + FluidTensor mFeature; + double mDescriptor; + algorithm::MelBands mMelBands; + algorithm::DCT mDCT{40, 13}; + algorithm::ChromaFilterBank mChroma; + algorithm::YINFFT mYinFFT; + algorithm::Loudness mLoudness; +}; +} // namespace noveltyfeature + +using RTNoveltyFeatureClient = ClientWrapper; + +auto constexpr NRTNoveltyFeatureParams = makeNRTParams( + InputBufferParam("source", "Source Buffer"), + BufferParam("features", "Feature Buffer")); + +using NRTNoveltyFeatureClient = NRTControlAdaptor; + +using NRTThreadedNoveltyFeatureClient = + NRTThreadingAdaptor; + +} // namespace client +} // namespace fluid diff --git a/include/clients/rt/NoveltySliceClient.hpp b/include/clients/rt/NoveltySliceClient.hpp index d872d81e8..106c723a4 100644 --- a/include/clients/rt/NoveltySliceClient.hpp +++ b/include/clients/rt/NoveltySliceClient.hpp @@ -44,7 +44,7 @@ enum NoveltyParamIndex { }; constexpr auto NoveltySliceParams = defineParameters( - EnumParam("feature", "Feature", 0, "Spectrum", "MFCC", "Chroma", "Pitch", + EnumParam("algorithm", "Feature", 0, "Spectrum", "MFCC", "Chroma", "Pitch", "Loudness"), LongParam("kernelSize", "KernelSize", 3, Min(3), Odd(), UpperLimit()), diff --git a/include/clients/rt/OnsetFeatureClient.hpp b/include/clients/rt/OnsetFeatureClient.hpp new file mode 100644 index 000000000..0c4e8f9b9 --- /dev/null +++ b/include/clients/rt/OnsetFeatureClient.hpp @@ -0,0 +1,154 @@ +/* +Part of the Fluid Corpus Manipulation Project (http://www.flucoma.org/) +Copyright 2017-2019 University of Huddersfield. +Licensed under the BSD-3 License. +See license.md file in the project root for full license information. +This project has received funding from the European Research Council (ERC) +under the European Union’s Horizon 2020 research and innovation programme +(grant agreement No 725899). +*/ +#pragma once + +#include "../common/AudioClient.hpp" +#include "../common/BufferedProcess.hpp" +#include "../common/FluidBaseClient.hpp" +#include "../common/FluidNRTClientWrapper.hpp" +#include "../common/ParameterConstraints.hpp" +#include "../common/ParameterSet.hpp" +#include "../common/ParameterTypes.hpp" +#include "../../algorithms/public/OnsetDetectionFunctions.hpp" +#include "../../data/TensorTypes.hpp" +#include + +namespace fluid { +namespace client { +namespace onsetfeature { + +enum OnsetParamIndex { + kFunction, + kFilterSize, + kFrameDelta, + kFFT, + kMaxFFTSize +}; + +constexpr auto OnsetFeatureParams = defineParameters( + EnumParam("metric", "Spectral Change Metric", 0, "Energy", + "High Frequency Content", "Spectral Flux", + "Modified Kullback-Leibler", "Itakura-Saito", "Cosine", + "Phase Deviation", "Weighted Phase Deviation", "Complex Domain", + "Rectified Complex Domain"), + LongParam("filterSize", "Filter Size", 5, Min(1), Odd(), Max(101)), + LongParam("frameDelta", "Frame Delta", 0, Min(0)), + FFTParam("fftSettings", "FFT Settings", 1024, -1, -1), + LongParam>("maxFFTSize", "Maxiumm FFT Size", 16384, Min(4), + PowerOfTwo{})); + +class OnsetFeatureClient : public FluidBaseClient, public AudioIn, public ControlOut +{ + + using OnsetDetectionFunctions = algorithm::OnsetDetectionFunctions; + +public: + using ParamDescType = decltype(OnsetFeatureParams); + + using ParamSetViewType = ParameterSetView; + std::reference_wrapper mParams; + + void setParams(ParamSetViewType& p) { mParams = p; } + + template + auto& get() const + { + return mParams.get().template get(); + } + + static constexpr auto& getParameterDescriptors() { return OnsetFeatureParams; } + + OnsetFeatureClient(ParamSetViewType& p) + : mParams{p}, mAlgorithm{get()} + { + audioChannelsIn(1); + controlChannelsOut({1,1}); + setInputLabels({"audio input"}); + setOutputLabels({"1 when slice detected, 0 otherwise"}); + } + + template + void process(std::vector>& input, + std::vector>& output, FluidContext& c) + { + using std::size_t; + + if (!input[0].data() || !output[0].data()) return; + + index hostVecSize = input[0].size(); + index totalWindow = get().winSize(); + if (get() > 1 && get() < 5) + totalWindow += get(); + if (mBufferParamsTracker.changed(hostVecSize, get().winSize(), + get())) + { + mBufferedProcess.hostSize(hostVecSize); + mBufferedProcess.maxSize(totalWindow, totalWindow, + FluidBaseClient::audioChannelsIn(), + FluidBaseClient::audioChannelsOut()); + } + if (mParamsTracker.changed(get().fftSize(), get().winSize())) + { + mAlgorithm.init(get().winSize(), get().fftSize(), + get()); + } + RealMatrix in(1, hostVecSize); + in.row(0) = input[0]; + + mBufferedProcess.push(RealMatrixView(in)); + mBufferedProcess.processInput( + totalWindow, get().hopSize(), c, [&, this](RealMatrixView in) { + mDescriptor = mAlgorithm.processFrame( + in.row(0), get(), get(), get()); + }); + + output[0](0) = mDescriptor; + } + + index latency() { return static_cast(get().hopSize()); } + + AnalysisSize analysisSettings() + { + return {get().winSize(), get().hopSize()}; + } + + void reset() + { + mBufferedProcess.reset(); + mAlgorithm.init(get().winSize(), get().fftSize(), + get()); + } + +private: + OnsetDetectionFunctions mAlgorithm; + double mDescriptor; + ParameterTrackChanges mBufferParamsTracker; + ParameterTrackChanges mParamsTracker; + BufferedProcess mBufferedProcess; +}; +} // namespace onsetfeature + +using RTOnsetFeatureClient = ClientWrapper; + +auto constexpr NRTOnsetFeatureParams = + makeNRTParams( + InputBufferParam("source", "Source Buffer"), + BufferParam("features", "Feature Buffer")); + + +using NRTOnsetFeatureClient = + NRTControlAdaptor; + + +using NRTThreadedOnsetFeatureClient = NRTThreadingAdaptor; + +} // namespace client +} // namespace fluid diff --git a/include/clients/rt/PitchClient.hpp b/include/clients/rt/PitchClient.hpp index 5371970fe..a4b809372 100644 --- a/include/clients/rt/PitchClient.hpp +++ b/include/clients/rt/PitchClient.hpp @@ -128,7 +128,12 @@ class PitchClient : public FluidBaseClient, public AudioIn, public ControlOut output[0](1) = static_cast(mDescriptors(1)); } index latency() { return get().winSize(); } - index controlRate() { return get().hopSize(); } + + AnalysisSize analysisSettings() + { + return { get().winSize(), get().hopSize() }; + } + void reset() { mSTFTBufferedProcess.reset(); diff --git a/include/clients/rt/SpectralShapeClient.hpp b/include/clients/rt/SpectralShapeClient.hpp index fb9eecae7..31268d9a3 100644 --- a/include/clients/rt/SpectralShapeClient.hpp +++ b/include/clients/rt/SpectralShapeClient.hpp @@ -111,7 +111,10 @@ class SpectralShapeClient : public FluidBaseClient, void reset() { mSTFTBufferedProcess.reset(); } - index controlRate() { return get().hopSize(); } + AnalysisSize analysisSettings() + { + return { get().winSize(), get().hopSize() }; + } private: ParameterTrackChanges mTracker; diff --git a/tests/data/TestFluidTensorSupport.cpp b/tests/data/TestFluidTensorSupport.cpp index e6ef3aae2..5ebb0b04b 100644 --- a/tests/data/TestFluidTensorSupport.cpp +++ b/tests/data/TestFluidTensorSupport.cpp @@ -118,8 +118,6 @@ TEST_CASE("FluidTensorSlice operator() maps indices back to flat layout","[Fluid SECTION("3D"){ FluidTensorSlice<3> x{2,9,3}; - for(auto& s:x.strides) std::cout << s <<','; - std::cout <