From a3558aa612e4793d7597620293296964abcc5bc1 Mon Sep 17 00:00:00 2001 From: Julian Santander Date: Sun, 4 Oct 2020 20:45:18 +0200 Subject: [PATCH 1/2] Initial working android code --- Jamulus.pro | 2 + android/sound.cpp | 120 ++++++++++++++++++++++------------------------ android/sound.h | 16 ++----- 3 files changed, 62 insertions(+), 76 deletions(-) diff --git a/Jamulus.pro b/Jamulus.pro index 5708a6df9d..94fc9c35b8 100755 --- a/Jamulus.pro +++ b/Jamulus.pro @@ -1019,3 +1019,5 @@ contains(CONFIG, "disable_version_check") { message(The version check is disabled.) DEFINES += DISABLE_VERSION_CHECK } + +ANDROID_ABIS = armeabi-v7a arm64-v8a diff --git a/android/sound.cpp b/android/sound.cpp index bb7105df28..770850b2e6 100644 --- a/android/sound.cpp +++ b/android/sound.cpp @@ -34,8 +34,6 @@ CSound::CSound ( void (*fpNewProcessCallback) ( CVector& psData const QString& ) : CSoundBase ( "OpenSL", fpNewProcessCallback, arg, iCtrlMIDIChannel ) { - pSound = this; - #ifdef ANDROIDDEBUG qInstallMessageHandler(myMessageHandler); #endif @@ -45,11 +43,12 @@ void CSound::setupCommonStreamParams(oboe::AudioStreamBuilder *builder) { // We request EXCLUSIVE mode since this will give us the lowest possible // latency. If EXCLUSIVE mode isn't available the builder will fall back to SHARED mode - builder->setCallback(this) + builder ->setFormat(oboe::AudioFormat::Float) - ->setSharingMode(oboe::SharingMode::Shared) - ->setChannelCount(oboe::ChannelCount::Mono) - // ->setSampleRate(48000) + ->setSharingMode(oboe::SharingMode::Exclusive) + ->setChannelCount(oboe::ChannelCount::Stereo) + ->setSampleRate(48000) + ->setFramesPerCallback(iOpenSLBufferSizeMono) // ->setSampleRateConversionQuality(oboe::SampleRateConversionQuality::Medium) ->setPerformanceMode(oboe::PerformanceMode::None); @@ -73,14 +72,17 @@ void CSound::openStreams() { return; } - - mPlayStream->setBufferSizeInFrames ( pSound->iOpenSLBufferSizeStereo ); + mPlayStream->setBufferSizeInFrames ( iOpenSLBufferSizeStereo ); warnIfNotLowLatency ( mPlayStream, "PlayStream" ); printStreamDetails ( mPlayStream ); // Setup input stream inBuilder.setDirection ( oboe::Direction::Input ); + + // Only set callback for the input direction + // the output will be handled writing directly on the stream + inBuilder.setCallback(this); setupCommonStreamParams ( &inBuilder ); result = inBuilder.openManagedStream ( mRecordingStream ); @@ -91,7 +93,7 @@ void CSound::openStreams() return; } - mRecordingStream->setBufferSizeInFrames ( pSound->iOpenSLBufferSizeStereo ); + mRecordingStream->setBufferSizeInFrames ( iOpenSLBufferSizeStereo ); warnIfNotLowLatency ( mRecordingStream, "RecordStream" ); printStreamDetails ( mRecordingStream ); @@ -184,7 +186,7 @@ void CSound::Stop() int CSound::Init ( const int iNewPrefMonoBufferSize ) { // store buffer size - iOpenSLBufferSizeMono = 512; // iNewPrefMonoBufferSize; + iOpenSLBufferSizeMono = iNewPrefMonoBufferSize; // 512 // init base class CSoundBase::Init ( iOpenSLBufferSizeMono ); @@ -193,19 +195,13 @@ int CSound::Init ( const int iNewPrefMonoBufferSize ) iOpenSLBufferSizeStereo = 2 * iOpenSLBufferSizeMono; // create memory for intermediate audio buffer - vecsTmpAudioSndCrdStereo.Init ( iOpenSLBufferSizeStereo ); + vecsTmpInputAudioSndCrdStereo.Init ( iOpenSLBufferSizeStereo ); + vecsTmpOutAudioSndCrdStereo.Init ( iOpenSLBufferSizeStereo ); // TEST #if ( SYSTEM_SAMPLE_RATE_HZ != 48000 ) # error "Only a system sample rate of 48 kHz is supported by this module" #endif -// audio interface number of channels is 1 and the sample rate -// is 16 kHz -> just copy samples and perform no filtering as a -// first simple solution -// 48 kHz / 16 kHz = factor 3 (note that the buffer size mono might -// be divisible by three, therefore we will get a lot of drop outs) -iModifiedInBufSize = iOpenSLBufferSizeMono / 3; -vecsTmpAudioInSndCrd.Init ( iModifiedInBufSize ); return iOpenSLBufferSizeMono; } @@ -216,31 +212,45 @@ vecsTmpAudioInSndCrd.Init ( iModifiedInBufSize ); oboe::DataCallbackResult CSound::onAudioReady ( oboe::AudioStream* oboeStream, void* audioData, int32_t numFrames ) { // only process if we are running - if ( ! pSound->bRun ) + if ( ! bRun ) { return oboe::DataCallbackResult::Continue; } - // Need to modify the size of the buffer based on the numFrames requested in this callback. - // Buffer size can change regularly by android devices - int& iBufferSizeMono = pSound->iOpenSLBufferSizeMono; - - // perform the processing for input and output -// QMutexLocker locker ( &pSound->Mutex ); -// locker.mutex(); - - // This can be called from both input and output at different times - if ( oboeStream == pSound->mPlayStream.get() && audioData ) + if ( oboeStream == mRecordingStream.get() && audioData ) { + // First things first, we need to discard the input queue a little for 500ms or so + if ( mCountCallbacksToDrain > 0 ) + { + // discard the input buffer + int32_t numBytes = numFrames * oboeStream->getBytesPerFrame(); + memset ( audioData, 0 /* value */, numBytes ); + mCountCallbacksToDrain--; + } + + // We're good to start recording now + // Take the data from the recording device output buffer and move + // it to the vector ready to send up to the server float *floatData = static_cast ( audioData ); + // Copy recording data to internal vector + for ( int frmNum = 0; frmNum < numFrames; ++frmNum ) + { + for ( int channelNum = 0; channelNum < oboeStream->getChannelCount(); channelNum++ ) + { + vecsTmpInputAudioSndCrdStereo[frmNum * oboeStream->getChannelCount() + channelNum] = + static_cast(floatData[frmNum * oboeStream->getChannelCount() + channelNum] * _MAXSHORT); + } + } + + // Tell parent class that we've put some data ready to send to the server + ProcessCallback ( vecsTmpInputAudioSndCrdStereo ); - // Zero out the incoming container array - memset ( audioData, 0, sizeof(float) * numFrames * oboeStream->getChannelCount() ); + // The callback has placed in the vector the samples to play // Only copy data if we have data to copy, otherwise fill with silence - if ( !pSound->vecsTmpAudioSndCrdStereo.empty() ) + if ( !vecsTmpInputAudioSndCrdStereo.empty() ) { - for ( int frmNum = 0; frmNum < numFrames; ++frmNum ) + for ( int frmNum = 0; frmNum < iOpenSLBufferSizeMono; ++frmNum ) { for ( int channelNum = 0; channelNum < oboeStream->getChannelCount(); channelNum++ ) { @@ -248,51 +258,35 @@ oboe::DataCallbackResult CSound::onAudioReady ( oboe::AudioStream* oboeStream, v // convert to 32 bit const int32_t iCurSam = static_cast ( - pSound->vecsTmpAudioSndCrdStereo [frmNum * oboeStream->getChannelCount() + channelNum] ); + vecsTmpInputAudioSndCrdStereo [frmNum * oboeStream->getChannelCount() + channelNum] ); - floatData[frmNum * oboeStream->getChannelCount() + channelNum] = (float) iCurSam / _MAXSHORT; + vecsTmpOutAudioSndCrdStereo[frmNum * oboeStream->getChannelCount() + channelNum] = ((float) iCurSam) / _MAXSHORT; } } } else { // prime output stream buffer with silence - memset(static_cast ( audioData ) + numFrames * oboeStream->getChannelCount(), 0, - ( numFrames ) * oboeStream->getBytesPerFrame() ); - } - } - else if ( oboeStream == pSound->mRecordingStream.get() && audioData ) - { - // First things first, we need to discard the input queue a little for 500ms or so - if ( pSound->mCountCallbacksToDrain > 0 ) - { - // discard the input buffer - int32_t numBytes = numFrames * oboeStream->getBytesPerFrame(); - memset ( audioData, 0 /* value */, numBytes ); - pSound->mCountCallbacksToDrain--; + vecsTmpOutAudioSndCrdStereo.Reset(0); } - // We're good to start recording now - // Take the data from the recording device output buffer and move - // it to the vector ready to send up to the server - float *floatData = static_cast ( audioData ); - - // Copy recording data to internal vector - for ( int frmNum = 0; frmNum < numFrames; ++frmNum ) + // Write without blocking. + oboe::ResultWithValue r = mPlayStream->write(vecsTmpOutAudioSndCrdStereo.data(),iOpenSLBufferSizeMono,0); + if(r) { - for ( int channelNum = 0; channelNum < oboeStream->getChannelCount(); channelNum++ ) + // Note: this could result in not writing the requested number of frames. + // We could buffer them and wait for the next occasion, but for now we just discard them. + if(r.value() < iOpenSLBufferSizeMono) { - pSound->vecsTmpAudioSndCrdStereo[frmNum * oboeStream->getChannelCount() + channelNum] = - (short) floatData[frmNum * oboeStream->getChannelCount() + channelNum] * _MAXSHORT; + qDebug() << "Could only write " << r.value() << " frames of the " << iOpenSLBufferSizeMono << " requested."; } } - - // Tell parent class that we've put some data ready to send to the server - pSound->ProcessCallback ( pSound->vecsTmpAudioSndCrdStereo ); + else + { + qDebug() << "Could not write to output stream: " << oboe::convertToText(r.error()); + } } -// locker.unlock(); - return oboe::DataCallbackResult::Continue; } diff --git a/android/sound.h b/android/sound.h index b97c3ffe4e..d21fdb375b 100644 --- a/android/sound.h +++ b/android/sound.h @@ -31,7 +31,6 @@ #include #include - /* Classes ********************************************************************/ class CSound : public CSoundBase, public oboe::AudioStreamCallback//, public IRenderableAudio, public IRestartable { @@ -52,9 +51,9 @@ class CSound : public CSoundBase, public oboe::AudioStreamCallback//, public IRe virtual void onErrorAfterClose ( oboe::AudioStream* oboeStream, oboe::Result result ); virtual void onErrorBeforeClose ( oboe::AudioStream* oboeStream, oboe::Result result ); - // these variables should be protected but cannot since we want - // to access them from the callback function - CVector vecsTmpAudioSndCrdStereo; +protected: + CVector vecsTmpInputAudioSndCrdStereo; + CVector vecsTmpOutAudioSndCrdStereo; static void android_message_handler ( QtMsgType type, const QMessageLogContext& context, @@ -73,10 +72,6 @@ class CSound : public CSoundBase, public oboe::AudioStreamCallback//, public IRe __android_log_print ( priority, "Qt", "%s", qPrintable ( message ) ); }; -// TEST -CVector vecsTmpAudioInSndCrd; -int iModifiedInBufSize; - int iOpenSLBufferSizeMono; int iOpenSLBufferSizeStereo; @@ -96,9 +91,4 @@ int iModifiedInBufSize; // empty and the garbage in the first 500ms or so is discarded static constexpr int32_t kNumCallbacksToDrain = 10; int32_t mCountCallbacksToDrain = kNumCallbacksToDrain; - - // Used to reference this instance of class from within the static callback - CSound *pSound; - - QMutex Mutex; }; From d2aa4eede1fbfe197f196ab11d9fec903695d36b Mon Sep 17 00:00:00 2001 From: Julian Santander Date: Mon, 5 Oct 2020 07:14:59 +0200 Subject: [PATCH 2/2] small changes from android_testing branch --- android/sound.cpp | 65 ++++++++++++++++++++++------------------------- android/sound.h | 3 +-- 2 files changed, 31 insertions(+), 37 deletions(-) diff --git a/android/sound.cpp b/android/sound.cpp index 770850b2e6..19bb79c41f 100644 --- a/android/sound.cpp +++ b/android/sound.cpp @@ -32,7 +32,7 @@ CSound::CSound ( void (*fpNewProcessCallback) ( CVector& psData const int iCtrlMIDIChannel, const bool , const QString& ) : - CSoundBase ( "OpenSL", fpNewProcessCallback, arg, iCtrlMIDIChannel ) + CSoundBase ( "Oboe", fpNewProcessCallback, arg, iCtrlMIDIChannel ) { #ifdef ANDROIDDEBUG qInstallMessageHandler(myMessageHandler); @@ -47,14 +47,32 @@ void CSound::setupCommonStreamParams(oboe::AudioStreamBuilder *builder) ->setFormat(oboe::AudioFormat::Float) ->setSharingMode(oboe::SharingMode::Exclusive) ->setChannelCount(oboe::ChannelCount::Stereo) - ->setSampleRate(48000) + ->setSampleRate(SYSTEM_SAMPLE_RATE_HZ) ->setFramesPerCallback(iOpenSLBufferSizeMono) - // ->setSampleRateConversionQuality(oboe::SampleRateConversionQuality::Medium) - ->setPerformanceMode(oboe::PerformanceMode::None); + ->setSampleRateConversionQuality(oboe::SampleRateConversionQuality::Medium) + ->setPerformanceMode(oboe::PerformanceMode::LowLatency); return; } +void CSound::closeStream ( oboe::ManagedStream& stream ) +{ + if ( stream ) + { + oboe::Result requestStopRes = stream->requestStop(); + oboe::Result result = stream->close(); + + if ( result != oboe::Result::OK ) + { + throw CGenErr ( tr ( "Error closing stream: $s", + oboe::convertToText ( result ) ) ); + } + + stream.reset(); + } +} + + void CSound::openStreams() { // Create callback @@ -112,10 +130,7 @@ void CSound::printStreamDetails ( oboe::ManagedStream& stream ) QString sDeviceID = QString::number ( stream->getDeviceId() ); QString sSampleRate = QString::number ( stream->getSampleRate() ); QString sAudioFormat = ( stream->getFormat()==oboe::AudioFormat::I16 ? "I16" : "Float" ); - QString sFramesPerCallback = QString::number ( stream->getFramesPerCallback() ); - //QString sSampleRateConversionQuality = (stream.getSampleRateConversionQuality()==oboe::SampleRateConversionQuality:: - qInfo() << "Stream details: [sDirection: " << sDirection << ", FramesPerBurst: " << sFramesPerBurst << ", BufferSizeInFrames: " << sBufferSizeInFrames << @@ -138,23 +153,6 @@ void CSound::warnIfNotLowLatency ( oboe::ManagedStream& stream, QString streamNa } } -void CSound::closeStream ( oboe::ManagedStream& stream ) -{ - if ( stream ) - { - oboe::Result requestStopRes = stream->requestStop(); - oboe::Result result = stream->close(); - - if ( result != oboe::Result::OK ) - { - throw CGenErr ( tr ( "Error closing stream: $s", - oboe::convertToText ( result ) ) ); - } - - stream.reset(); - } -} - void CSound::closeStreams() { // clean up @@ -172,7 +170,6 @@ void CSound::Start() // finally start the streams so the callback begins, start with inputstream first. mRecordingStream->requestStart(); mPlayStream->requestStart(); - } void CSound::Stop() @@ -198,18 +195,15 @@ int CSound::Init ( const int iNewPrefMonoBufferSize ) vecsTmpInputAudioSndCrdStereo.Init ( iOpenSLBufferSizeStereo ); vecsTmpOutAudioSndCrdStereo.Init ( iOpenSLBufferSizeStereo ); -// TEST -#if ( SYSTEM_SAMPLE_RATE_HZ != 48000 ) -# error "Only a system sample rate of 48 kHz is supported by this module" -#endif - return iOpenSLBufferSizeMono; } // This is the main callback method for when an audio stream is ready to publish data to an output stream -// or has received data on an input stream. As per manual much be very careful not to do anything in this back that -// can cause delays such as sleeping, file processing, allocate memory, etc -oboe::DataCallbackResult CSound::onAudioReady ( oboe::AudioStream* oboeStream, void* audioData, int32_t numFrames ) +// or has received data on an input stream. As per manual much be very careful not to do anything in this back that +// can cause delays such as sleeping, file processing, allocate memory, etc. +oboe::DataCallbackResult CSound::onAudioReady ( oboe::AudioStream* oboeStream, + void* audioData, + int32_t numFrames ) { // only process if we are running if ( ! bRun ) @@ -231,7 +225,8 @@ oboe::DataCallbackResult CSound::onAudioReady ( oboe::AudioStream* oboeStream, v // We're good to start recording now // Take the data from the recording device output buffer and move // it to the vector ready to send up to the server - float *floatData = static_cast ( audioData ); + float* floatData = static_cast ( audioData ); + // Copy recording data to internal vector for ( int frmNum = 0; frmNum < numFrames; ++frmNum ) { @@ -258,7 +253,7 @@ oboe::DataCallbackResult CSound::onAudioReady ( oboe::AudioStream* oboeStream, v // convert to 32 bit const int32_t iCurSam = static_cast ( - vecsTmpInputAudioSndCrdStereo [frmNum * oboeStream->getChannelCount() + channelNum] ); + vecsTmpInputAudioSndCrdStereo[frmNum * oboeStream->getChannelCount() + channelNum] ); vecsTmpOutAudioSndCrdStereo[frmNum * oboeStream->getChannelCount() + channelNum] = ((float) iCurSam) / _MAXSHORT; } diff --git a/android/sound.h b/android/sound.h index d21fdb375b..2ef2af74f1 100644 --- a/android/sound.h +++ b/android/sound.h @@ -25,14 +25,13 @@ #pragma once #include -#include #include "soundbase.h" #include "global.h" #include #include /* Classes ********************************************************************/ -class CSound : public CSoundBase, public oboe::AudioStreamCallback//, public IRenderableAudio, public IRestartable +class CSound : public CSoundBase, public oboe::AudioStreamCallback { public: CSound ( void (*fpNewProcessCallback) ( CVector& psData, void* arg ),