From 4f729b07abd71b6cd304e81b0adc708beb98a622 Mon Sep 17 00:00:00 2001 From: Kai Blaschke Date: Tue, 8 Sep 2026 14:20:11 +0200 Subject: [PATCH] Decouple audio recording from the projectM instance This change uses notifications to notify the projectM wrapper that new audio is available, either from an async recording thread or via the FillBuffer() method. This pattern allows to pass audio in a thread-safe manner to projectM and remove any direct dependency on projectM from the capture implementations. Additionally, using notifications also allows to transparently pass the same audio data to multiple projectM instances. This enables implementation of multi-window support as well as animated preset preview windows. --- src/AudioCapture.cpp | 4 +- src/AudioCaptureImpl_CoreAudioTap.h | 3 +- src/AudioCaptureImpl_CoreAudioTap.mm | 16 +++---- src/AudioCaptureImpl_SDL.cpp | 27 +++++++----- src/AudioCaptureImpl_SDL.h | 9 +--- src/AudioCaptureImpl_WASAPI.cpp | 15 ++++--- src/AudioCaptureImpl_WASAPI.h | 3 +- src/AudioCaptureImpl_macOS.h | 2 +- src/AudioCaptureImpl_macOS.mm | 6 +-- src/ProjectMWrapper.cpp | 28 +++++++++++- src/ProjectMWrapper.h | 10 ++++- .../AudioDataAvailableNotification.cpp | 32 ++++++++++++++ .../AudioDataAvailableNotification.h | 43 +++++++++++++++++++ src/notifications/CMakeLists.txt | 10 ++++- 14 files changed, 163 insertions(+), 45 deletions(-) create mode 100644 src/notifications/AudioDataAvailableNotification.cpp create mode 100644 src/notifications/AudioDataAvailableNotification.h diff --git a/src/AudioCapture.cpp b/src/AudioCapture.cpp index 44d23638..41680527 100644 --- a/src/AudioCapture.cpp +++ b/src/AudioCapture.cpp @@ -19,8 +19,6 @@ void AudioCapture::initialize(Poco::Util::Application& app) { _config = app.config().createView("audio"); - auto& projectMWrapper = app.getSubsystem(); - if (!_impl) { _impl = new AudioCaptureImpl; @@ -31,7 +29,7 @@ void AudioCapture::initialize(Poco::Util::Application& app) PrintDeviceList(deviceList); - _impl->StartRecording(projectMWrapper.ProjectM(), audioDeviceIndex); + _impl->StartRecording(audioDeviceIndex); } void AudioCapture::uninitialize() diff --git a/src/AudioCaptureImpl_CoreAudioTap.h b/src/AudioCaptureImpl_CoreAudioTap.h index 7e80097f..c364816c 100644 --- a/src/AudioCaptureImpl_CoreAudioTap.h +++ b/src/AudioCaptureImpl_CoreAudioTap.h @@ -40,10 +40,9 @@ class CoreAudioTapCapture /** * @brief Starts capturing system audio and forwarding it to projectM. - * @param projectMHandle projectM instance handle that will receive the captured data. * @param audioDeviceIndex Ignored in v1 (single system-audio source). */ - void StartRecording(projectm* projectMHandle, int audioDeviceIndex); + void StartRecording(int audioDeviceIndex); /** * @brief Stops capturing and tears down the tap and aggregate device. diff --git a/src/AudioCaptureImpl_CoreAudioTap.mm b/src/AudioCaptureImpl_CoreAudioTap.mm index 8cdc7ecb..d4e9da89 100644 --- a/src/AudioCaptureImpl_CoreAudioTap.mm +++ b/src/AudioCaptureImpl_CoreAudioTap.mm @@ -1,6 +1,8 @@ #include "AudioCaptureImpl_CoreAudioTap.h" -#include +#include "notifications/AudioDataAvailableNotification.h" + +#include #import #import @@ -26,9 +28,8 @@ return {{-1, _systemAudioDeviceName}}; } -void CoreAudioTapCapture::StartRecording(projectm* projectMHandle, int audioDeviceIndex) +void CoreAudioTapCapture::StartRecording(int audioDeviceIndex) { - _projectMHandle = projectMHandle; _currentAudioDeviceIndex = -1; // Only one source in v1. if (StartTap()) @@ -188,7 +189,6 @@ // 4. Install the IO proc. Runs on Core Audio's realtime thread; keep it allocation- // and lock-free. Core Audio delivers deinterleaved float buffers (one buffer per // channel) for tap aggregates, so forward the first buffer's frames to projectM. - projectm* handle = _projectMHandle; uint32_t channels = _channels; status = AudioDeviceCreateIOProcIDWithBlock( @@ -196,7 +196,7 @@ ^(const AudioTimeStamp* /*inNow*/, const AudioBufferList* inInputData, const AudioTimeStamp* /*inInputTime*/, AudioBufferList* /*outOutputData*/, const AudioTimeStamp* /*inOutputTime*/) { - if (handle == nullptr || inInputData == nullptr || inInputData->mNumberBuffers == 0) + if (inInputData == nullptr || inInputData->mNumberBuffers == 0) { return; } @@ -210,10 +210,10 @@ auto* samples = static_cast(buffer.mData); // mNumberChannels reflects the interleaving of this buffer. uint32_t bufferChannels = buffer.mNumberChannels > 0 ? buffer.mNumberChannels : channels; - unsigned int frameCount = buffer.mDataByteSize / sizeof(float) / bufferChannels; + unsigned int sampleCount = buffer.mDataByteSize / sizeof(float); - projectm_pcm_add_float(handle, samples, frameCount, - static_cast(bufferChannels)); + Poco::NotificationCenter::defaultCenter().postNotification( + new AudioDataAvailableNotification(bufferChannels, samples, sampleCount)); }); if (status != noErr || _ioProcID == nullptr) diff --git a/src/AudioCaptureImpl_SDL.cpp b/src/AudioCaptureImpl_SDL.cpp index 2b9c2b58..f815a49a 100644 --- a/src/AudioCaptureImpl_SDL.cpp +++ b/src/AudioCaptureImpl_SDL.cpp @@ -1,11 +1,11 @@ #include "AudioCaptureImpl_SDL.h" -#include +#include "notifications/AudioDataAvailableNotification.h" -#include +#include +#include AudioCaptureImpl::AudioCaptureImpl() - : _requestedSampleCount(projectm_pcm_get_max_samples()) { auto targetFps = Poco::Util::Application::instance().config().getUInt("projectM.fps", 60); if (targetFps > 0) @@ -50,9 +50,8 @@ std::map AudioCaptureImpl::AudioDeviceList() return deviceList; } -void AudioCaptureImpl::StartRecording(projectm* projectMHandle, int audioDeviceIndex) +void AudioCaptureImpl::StartRecording(int audioDeviceIndex) { - _projectMHandle = projectMHandle; _currentAudioDeviceIndex = audioDeviceIndex; poco_debug_f1(_logger, "Using SDL audio driver \"%s\".", std::string(SDL_GetCurrentAudioDriver())); @@ -84,7 +83,7 @@ void AudioCaptureImpl::NextAudioDevice() // Will wrap around to default capture device (-1). int nextAudioDeviceId = ((_currentAudioDeviceIndex + 2) % (SDL_GetNumAudioDevices(true) + 1)) - 1; - StartRecording(_projectMHandle, nextAudioDeviceId); + StartRecording(nextAudioDeviceId); } void AudioCaptureImpl::AudioDeviceIndex(int index) @@ -93,7 +92,7 @@ void AudioCaptureImpl::AudioDeviceIndex(int index) { StopRecording(); _currentAudioDeviceIndex = index; - StartRecording(_projectMHandle, index); + StartRecording(index); } } @@ -153,10 +152,18 @@ bool AudioCaptureImpl::OpenAudioDevice() void AudioCaptureImpl::AudioInputCallback(void* userData, unsigned char* stream, int len) { poco_assert_dbg(userData); + + if (len < sizeof(float)) + { + return; + } + auto instance = reinterpret_cast(userData); - unsigned int samples = len / sizeof(float) / instance->_channels; + unsigned int samples = len / sizeof(float); - projectm_pcm_add_float(instance->_projectMHandle, reinterpret_cast(stream), samples, - static_cast(instance->_channels)); + Poco::NotificationCenter::defaultCenter().postNotification( + new AudioDataAvailableNotification(instance->_channels, + reinterpret_cast(stream), + samples)); } diff --git a/src/AudioCaptureImpl_SDL.h b/src/AudioCaptureImpl_SDL.h index f951576a..9f8e7c61 100644 --- a/src/AudioCaptureImpl_SDL.h +++ b/src/AudioCaptureImpl_SDL.h @@ -5,7 +5,6 @@ #include #include -#include class projectm; @@ -29,11 +28,10 @@ class AudioCaptureImpl /** * @brief Starts audio capturing with the first available device. - * @param projectMHandle projectM instance handle that will receive the captured data. * @param audioDeviceIndex The initial audio device ID to capture from. Use -1 to select the implementation's * default device. */ - void StartRecording(projectm* projectMHandle, int audioDeviceIndex); + void StartRecording(int audioDeviceIndex); /** * @brief Stops audio recording. @@ -66,9 +64,7 @@ class AudioCaptureImpl /** * @brief Asks the capture client to fill projectM's audio buffer for the next frame. * - * As of now, SDL uses async callbacks to directly fill projectM's audio buffer. - * - * @todo Store audio samples internally and push them to projectM when requested. + * SDL uses async callbacks to directly fill the staging audio buffer(s) as soon as new data is available. */ void FillBuffer(){}; @@ -90,7 +86,6 @@ class AudioCaptureImpl */ static void AudioInputCallback(void* userData, unsigned char* stream, int len); - projectm* _projectMHandle{nullptr}; //!< Handle if the projectM instance that will receive the audio data. int32_t _currentAudioDeviceIndex{-1}; //!< Currently selected audio device index. SDL_AudioDeviceID _currentAudioDeviceID{0}; //!< Device ID of the currently opened audio device. uint32_t _channels{2}; diff --git a/src/AudioCaptureImpl_WASAPI.cpp b/src/AudioCaptureImpl_WASAPI.cpp index 94aba1fe..3cdeac86 100644 --- a/src/AudioCaptureImpl_WASAPI.cpp +++ b/src/AudioCaptureImpl_WASAPI.cpp @@ -1,9 +1,10 @@ #include "AudioCaptureImpl_WASAPI.h" -#include +#include "notifications/AudioDataAvailableNotification.h" #include +#include #include #include #include @@ -42,9 +43,8 @@ std::map AudioCaptureImpl::AudioDeviceList() return deviceList; } -void AudioCaptureImpl::StartRecording(projectm* projectMHandle, int audioDeviceIndex) +void AudioCaptureImpl::StartRecording(int audioDeviceIndex) { - _projectMHandle = projectMHandle; _currentAudioDeviceIndex = audioDeviceIndex; _isCapturing = true; @@ -74,7 +74,7 @@ void AudioCaptureImpl::NextAudioDevice() // Will wrap around to loopback capture device (-1). int nextAudioDeviceId = ((_currentAudioDeviceIndex + 2) % (static_cast(captureDevices.size()) + 1)) - 1; - StartRecording(_projectMHandle, nextAudioDeviceId); + StartRecording(nextAudioDeviceId); } void AudioCaptureImpl::AudioDeviceIndex(int index) @@ -87,7 +87,7 @@ void AudioCaptureImpl::AudioDeviceIndex(int index) { _currentAudioDeviceIndex = index; StopRecording(); - StartRecording(_projectMHandle, index); + StartRecording(index); } } @@ -447,7 +447,10 @@ void AudioCaptureImpl::CaptureThread() if (framesAvailable > 0 && data != nullptr) { - projectm_pcm_add_float(_projectMHandle, reinterpret_cast(data), framesAvailable, static_cast(_channels)); + Poco::NotificationCenter::defaultCenter().postNotification( + new AudioDataAvailableNotification(_channels, + reinterpret_cast(data), + framesAvailable * _channels)); } _audioCaptureClient->ReleaseBuffer(framesAvailable); diff --git a/src/AudioCaptureImpl_WASAPI.h b/src/AudioCaptureImpl_WASAPI.h index 9cbd165c..eea2adff 100644 --- a/src/AudioCaptureImpl_WASAPI.h +++ b/src/AudioCaptureImpl_WASAPI.h @@ -45,11 +45,10 @@ class AudioCaptureImpl : public IMMNotificationClient /** * @brief Starts audio capturing with the first available device. - * @param projectMHandle projectM instance handle that will receive the captured data. * @param audioDeviceIndex The initial audio device ID to capture from. Use -1 to select the implementation's * default device. */ - void StartRecording(projectm* projectMHandle, int audioDeviceIndex); + void StartRecording(int audioDeviceIndex); /** * @brief Stops audio recording. diff --git a/src/AudioCaptureImpl_macOS.h b/src/AudioCaptureImpl_macOS.h index 57300b24..f59f5914 100644 --- a/src/AudioCaptureImpl_macOS.h +++ b/src/AudioCaptureImpl_macOS.h @@ -30,7 +30,7 @@ class AudioCaptureImpl std::map AudioDeviceList(); - void StartRecording(projectm* projectMHandle, int audioDeviceIndex); + void StartRecording(int audioDeviceIndex); void StopRecording(); diff --git a/src/AudioCaptureImpl_macOS.mm b/src/AudioCaptureImpl_macOS.mm index b4f07090..50ca6ab6 100644 --- a/src/AudioCaptureImpl_macOS.mm +++ b/src/AudioCaptureImpl_macOS.mm @@ -42,14 +42,14 @@ return _sdl->AudioDeviceList(); } -void AudioCaptureImpl::StartRecording(projectm* projectMHandle, int audioDeviceIndex) +void AudioCaptureImpl::StartRecording(int audioDeviceIndex) { if (_tap) { - _tap->StartRecording(projectMHandle, audioDeviceIndex); + _tap->StartRecording(audioDeviceIndex); return; } - _sdl->StartRecording(projectMHandle, audioDeviceIndex); + _sdl->StartRecording(audioDeviceIndex); } void AudioCaptureImpl::StopRecording() diff --git a/src/ProjectMWrapper.cpp b/src/ProjectMWrapper.cpp index 75894f12..f176a348 100644 --- a/src/ProjectMWrapper.cpp +++ b/src/ProjectMWrapper.cpp @@ -108,6 +108,7 @@ void ProjectMWrapper::initialize(Poco::Util::Application& app) } Poco::NotificationCenter::defaultCenter().addObserver(_playbackControlNotificationObserver); + Poco::NotificationCenter::defaultCenter().addObserver(_audioDataAvailableNotificationObserver); // Observe user configuration changes (set via the settings window) _userConfig->propertyChanged += Poco::delegate(this, &ProjectMWrapper::OnConfigurationPropertyChanged); @@ -119,6 +120,7 @@ void ProjectMWrapper::uninitialize() _userConfig->propertyRemoved -= Poco::delegate(this, &ProjectMWrapper::OnConfigurationPropertyRemoved); _userConfig->propertyChanged -= Poco::delegate(this, &ProjectMWrapper::OnConfigurationPropertyChanged); Poco::NotificationCenter::defaultCenter().removeObserver(_playbackControlNotificationObserver); + Poco::NotificationCenter::defaultCenter().removeObserver(_audioDataAvailableNotificationObserver); if (_projectM) { @@ -153,7 +155,7 @@ void ProjectMWrapper::UpdateRealFPS(float fps) projectm_set_fps(_projectM, static_cast(std::round(fps))); } -void ProjectMWrapper::RenderFrame() const +void ProjectMWrapper::RenderFrame() { glClearColor(0.0, 0.0, 0.0, 0.0); glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); @@ -167,6 +169,16 @@ void ProjectMWrapper::RenderFrame() const projectm_set_mesh_size(_projectM, _projectMConfigView->getInt("meshX", 220), _projectMConfigView->getInt("meshY", 125)); } + // Add new audio data to the instance from the staging buffer + { + Poco::ScopedLock lock(_audioBufferMutex); + if (!_audioStagingBuffer.empty()) + { + projectm_pcm_add_float(_projectM, _audioStagingBuffer.data(), _audioStagingBuffer.size() / _audioChannels, static_cast(_audioChannels)); + _audioStagingBuffer.clear(); + } + } + projectm_opengl_render_frame(_projectM); } @@ -263,6 +275,20 @@ void ProjectMWrapper::PlaybackControlNotificationHandler(const Poco::AutoPtr& notification) +{ + Poco::ScopedLock lock(_audioBufferMutex); + + // Clear existing buffer data if channel count differs, e.g. if audio device has changed + if (_audioChannels != notification->Channels()) + { + _audioChannels = notification->Channels(); + _audioStagingBuffer.clear(); + } + + std::copy(notification->Samples().cbegin(), notification->Samples().cend(), std::back_inserter(_audioStagingBuffer)); +} + std::vector ProjectMWrapper::GetPathListWithDefault(const std::string& baseKey, const std::string& defaultPath) { using Poco::Util::AbstractConfiguration; diff --git a/src/ProjectMWrapper.h b/src/ProjectMWrapper.h index 3493dc41..c8eb89d8 100644 --- a/src/ProjectMWrapper.h +++ b/src/ProjectMWrapper.h @@ -1,5 +1,6 @@ #pragma once +#include "notifications/AudioDataAvailableNotification.h" #include "notifications/PlaybackControlNotification.h" #include @@ -37,7 +38,7 @@ class ProjectMWrapper : public Poco::Util::Subsystem /** * Renders a single projectM frame. */ - void RenderFrame() const; + void RenderFrame(); /** * @brief Returns the targeted FPS value. @@ -91,6 +92,8 @@ class ProjectMWrapper : public Poco::Util::Subsystem void PlaybackControlNotificationHandler(const Poco::AutoPtr& notification); + void AudioDataAvailableNotificationHandler(const Poco::AutoPtr& notification); + std::vector GetPathListWithDefault(const std::string& baseKey, const std::string& defaultPath); /** @@ -111,7 +114,12 @@ class ProjectMWrapper : public Poco::Util::Subsystem projectm_handle _projectM{nullptr}; //!< Pointer to the projectM instance used by the application. projectm_playlist_handle _playlist{nullptr}; //!< Pointer to the projectM playlist manager instance. + uint32_t _audioChannels{0}; //!< Number of audio channels of the current capture device. + std::vector _audioStagingBuffer; //!< Buffer which receives audio data from the capture implementation. + mutable Poco::Mutex _audioBufferMutex; //!< Mutex protecting access to the audio staging buffer. + Poco::NObserver _playbackControlNotificationObserver{*this, &ProjectMWrapper::PlaybackControlNotificationHandler}; + Poco::NObserver _audioDataAvailableNotificationObserver{*this, &ProjectMWrapper::AudioDataAvailableNotificationHandler}; Poco::Logger& _logger{Poco::Logger::get("SDLRenderingWindow")}; //!< The class logger. }; diff --git a/src/notifications/AudioDataAvailableNotification.cpp b/src/notifications/AudioDataAvailableNotification.cpp new file mode 100644 index 00000000..2b9a341c --- /dev/null +++ b/src/notifications/AudioDataAvailableNotification.cpp @@ -0,0 +1,32 @@ +#include "AudioDataAvailableNotification.h" + +#include + +#include + +std::string AudioDataAvailableNotification::name() const +{ + return "AudioDataAvailableNotification"; +} + +AudioDataAvailableNotification::AudioDataAvailableNotification(uint32_t channels, const float* samples, const uint32_t sampleCount) + : _channels(channels) +{ + poco_assert(channels > 0); + poco_assert(samples != nullptr); + poco_assert(sampleCount != 0); + poco_assert(sampleCount % channels == 0); + + _samples.resize(sampleCount); + std::memcpy(_samples.data(), samples, sampleCount * sizeof(float)); +} + +uint32_t AudioDataAvailableNotification::Channels() const +{ + return _channels; +} + +const std::vector& AudioDataAvailableNotification::Samples() const +{ + return _samples; +} \ No newline at end of file diff --git a/src/notifications/AudioDataAvailableNotification.h b/src/notifications/AudioDataAvailableNotification.h new file mode 100644 index 00000000..4f23bc0c --- /dev/null +++ b/src/notifications/AudioDataAvailableNotification.h @@ -0,0 +1,43 @@ +#pragma once + +#include + +#include +#include + +/** + * @brief Passes new audio data to all active projectM instances. + * @note Channels must be >0 and samples must contain at least one audio frame/sample. + */ +class AudioDataAvailableNotification : public Poco::Notification +{ +public: + std::string name() const override; + + AudioDataAvailableNotification() = delete; + + /** + * @brief Creates a new AudioDataAvailableNotification instance + * The number of @a samples must be divisible by @a channels. + * @param channels The channel count of the current capture device. + * @param samples The new audio samples captured from the active audio device. + * @param sampleCount The number of float samples passed to @a samples. + */ + explicit AudioDataAvailableNotification(uint32_t channels, const float* samples, uint32_t sampleCount); + + /** + * Returns the number of audio channels of the recording device. + * @return The number of audio channels of the recording device. + */ + [[nodiscard]] uint32_t Channels() const; + + /** + * Returns the new audio samples captured from the active audio device. + * @return The new audio samples captured from the active audio device. + */ + [[nodiscard]] const std::vector& Samples() const; + +private: + uint32_t _channels; //!< Number of audio channels of the capture device. + std::vector _samples; //!< The new audio samples, containing frames x channels floats. +}; diff --git a/src/notifications/CMakeLists.txt b/src/notifications/CMakeLists.txt index 582f24ce..3064db30 100644 --- a/src/notifications/CMakeLists.txt +++ b/src/notifications/CMakeLists.txt @@ -1,7 +1,15 @@ add_library(ProjectMSDL-Notifications STATIC + AudioDataAvailableNotification.cpp + AudioDataAvailableNotification.h DisplayToastNotification.cpp DisplayToastNotification.h - QuitNotification.cpp QuitNotification.h PlaybackControlNotification.cpp PlaybackControlNotification.h UpdateWindowTitleNotification.cpp UpdateWindowTitleNotification.h) + PlaybackControlNotification.cpp + PlaybackControlNotification.h + QuitNotification.cpp + QuitNotification.h + UpdateWindowTitleNotification.cpp + UpdateWindowTitleNotification.h + ) target_include_directories(ProjectMSDL-Notifications PRIVATE