From 544ee9add4fd70d9fc4116fa5179a11bf8575f49 Mon Sep 17 00:00:00 2001 From: Sasa Mudri Date: Wed, 29 Apr 2026 15:20:40 +0200 Subject: [PATCH 1/7] Add first-video-frame-callback signal implementation. --- media/client/ipc/include/MediaPipelineIpc.h | 7 ++ .../ipc/interface/IMediaPipelineIpcClient.h | 9 ++ media/client/ipc/source/MediaPipelineIpc.cpp | 16 +++ media/client/main/include/MediaPipeline.h | 2 + media/client/main/source/MediaPipeline.cpp | 11 ++ media/public/include/IMediaPipelineClient.h | 9 ++ media/server/gstplayer/CMakeLists.txt | 1 + .../gstplayer/include/GstGenericPlayer.h | 1 + .../include/IGstGenericPlayerPrivate.h | 5 + media/server/gstplayer/include/Utils.h | 2 + .../include/tasks/IGenericPlayerTaskFactory.h | 13 ++ .../tasks/generic/FirstFrameReceived.h | 47 ++++++++ .../tasks/generic/GenericPlayerTaskFactory.h | 2 + .../interface/IGstGenericPlayerClient.h | 9 ++ .../gstplayer/source/GstGenericPlayer.cpp | 8 ++ media/server/gstplayer/source/Utils.cpp | 27 +++++ .../tasks/generic/FirstFrameReceived.cpp | 47 ++++++++ .../generic/GenericPlayerTaskFactory.cpp | 8 ++ .../source/tasks/generic/SetupElement.cpp | 22 ++++ .../server/ipc/include/MediaPipelineClient.h | 1 + .../server/ipc/source/MediaPipelineClient.cpp | 11 ++ .../include/MediaPipelineServerInternal.h | 2 + .../source/MediaPipelineServerInternal.cpp | 22 ++++ proto/mediapipelinemodule.proto | 13 ++ .../MediaPipelineClientMock.h | 1 + .../server/stubs/ClientStub.cpp | 16 +-- .../tests/mediaPipeline/UnderflowTest.cpp | 2 +- .../base/MediaPipelineIpcTestBase.cpp | 10 ++ .../base/MediaPipelineIpcTestBase.h | 2 + .../mocks/ipc/MediaPipelineIpcClientMock.h | 1 + .../common/GenericTasksTestsBase.cpp | 112 +++++++++++++++++- .../common/GenericTasksTestsBase.h | 5 + .../common/GenericTasksTestsContext.h | 1 + .../GenericPlayerTaskFactoryTest.cpp | 8 ++ .../tasksTests/SetupElementTest.cpp | 9 ++ .../gstplayer/GenericPlayerTaskFactoryMock.h | 3 + .../gstplayer/GstGenericPlayerClientMock.h | 1 + .../gstplayer/GstGenericPlayerPrivateMock.h | 1 + 38 files changed, 452 insertions(+), 15 deletions(-) create mode 100644 media/server/gstplayer/include/tasks/generic/FirstFrameReceived.h create mode 100644 media/server/gstplayer/source/tasks/generic/FirstFrameReceived.cpp diff --git a/media/client/ipc/include/MediaPipelineIpc.h b/media/client/ipc/include/MediaPipelineIpc.h index b93cd98fc..ef5ea0ea6 100644 --- a/media/client/ipc/include/MediaPipelineIpc.h +++ b/media/client/ipc/include/MediaPipelineIpc.h @@ -215,6 +215,13 @@ class MediaPipelineIpc : public IMediaPipelineIpc, public IpcModule */ void onBufferUnderflow(const std::shared_ptr &event); + /** + * @brief Handler for a first frame received notification from the server. + * + * @param[in] event : The first frame received event structure. + */ + void onFirstFrameReceived(const std::shared_ptr &event); + /** * @brief Handler for a playback error notification from the server. * diff --git a/media/client/ipc/interface/IMediaPipelineIpcClient.h b/media/client/ipc/interface/IMediaPipelineIpcClient.h index e393152da..b4d41f556 100644 --- a/media/client/ipc/interface/IMediaPipelineIpcClient.h +++ b/media/client/ipc/interface/IMediaPipelineIpcClient.h @@ -96,6 +96,15 @@ class IMediaPipelineIpcClient */ virtual void notifyBufferUnderflow(int32_t sourceId) = 0; + /** + * @brief Notifies the client that the first frame has been received. + * + * Notification shall be sent whenever a video/audio first frame is received + * + * @param[in] sourceId : The id of the source that received the first frame + */ + virtual void notifyFirstFrameReceived(int32_t sourceId) = 0; + /** * @brief Notifies the client that a non-fatal error has occurred in the player. * diff --git a/media/client/ipc/source/MediaPipelineIpc.cpp b/media/client/ipc/source/MediaPipelineIpc.cpp index 3949629ed..a376ed7fb 100644 --- a/media/client/ipc/source/MediaPipelineIpc.cpp +++ b/media/client/ipc/source/MediaPipelineIpc.cpp @@ -158,6 +158,13 @@ bool MediaPipelineIpc::subscribeToEvents(const std::shared_ptr &i return false; m_eventTags.push_back(eventTag); + eventTag = ipcChannel->subscribe( + [this](const std::shared_ptr &event) + { m_eventThread->add(&MediaPipelineIpc::onFirstFrameReceived, this, event); }); + if (eventTag < 0) + return false; + m_eventTags.push_back(eventTag); + eventTag = ipcChannel->subscribe( [this](const std::shared_ptr &event) { m_eventThread->add(&MediaPipelineIpc::onPlaybackError, this, event); }); @@ -1533,6 +1540,15 @@ void MediaPipelineIpc::onBufferUnderflow(const std::shared_ptr &event) +{ + // Ignore event if not for this session + if (event->session_id() == m_sessionId) + { + m_mediaPipelineIpcClient->notifyFirstFrameReceived(event->source_id()); + } +} + void MediaPipelineIpc::onPlaybackError(const std::shared_ptr &event) { // Ignore event if not for this session diff --git a/media/client/main/include/MediaPipeline.h b/media/client/main/include/MediaPipeline.h index 14dad7f81..6c91b8bfd 100644 --- a/media/client/main/include/MediaPipeline.h +++ b/media/client/main/include/MediaPipeline.h @@ -156,6 +156,8 @@ class MediaPipeline : public IMediaPipelineAndIControlClient, public IMediaPipel void notifyBufferUnderflow(int32_t sourceId) override; + void notifyFirstFrameReceived(int32_t sourceId) override; + void notifyPlaybackError(int32_t sourceId, PlaybackError error) override; void notifySourceFlushed(int32_t sourceId) override; diff --git a/media/client/main/source/MediaPipeline.cpp b/media/client/main/source/MediaPipeline.cpp index ed0506432..d61317c6c 100644 --- a/media/client/main/source/MediaPipeline.cpp +++ b/media/client/main/source/MediaPipeline.cpp @@ -877,6 +877,17 @@ void MediaPipeline::notifyBufferUnderflow(int32_t sourceId) } } +void MediaPipeline::notifyFirstFrameReceived(int32_t sourceId) +{ + RIALTO_CLIENT_LOG_DEBUG("entry:"); + + std::shared_ptr client = m_mediaPipelineClient.lock(); + if (client) + { + client->notifyFirstFrameReceived(sourceId); + } +} + void MediaPipeline::notifyPlaybackError(int32_t sourceId, PlaybackError error) { RIALTO_CLIENT_LOG_DEBUG("entry:"); diff --git a/media/public/include/IMediaPipelineClient.h b/media/public/include/IMediaPipelineClient.h index 0cbdc2af7..3334b578c 100644 --- a/media/public/include/IMediaPipelineClient.h +++ b/media/public/include/IMediaPipelineClient.h @@ -197,6 +197,15 @@ class IMediaPipelineClient */ virtual void notifyBufferUnderflow(int32_t sourceId) = 0; + /** + * @brief Notifies the client that the first frame has been received. + * + * Notification shall be sent whenever a video/audio first frame is received + * + * @param[in] sourceId : The id of the source that received the first frame + */ + virtual void notifyFirstFrameReceived(int32_t sourceId) = 0; + /** * @brief Notifies the client that a non-fatal error has occurred in the player. * diff --git a/media/server/gstplayer/CMakeLists.txt b/media/server/gstplayer/CMakeLists.txt index 11d7fc63b..0e147af6b 100644 --- a/media/server/gstplayer/CMakeLists.txt +++ b/media/server/gstplayer/CMakeLists.txt @@ -39,6 +39,7 @@ add_library( source/tasks/generic/DeepElementAdded.cpp source/tasks/generic/EnoughData.cpp source/tasks/generic/Eos.cpp + source/tasks/generic/FirstFrameReceived.cpp source/tasks/generic/FinishSetupSource.cpp source/tasks/generic/Flush.cpp source/tasks/generic/GenericPlayerTaskFactory.cpp diff --git a/media/server/gstplayer/include/GstGenericPlayer.h b/media/server/gstplayer/include/GstGenericPlayer.h index 03436d67b..bdd3d4a39 100644 --- a/media/server/gstplayer/include/GstGenericPlayer.h +++ b/media/server/gstplayer/include/GstGenericPlayer.h @@ -154,6 +154,7 @@ class GstGenericPlayer : public IGstGenericPlayer, public IGstGenericPlayerPriva void scheduleEnoughData(GstAppSrc *src) override; void scheduleAudioUnderflow() override; void scheduleVideoUnderflow() override; + void scheduleFirstVideoFrameReceived() override; void scheduleAllSourcesAttached() override; bool setVideoSinkRectangle() override; bool setImmediateOutput() override; diff --git a/media/server/gstplayer/include/IGstGenericPlayerPrivate.h b/media/server/gstplayer/include/IGstGenericPlayerPrivate.h index fb48a2816..139078515 100644 --- a/media/server/gstplayer/include/IGstGenericPlayerPrivate.h +++ b/media/server/gstplayer/include/IGstGenericPlayerPrivate.h @@ -61,6 +61,11 @@ class IGstGenericPlayerPrivate */ virtual void scheduleVideoUnderflow() = 0; + /** + * @brief Schedules first video frame received task. Called by the worker thread. + */ + virtual void scheduleFirstVideoFrameReceived() = 0; + /** * @brief Schedules all sources attached task. Called by the worker thread. */ diff --git a/media/server/gstplayer/include/Utils.h b/media/server/gstplayer/include/Utils.h index 7351ec903..ef4c8bb14 100644 --- a/media/server/gstplayer/include/Utils.h +++ b/media/server/gstplayer/include/Utils.h @@ -41,6 +41,8 @@ bool isAudio(const firebolt::rialto::wrappers::IGstWrapper &gstWrapper, GstEleme bool isVideo(const firebolt::rialto::wrappers::IGstWrapper &gstWrapper, GstElement *element); std::optional getUnderflowSignalName(const firebolt::rialto::wrappers::IGlibWrapper &glibWrapper, GstElement *element); +std::optional getFirstVideoFrameSignalName(const firebolt::rialto::wrappers::IGlibWrapper &glibWrapper, + GstElement *element); GstCaps *createCapsFromMediaSource(const std::shared_ptr &gstWrapper, const std::shared_ptr &glibWrapper, const std::unique_ptr &source); diff --git a/media/server/gstplayer/include/tasks/IGenericPlayerTaskFactory.h b/media/server/gstplayer/include/tasks/IGenericPlayerTaskFactory.h index f8de67cb4..f33864276 100644 --- a/media/server/gstplayer/include/tasks/IGenericPlayerTaskFactory.h +++ b/media/server/gstplayer/include/tasks/IGenericPlayerTaskFactory.h @@ -376,6 +376,19 @@ class IGenericPlayerTaskFactory virtual std::unique_ptr createUnderflow(GenericPlayerContext &context, IGstGenericPlayerPrivate &player, bool underflowEnabled, MediaSourceType sourceType) const = 0; + /** + * @brief Creates an FirstFrameReceived task. + * + * @param[in] context : The GstGenericPlayer context + * @param[in] player : The GstPlayer instance + * @param[in] sourceType : Source type (audio or video). + * + * @retval the new FirstFrameReceived task instance. + */ + virtual std::unique_ptr createFirstFrameReceived(GenericPlayerContext &context, + IGstGenericPlayerPrivate &player, + MediaSourceType sourceType) const = 0; + /** * @brief Creates an UpdatePlaybackGroup task. * diff --git a/media/server/gstplayer/include/tasks/generic/FirstFrameReceived.h b/media/server/gstplayer/include/tasks/generic/FirstFrameReceived.h new file mode 100644 index 000000000..38802b8ef --- /dev/null +++ b/media/server/gstplayer/include/tasks/generic/FirstFrameReceived.h @@ -0,0 +1,47 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2026 Sky UK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FIREBOLT_RIALTO_SERVER_TASKS_GENERIC_FIRST_FRAME_RECEIVED_H_ +#define FIREBOLT_RIALTO_SERVER_TASKS_GENERIC_FIRST_FRAME_RECEIVED_H_ + +#include "GenericPlayerContext.h" +#include "IGstGenericPlayerClient.h" +#include "IGstGenericPlayerPrivate.h" +#include "IPlayerTask.h" + +namespace firebolt::rialto::server::tasks::generic +{ +class FirstFrameReceived : public IPlayerTask +{ +public: + FirstFrameReceived(GenericPlayerContext &context, IGstGenericPlayerPrivate &player, IGstGenericPlayerClient *client, + MediaSourceType sourceType); + ~FirstFrameReceived() override; + + void execute() const override; + +private: + GenericPlayerContext &m_context; + IGstGenericPlayerPrivate &m_player; + IGstGenericPlayerClient *m_gstPlayerClient; + MediaSourceType m_sourceType; +}; +} // namespace firebolt::rialto::server::tasks::generic + +#endif // FIREBOLT_RIALTO_SERVER_TASKS_GENERIC_FIRST_FRAME_RECEIVED_H_ diff --git a/media/server/gstplayer/include/tasks/generic/GenericPlayerTaskFactory.h b/media/server/gstplayer/include/tasks/generic/GenericPlayerTaskFactory.h index aa9a06e55..22fd2d082 100644 --- a/media/server/gstplayer/include/tasks/generic/GenericPlayerTaskFactory.h +++ b/media/server/gstplayer/include/tasks/generic/GenericPlayerTaskFactory.h @@ -99,6 +99,8 @@ class GenericPlayerTaskFactory : public IGenericPlayerTaskFactory IGstGenericPlayerPrivate &player) const override; std::unique_ptr createUnderflow(GenericPlayerContext &context, IGstGenericPlayerPrivate &player, bool underflowEnable, MediaSourceType sourceType) const override; + std::unique_ptr createFirstFrameReceived(GenericPlayerContext &context, IGstGenericPlayerPrivate &player, + MediaSourceType sourceType) const override; std::unique_ptr createUpdatePlaybackGroup(GenericPlayerContext &context, IGstGenericPlayerPrivate &player, GstElement *typefind, const GstCaps *caps) const override; diff --git a/media/server/gstplayer/interface/IGstGenericPlayerClient.h b/media/server/gstplayer/interface/IGstGenericPlayerClient.h index 850f727a4..ab645db8c 100644 --- a/media/server/gstplayer/interface/IGstGenericPlayerClient.h +++ b/media/server/gstplayer/interface/IGstGenericPlayerClient.h @@ -150,6 +150,15 @@ class IGstGenericPlayerClient */ virtual void notifyBufferUnderflow(MediaSourceType mediaSourceType) = 0; + /** + * @brief Notifies the client that the first frame has been received. + * + * Notification shall be sent whenever a video/audio first frame is received + * + * @param[in] mediaSourceType : The type of the source that received the first frame + */ + virtual void notifyFirstFrameReceived(MediaSourceType mediaSourceType) = 0; + /** * @brief Notifies the client that a non-fatal error has occurred in the player. * diff --git a/media/server/gstplayer/source/GstGenericPlayer.cpp b/media/server/gstplayer/source/GstGenericPlayer.cpp index b3bc8fa99..c6634e929 100644 --- a/media/server/gstplayer/source/GstGenericPlayer.cpp +++ b/media/server/gstplayer/source/GstGenericPlayer.cpp @@ -1674,6 +1674,14 @@ void GstGenericPlayer::scheduleVideoUnderflow() } } +void GstGenericPlayer::scheduleFirstVideoFrameReceived() +{ + if (m_workerThread) + { + m_workerThread->enqueueTask(m_taskFactory->createFirstFrameReceived(m_context, *this, MediaSourceType::VIDEO)); + } +} + void GstGenericPlayer::scheduleAllSourcesAttached() { allSourcesAttached(); diff --git a/media/server/gstplayer/source/Utils.cpp b/media/server/gstplayer/source/Utils.cpp index 520778de4..85b986d64 100644 --- a/media/server/gstplayer/source/Utils.cpp +++ b/media/server/gstplayer/source/Utils.cpp @@ -29,6 +29,7 @@ namespace { const char *underflowSignals[]{"buffer-underflow-callback", "vidsink-underflow-callback", "underrun-callback"}; +const char *firstVideoFrameSignals[]{"first-video-frame-callback"}; bool isType(const firebolt::rialto::wrappers::IGstWrapper &gstWrapper, GstElement *element, GstElementFactoryListType type) { @@ -125,6 +126,32 @@ std::optional getUnderflowSignalName(const firebolt::rialto::wrappe return std::nullopt; } +std::optional getFirstVideoFrameSignalName(const firebolt::rialto::wrappers::IGlibWrapper &glibWrapper, + GstElement *element) +{ + GType type = glibWrapper.gObjectType(element); + guint nsignals{0}; + guint *signals = glibWrapper.gSignalListIds(type, &nsignals); + + for (guint i = 0; i < nsignals; i++) + { + GSignalQuery query; + glibWrapper.gSignalQuery(signals[i], &query); + const auto signalNameIt = std::find_if(std::begin(firstVideoFrameSignals), std::end(firstVideoFrameSignals), + [&](const auto *signalName) + { return strcmp(signalName, query.signal_name) == 0; }); + + if (std::end(firstVideoFrameSignals) != signalNameIt) + { + glibWrapper.gFree(signals); + return std::string(*signalNameIt); + } + } + glibWrapper.gFree(signals); + + return std::nullopt; +} + GstCaps *createCapsFromMediaSource(const std::shared_ptr &gstWrapper, const std::shared_ptr &glibWrapper, const std::unique_ptr &source) diff --git a/media/server/gstplayer/source/tasks/generic/FirstFrameReceived.cpp b/media/server/gstplayer/source/tasks/generic/FirstFrameReceived.cpp new file mode 100644 index 000000000..59252c127 --- /dev/null +++ b/media/server/gstplayer/source/tasks/generic/FirstFrameReceived.cpp @@ -0,0 +1,47 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2026 Sky UK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tasks/generic/FirstFrameReceived.h" +#include "RialtoServerLogging.h" +#include "TypeConverters.h" + +namespace firebolt::rialto::server::tasks::generic +{ +FirstFrameReceived::FirstFrameReceived(GenericPlayerContext &context, IGstGenericPlayerPrivate &player, + IGstGenericPlayerClient *client, MediaSourceType sourceType) + : m_context{context}, m_player{player}, m_gstPlayerClient{client}, m_sourceType{sourceType} +{ + RIALTO_SERVER_LOG_DEBUG("Constructing FirstFrameReceived"); +} + +FirstFrameReceived::~FirstFrameReceived() +{ + RIALTO_SERVER_LOG_DEBUG("FirstFrameReceived finished"); +} + +void FirstFrameReceived::execute() const +{ + RIALTO_SERVER_LOG_WARN("Executing FirstFrameReceived for %s source", common::convertMediaSourceType(m_sourceType)); + + if (m_gstPlayerClient) + { + m_gstPlayerClient->notifyFirstFrameReceived(m_sourceType); + } +} +} // namespace firebolt::rialto::server::tasks::generic diff --git a/media/server/gstplayer/source/tasks/generic/GenericPlayerTaskFactory.cpp b/media/server/gstplayer/source/tasks/generic/GenericPlayerTaskFactory.cpp index 318f04379..1ad49eaef 100644 --- a/media/server/gstplayer/source/tasks/generic/GenericPlayerTaskFactory.cpp +++ b/media/server/gstplayer/source/tasks/generic/GenericPlayerTaskFactory.cpp @@ -25,6 +25,7 @@ #include "tasks/generic/EnoughData.h" #include "tasks/generic/Eos.h" #include "tasks/generic/FinishSetupSource.h" +#include "tasks/generic/FirstFrameReceived.h" #include "tasks/generic/Flush.h" #include "tasks/generic/HandleBusMessage.h" #include "tasks/generic/NeedData.h" @@ -264,6 +265,13 @@ std::unique_ptr GenericPlayerTaskFactory::createUnderflow(GenericPl return std::make_unique(context, player, m_client, underflowEnabled, sourceType); } +std::unique_ptr GenericPlayerTaskFactory::createFirstFrameReceived(GenericPlayerContext &context, + IGstGenericPlayerPrivate &player, + MediaSourceType sourceType) const +{ + return std::make_unique(context, player, m_client, sourceType); +} + std::unique_ptr GenericPlayerTaskFactory::createUpdatePlaybackGroup(GenericPlayerContext &context, IGstGenericPlayerPrivate &player, GstElement *typefind, diff --git a/media/server/gstplayer/source/tasks/generic/SetupElement.cpp b/media/server/gstplayer/source/tasks/generic/SetupElement.cpp index d3fd0f12a..0b598d2ce 100644 --- a/media/server/gstplayer/source/tasks/generic/SetupElement.cpp +++ b/media/server/gstplayer/source/tasks/generic/SetupElement.cpp @@ -61,6 +61,19 @@ void videoUnderflowCallback(GstElement *object, guint fifoDepth, gpointer queueD player->scheduleVideoUnderflow(); } +/** + * @brief Callback for first video frame event from sink. Called by the Gstreamer thread. + * + * @param[in] object : the object that emitted the signal + * @param[in] self : The pointer to IGstGenericPlayerPrivate + */ +void firstVideoFrameCallback(GstElement *object, guint fifoDepth, gpointer queueDepth, gpointer self) +{ + firebolt::rialto::server::IGstGenericPlayerPrivate *player = + static_cast(self); + player->scheduleFirstVideoFrameReceived(); +} + /** * @brief Callback for a autovideosink when a child has been added to the sink. * @@ -269,6 +282,15 @@ void SetupElement::execute() const G_CALLBACK(videoUnderflowCallback), &m_player); } } + + std::optional firstVideoFrameSignalName = getFirstVideoFrameSignalName(*m_glibWrapper, m_element); + if (firstVideoFrameSignalName && isVideo(*m_gstWrapper, m_element)) + { + RIALTO_SERVER_LOG_INFO("Connecting first video frame callback for signal: %s", + firstVideoFrameSignalName.value().c_str()); + m_glibWrapper->gSignalConnect(m_element, firstVideoFrameSignalName.value().c_str(), + G_CALLBACK(firstVideoFrameCallback), &m_player); + } } if (isVideoSink(*m_gstWrapper, m_element)) diff --git a/media/server/ipc/include/MediaPipelineClient.h b/media/server/ipc/include/MediaPipelineClient.h index 10af76a1e..685f2d325 100644 --- a/media/server/ipc/include/MediaPipelineClient.h +++ b/media/server/ipc/include/MediaPipelineClient.h @@ -45,6 +45,7 @@ class MediaPipelineClient : public IMediaPipelineClient void notifyCancelNeedMediaData(int32_t sourceId) override; void notifyQos(int32_t sourceId, const QosInfo &qosInfo) override; void notifyBufferUnderflow(int32_t sourceId) override; + void notifyFirstFrameReceived(int32_t sourceId) override; void notifyPlaybackError(int32_t sourceId, PlaybackError error) override; void notifySourceFlushed(int32_t sourceId) override; void notifyPlaybackInfo(const PlaybackInfo &playbackInfo) override; diff --git a/media/server/ipc/source/MediaPipelineClient.cpp b/media/server/ipc/source/MediaPipelineClient.cpp index 655fa8f19..989da34fb 100644 --- a/media/server/ipc/source/MediaPipelineClient.cpp +++ b/media/server/ipc/source/MediaPipelineClient.cpp @@ -240,6 +240,17 @@ void MediaPipelineClient::notifyBufferUnderflow(int32_t sourceId) m_ipcClient->sendEvent(event); } +void MediaPipelineClient::notifyFirstFrameReceived(int32_t sourceId) +{ + RIALTO_SERVER_LOG_DEBUG("Sending FirstFrameReceivedEvent..."); + + auto event = std::make_shared(); + event->set_session_id(m_sessionId); + event->set_source_id(sourceId); + + m_ipcClient->sendEvent(event); +} + void MediaPipelineClient::notifyPlaybackError(int32_t sourceId, PlaybackError error) { RIALTO_SERVER_LOG_DEBUG("Sending notifyPlaybackError..."); diff --git a/media/server/main/include/MediaPipelineServerInternal.h b/media/server/main/include/MediaPipelineServerInternal.h index b165419ee..34807cc1f 100644 --- a/media/server/main/include/MediaPipelineServerInternal.h +++ b/media/server/main/include/MediaPipelineServerInternal.h @@ -193,6 +193,8 @@ class MediaPipelineServerInternal : public IMediaPipelineServerInternal, public void notifyBufferUnderflow(MediaSourceType mediaSourceType) override; + void notifyFirstFrameReceived(MediaSourceType mediaSourceType) override; + void notifyPlaybackError(MediaSourceType mediaSourceType, PlaybackError error) override; void notifySourceFlushed(MediaSourceType mediaSourceType) override; diff --git a/media/server/main/source/MediaPipelineServerInternal.cpp b/media/server/main/source/MediaPipelineServerInternal.cpp index 6476c9504..385181f13 100644 --- a/media/server/main/source/MediaPipelineServerInternal.cpp +++ b/media/server/main/source/MediaPipelineServerInternal.cpp @@ -1560,6 +1560,28 @@ void MediaPipelineServerInternal::notifyBufferUnderflow(MediaSourceType mediaSou m_mainThread->enqueueTask(m_mainThreadClientId, task); } +void MediaPipelineServerInternal::notifyFirstFrameReceived(MediaSourceType mediaSourceType) +{ + RIALTO_SERVER_LOG_DEBUG("entry:"); + + auto task = [&, mediaSourceType]() + { + if (m_mediaPipelineClient) + { + const auto kSourceIter = m_attachedSources.find(mediaSourceType); + if (m_attachedSources.cend() == kSourceIter) + { + RIALTO_SERVER_LOG_WARN("First frame notification failed - sourceId not found for %s", + common::convertMediaSourceType(mediaSourceType)); + return; + } + m_mediaPipelineClient->notifyFirstFrameReceived(kSourceIter->second); + } + }; + + m_mainThread->enqueueTask(m_mainThreadClientId, task); +} + void MediaPipelineServerInternal::notifyPlaybackError(MediaSourceType mediaSourceType, PlaybackError error) { RIALTO_SERVER_LOG_DEBUG("entry:"); diff --git a/proto/mediapipelinemodule.proto b/proto/mediapipelinemodule.proto index ed8bbab63..0c95b9c43 100644 --- a/proto/mediapipelinemodule.proto +++ b/proto/mediapipelinemodule.proto @@ -953,6 +953,19 @@ message BufferUnderflowEvent { optional int32 source_id = 2 [default = -1]; } +/** + * @brief Event sent by the server when the first frame has been received. + * + * @param session_id The id of the A/V session the request is for. + * @param source_id The id of the media source the request is for. + * + * This is sent by the server whenever a video/audio first frame is received + */ +message FirstFrameReceivedEvent { + optional int32 session_id = 1 [default = -1]; + optional int32 source_id = 2 [default = -1]; +} + /** * @brief Event sent by the server when a non-fatal error has occurred in the player. * diff --git a/tests/common/publicClientMocks/MediaPipelineClientMock.h b/tests/common/publicClientMocks/MediaPipelineClientMock.h index d390fb72b..d5c8b5c2c 100644 --- a/tests/common/publicClientMocks/MediaPipelineClientMock.h +++ b/tests/common/publicClientMocks/MediaPipelineClientMock.h @@ -46,6 +46,7 @@ class MediaPipelineClientMock : public IMediaPipelineClient MOCK_METHOD(void, notifyCancelNeedMediaData, (int32_t sourceId), (override)); MOCK_METHOD(void, notifyQos, (int32_t sourceId, const QosInfo &qosInfo), (override)); MOCK_METHOD(void, notifyBufferUnderflow, (int32_t sourceId), (override)); + MOCK_METHOD(void, notifyFirstFrameReceived, (int32_t sourceId), (override)); MOCK_METHOD(void, notifyPlaybackError, (int32_t sourceId, PlaybackError error), (override)); MOCK_METHOD(void, notifySourceFlushed, (int32_t sourceId), (override)); MOCK_METHOD(void, notifyPlaybackInfo, (const PlaybackInfo &playbackInfo), (override)); diff --git a/tests/componenttests/server/stubs/ClientStub.cpp b/tests/componenttests/server/stubs/ClientStub.cpp index 60896dc6d..1da9e9147 100644 --- a/tests/componenttests/server/stubs/ClientStub.cpp +++ b/tests/componenttests/server/stubs/ClientStub.cpp @@ -43,14 +43,14 @@ bool ClientStub::connect() { return false; } - setupSubscriptions(m_ipcChannel); + setupSubscriptions< + firebolt::rialto::PlaybackStateChangeEvent, firebolt::rialto::NetworkStateChangeEvent, + firebolt::rialto::PositionChangeEvent, firebolt::rialto::NeedMediaDataEvent, firebolt::rialto::QosEvent, + firebolt::rialto::BufferUnderflowEvent, firebolt::rialto::FirstFrameReceivedEvent, + firebolt::rialto::PlaybackErrorEvent, firebolt::rialto::SetLogLevelsEvent, firebolt::rialto::SourceFlushedEvent, + firebolt::rialto::WebAudioPlayerStateEvent, firebolt::rialto::ApplicationStateChangeEvent, + firebolt::rialto::PingEvent, firebolt::rialto::LicenseRequestEvent, firebolt::rialto::LicenseRenewalEvent, + firebolt::rialto::KeyStatusesChangedEvent, firebolt::rialto::PlaybackInfoEvent>(m_ipcChannel); m_ipcThread = std::thread(&ClientStub::ipcThread, this); return true; } diff --git a/tests/componenttests/server/tests/mediaPipeline/UnderflowTest.cpp b/tests/componenttests/server/tests/mediaPipeline/UnderflowTest.cpp index ed58398c2..7d35387c4 100644 --- a/tests/componenttests/server/tests/mediaPipeline/UnderflowTest.cpp +++ b/tests/componenttests/server/tests/mediaPipeline/UnderflowTest.cpp @@ -75,7 +75,7 @@ class UnderflowTest : public MediaPipelineTest EXPECT_CALL(*m_glibWrapperMock, gSignalQuery(m_signals[0], _)) .WillRepeatedly(Invoke([&](guint signal_id, GSignalQuery *query) { query->signal_name = "buffer-underflow-callback"; })); - EXPECT_CALL(*m_glibWrapperMock, gFree(m_signals)).Times(2); + EXPECT_CALL(*m_glibWrapperMock, gFree(m_signals)).Times(4); } void willSetupAudioDecoder() diff --git a/tests/unittests/media/client/ipc/mediaPipelineIpc/base/MediaPipelineIpcTestBase.cpp b/tests/unittests/media/client/ipc/mediaPipelineIpc/base/MediaPipelineIpcTestBase.cpp index 919729f74..3d0e56854 100644 --- a/tests/unittests/media/client/ipc/mediaPipelineIpc/base/MediaPipelineIpcTestBase.cpp +++ b/tests/unittests/media/client/ipc/mediaPipelineIpc/base/MediaPipelineIpcTestBase.cpp @@ -112,6 +112,15 @@ void MediaPipelineIpcTestBase::expectSubscribeEvents() return static_cast(EventTags::BufferUnderflowEvent); })) .RetiresOnSaturation(); + EXPECT_CALL(*m_channelMock, subscribeImpl("firebolt.rialto.FirstFrameReceivedEvent", _, _)) + .WillOnce(Invoke( + [this](const std::string &eventName, const google::protobuf::Descriptor *descriptor, + std::function &msg)> &&handler) + { + m_firstFrameReceivedCb = std::move(handler); + return static_cast(EventTags::FirstFrameReceivedEvent); + })) + .RetiresOnSaturation(); EXPECT_CALL(*m_channelMock, subscribeImpl("firebolt.rialto.PlaybackErrorEvent", _, _)) .WillOnce(Invoke( [this](const std::string &eventName, const google::protobuf::Descriptor *descriptor, @@ -149,6 +158,7 @@ void MediaPipelineIpcTestBase::expectUnsubscribeEvents() EXPECT_CALL(*m_channelMock, unsubscribe(static_cast(EventTags::NeedMediaDataEvent))).WillOnce(Return(true)); EXPECT_CALL(*m_channelMock, unsubscribe(static_cast(EventTags::QosEvent))).WillOnce(Return(true)); EXPECT_CALL(*m_channelMock, unsubscribe(static_cast(EventTags::BufferUnderflowEvent))).WillOnce(Return(true)); + EXPECT_CALL(*m_channelMock, unsubscribe(static_cast(EventTags::FirstFrameReceivedEvent))).WillOnce(Return(true)); EXPECT_CALL(*m_channelMock, unsubscribe(static_cast(EventTags::PlaybackErrorEvent))).WillOnce(Return(true)); EXPECT_CALL(*m_channelMock, unsubscribe(static_cast(EventTags::SourceFlushedEvent))).WillOnce(Return(true)); EXPECT_CALL(*m_channelMock, unsubscribe(static_cast(EventTags::PlaybackInfoEvent))).WillOnce(Return(true)); diff --git a/tests/unittests/media/client/ipc/mediaPipelineIpc/base/MediaPipelineIpcTestBase.h b/tests/unittests/media/client/ipc/mediaPipelineIpc/base/MediaPipelineIpcTestBase.h index af26f067f..58e3a5898 100644 --- a/tests/unittests/media/client/ipc/mediaPipelineIpc/base/MediaPipelineIpcTestBase.h +++ b/tests/unittests/media/client/ipc/mediaPipelineIpc/base/MediaPipelineIpcTestBase.h @@ -65,6 +65,7 @@ class MediaPipelineIpcTestBase : public IpcModuleBase, public ::testing::Test NeedMediaDataEvent, QosEvent, BufferUnderflowEvent, + FirstFrameReceivedEvent, PlaybackErrorEvent, SourceFlushedEvent, PlaybackInfoEvent @@ -77,6 +78,7 @@ class MediaPipelineIpcTestBase : public IpcModuleBase, public ::testing::Test std::function &msg)> m_positionChangeCb; std::function &msg)> m_qosCb; std::function &msg)> m_bufferUnderflowCb; + std::function &msg)> m_firstFrameReceivedCb; std::function &msg)> m_playbackErrorCb; std::function &msg)> m_sourceFlushedCb; std::function &msg)> m_playbackInfoCb; diff --git a/tests/unittests/media/client/mocks/ipc/MediaPipelineIpcClientMock.h b/tests/unittests/media/client/mocks/ipc/MediaPipelineIpcClientMock.h index 5d34a7a49..51716ac0d 100644 --- a/tests/unittests/media/client/mocks/ipc/MediaPipelineIpcClientMock.h +++ b/tests/unittests/media/client/mocks/ipc/MediaPipelineIpcClientMock.h @@ -41,6 +41,7 @@ class MediaPipelineIpcClientMock : public IMediaPipelineIpcClient MOCK_METHOD(void, notifyPosition, (int64_t position), (override)); MOCK_METHOD(void, notifyQos, (int32_t sourceId, const QosInfo &qosInfo), (override)); MOCK_METHOD(void, notifyBufferUnderflow, (int32_t sourceId), (override)); + MOCK_METHOD(void, notifyFirstFrameReceived, (int32_t sourceId), (override)); MOCK_METHOD(void, notifyPlaybackError, (int32_t sourceId, PlaybackError error), (override)); MOCK_METHOD(void, notifySourceFlushed, (int32_t sourceId), (override)); MOCK_METHOD(void, notifyPlaybackInfo, (const PlaybackInfo &playbackInfo), (override)); diff --git a/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.cpp b/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.cpp index 5e26811cc..c9d971378 100644 --- a/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.cpp +++ b/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.cpp @@ -400,15 +400,16 @@ void GenericTasksTestsBase::expectVideoUnderflowSignalConnection() { EXPECT_CALL(*testContext->m_glibWrapper, gObjectType(testContext->m_element)).WillRepeatedly(Return(G_TYPE_PARAM)); EXPECT_CALL(*testContext->m_glibWrapper, gSignalListIds(_, _)) - .WillOnce(Invoke( + .WillRepeatedly(Invoke( [&](GType itype, guint *n_ids) { *n_ids = 1; return testContext->m_signals; })); EXPECT_CALL(*testContext->m_glibWrapper, gSignalQuery(testContext->m_signals[0], _)) - .WillOnce(Invoke([&](guint signal_id, GSignalQuery *query) { query->signal_name = "buffer-underflow-callback"; })); - EXPECT_CALL(*testContext->m_glibWrapper, gFree(testContext->m_signals)); + .WillRepeatedly( + Invoke([&](guint signal_id, GSignalQuery *query) { query->signal_name = "buffer-underflow-callback"; })); + EXPECT_CALL(*testContext->m_glibWrapper, gFree(testContext->m_signals)).Times(2); EXPECT_CALL(*testContext->m_glibWrapper, gSignalConnect(_, StrEq("buffer-underflow-callback"), _, _)) .WillOnce(Invoke( [&](gpointer instance, const gchar *detailed_signal, GCallback c_handler, gpointer data) @@ -418,19 +419,34 @@ void GenericTasksTestsBase::expectVideoUnderflowSignalConnection() })); } +void GenericTasksTestsBase::expectFirstVideoFrameSignalConnection() +{ + EXPECT_CALL(*testContext->m_glibWrapper, gSignalQuery(testContext->m_signals[0], _)) + .WillOnce( + Invoke([&](guint signal_id, GSignalQuery *query) { query->signal_name = "first-video-frame-callback"; })); + EXPECT_CALL(*testContext->m_glibWrapper, gSignalConnect(_, StrEq("first-video-frame-callback"), _, _)) + .WillOnce(Invoke( + [&](gpointer instance, const gchar *detailed_signal, GCallback c_handler, gpointer data) + { + testContext->m_firstVideoFrameCallback = c_handler; + return kSignalId; + })); +} + void GenericTasksTestsBase::expectAudioUnderflowSignalConnection() { EXPECT_CALL(*testContext->m_glibWrapper, gObjectType(testContext->m_element)).WillRepeatedly(Return(G_TYPE_PARAM)); EXPECT_CALL(*testContext->m_glibWrapper, gSignalListIds(_, _)) - .WillOnce(Invoke( + .WillRepeatedly(Invoke( [&](GType itype, guint *n_ids) { *n_ids = 1; return testContext->m_signals; })); EXPECT_CALL(*testContext->m_glibWrapper, gSignalQuery(testContext->m_signals[0], _)) - .WillOnce(Invoke([&](guint signal_id, GSignalQuery *query) { query->signal_name = "buffer-underflow-callback"; })); - EXPECT_CALL(*testContext->m_glibWrapper, gFree(testContext->m_signals)); + .WillRepeatedly( + Invoke([&](guint signal_id, GSignalQuery *query) { query->signal_name = "buffer-underflow-callback"; })); + EXPECT_CALL(*testContext->m_glibWrapper, gFree(testContext->m_signals)).Times(2); EXPECT_CALL(*testContext->m_glibWrapper, gSignalConnect(_, StrEq("buffer-underflow-callback"), _, _)) .WillOnce(Invoke( [&](gpointer instance, const gchar *detailed_signal, GCallback c_handler, gpointer data) @@ -511,6 +527,67 @@ void GenericTasksTestsBase::expectSetupVideoDecoderElement() EXPECT_CALL(*testContext->m_gstWrapper, gstObjectUnref(_)); } +void GenericTasksTestsBase::expectSetupVideoDecoderElementWithFirstVideoFrameCallback() +{ + EXPECT_CALL(*testContext->m_gstWrapper, gstElementGetFactory(_)).WillRepeatedly(Return(testContext->m_elementFactory)); + + EXPECT_CALL(*testContext->m_gstWrapper, + gstElementFactoryListIsType(testContext->m_elementFactory, GST_ELEMENT_FACTORY_TYPE_DECODER)) + .WillOnce(Return(TRUE)); + + EXPECT_CALL(*testContext->m_gstWrapper, + gstElementFactoryListIsType(testContext->m_elementFactory, GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO)) + .WillOnce(Return(FALSE)); + + EXPECT_CALL(*testContext->m_gstWrapper, + gstElementFactoryListIsType(testContext->m_elementFactory, GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO)) + .Times(2) + .WillRepeatedly(Return(TRUE)); + + EXPECT_CALL(*testContext->m_gstWrapper, + gstElementFactoryListIsType(testContext->m_elementFactory, + GST_ELEMENT_FACTORY_TYPE_SINK | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO)) + .WillOnce(Return(FALSE)); + + EXPECT_CALL(*testContext->m_gstWrapper, + gstElementFactoryListIsType(testContext->m_elementFactory, + GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO)) + .WillOnce(Return(FALSE)); + + EXPECT_CALL(*testContext->m_gstWrapper, + gstElementFactoryListIsType(testContext->m_elementFactory, + GST_ELEMENT_FACTORY_TYPE_SINK | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO)) + .WillOnce(Return(FALSE)); + + EXPECT_CALL(*testContext->m_gstWrapper, + gstElementFactoryListIsType(testContext->m_elementFactory, + GST_ELEMENT_FACTORY_TYPE_PARSER | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO)) + .WillOnce(Return(FALSE)); + + EXPECT_CALL(*testContext->m_glibWrapper, gObjectType(testContext->m_element)).WillRepeatedly(Return(G_TYPE_PARAM)); + EXPECT_CALL(*testContext->m_glibWrapper, gSignalListIds(_, _)) + .WillRepeatedly(Invoke( + [&](GType itype, guint *n_ids) + { + *n_ids = 1; + return testContext->m_signals; + })); + EXPECT_CALL(*testContext->m_glibWrapper, gFree(testContext->m_signals)).Times(2); + expectFirstVideoFrameSignalConnection(); + EXPECT_CALL(*testContext->m_glibWrapper, gSignalQuery(testContext->m_signals[0], _)) + .WillOnce(Invoke([&](guint signal_id, GSignalQuery *query) { query->signal_name = "buffer-underflow-callback"; })) + .RetiresOnSaturation(); + EXPECT_CALL(*testContext->m_glibWrapper, gSignalConnect(_, StrEq("buffer-underflow-callback"), _, _)) + .WillOnce(Invoke( + [&](gpointer instance, const gchar *detailed_signal, GCallback c_handler, gpointer data) + { + testContext->m_videoUnderflowCallback = c_handler; + return kSignalId; + })); + + EXPECT_CALL(*testContext->m_gstWrapper, gstObjectUnref(_)); +} + void GenericTasksTestsBase::expectSetupAudioSinkElement() { EXPECT_CALL(*testContext->m_gstWrapper, gstElementGetFactory(_)).WillRepeatedly(Return(testContext->m_elementFactory)); @@ -673,6 +750,17 @@ void GenericTasksTestsBase::shouldSetupVideoDecoderElementOnly() expectSetupVideoDecoderElement(); } +void GenericTasksTestsBase::shouldSetupVideoDecoderElementWithFirstVideoFrameCallback() +{ + EXPECT_CALL(*testContext->m_glibWrapper, gTypeName(G_OBJECT_TYPE(testContext->m_element))) + .WillOnce(Return(kElementTypeName.c_str())); + EXPECT_CALL(*testContext->m_glibWrapper, gStrHasPrefix(_, StrEq("amlhalasink"))).WillOnce(Return(FALSE)); + EXPECT_CALL(*testContext->m_glibWrapper, gStrHasPrefix(_, StrEq("brcmaudiosink"))).WillOnce(Return(FALSE)); + EXPECT_CALL(*testContext->m_glibWrapper, gStrHasPrefix(_, StrEq("rialtotexttracksink"))).WillOnce(Return(FALSE)); + EXPECT_CALL(*testContext->m_gstWrapper, gstIsBaseParse(_)).WillOnce(Return(FALSE)); + expectSetupVideoDecoderElementWithFirstVideoFrameCallback(); +} + void GenericTasksTestsBase::shouldSetupVideoElementWithPendingGeometry() { testContext->m_context.pendingGeometry = kRectangle; @@ -981,6 +1069,12 @@ void GenericTasksTestsBase::shouldSetVideoUnderflowCallback() EXPECT_CALL(testContext->m_gstPlayer, scheduleVideoUnderflow()); } +void GenericTasksTestsBase::shouldSetFirstVideoFrameCallback() +{ + ASSERT_TRUE(testContext->m_firstVideoFrameCallback); + EXPECT_CALL(testContext->m_gstPlayer, scheduleFirstVideoFrameReceived()); +} + void GenericTasksTestsBase::shouldSetupBaseParse() { EXPECT_CALL(*testContext->m_gstWrapper, gstBaseParseSetPtsInterpolation(_, FALSE)); @@ -993,6 +1087,12 @@ void GenericTasksTestsBase::triggerVideoUnderflowCallback() testContext->m_videoUnderflowCallback)(testContext->m_element, 0, nullptr, &testContext->m_gstPlayer); } +void GenericTasksTestsBase::triggerFirstVideoFrameCallback() +{ + reinterpret_cast( + testContext->m_firstVideoFrameCallback)(testContext->m_element, 0, nullptr, &testContext->m_gstPlayer); +} + void GenericTasksTestsBase::shouldSetAudioUnderflowCallback() { ASSERT_TRUE(testContext->m_audioUnderflowCallback); diff --git a/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.h b/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.h index 7d212b97d..4aca17fa7 100644 --- a/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.h +++ b/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.h @@ -78,6 +78,7 @@ class GenericTasksTestsBase : public ::testing::Test // SetupElement test methods void shouldSetupVideoSinkElementOnly(); void shouldSetupVideoDecoderElementOnly(); + void shouldSetupVideoDecoderElementWithFirstVideoFrameCallback(); void shouldSetupVideoElementWithPendingGeometry(); void shouldSetupVideoElementWithPendingImmediateOutput(); void shouldSetupAudioSinkElementWithPendingLowLatency(); @@ -99,9 +100,11 @@ class GenericTasksTestsBase : public ::testing::Test void shouldSetupAudioSinkElementOnly(); void shouldSetupAudioDecoderElementOnly(); void shouldSetVideoUnderflowCallback(); + void shouldSetFirstVideoFrameCallback(); void shouldSetupBaseParse(); void triggerSetupElement(); void triggerVideoUnderflowCallback(); + void triggerFirstVideoFrameCallback(); void shouldSetAudioUnderflowCallback(); void triggerAudioUnderflowCallback(); void shouldAddFirstAutoVideoSinkChild(); @@ -432,9 +435,11 @@ class GenericTasksTestsBase : public ::testing::Test private: // SetupElement helper methods void expectVideoUnderflowSignalConnection(); + void expectFirstVideoFrameSignalConnection(); void expectAudioUnderflowSignalConnection(); void expectSetupVideoSinkElement(); void expectSetupVideoDecoderElement(); + void expectSetupVideoDecoderElementWithFirstVideoFrameCallback(); void expectSetupAudioSinkElement(); void expectSetupAudioDecoderElement(); void expectSetupVideoParserElement(); diff --git a/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsContext.h b/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsContext.h index 4918f723c..c39eb5303 100644 --- a/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsContext.h +++ b/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsContext.h @@ -99,6 +99,7 @@ class GenericTasksTestsContext guint m_signals[1]{123}; GCallback m_audioUnderflowCallback; GCallback m_videoUnderflowCallback; + GCallback m_firstVideoFrameCallback; GCallback m_childAddedCallback; GCallback m_childRemovedCallback; gchar m_capsStr{}; diff --git a/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/GenericPlayerTaskFactoryTest.cpp b/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/GenericPlayerTaskFactoryTest.cpp index 70c305e22..9b4911e15 100644 --- a/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/GenericPlayerTaskFactoryTest.cpp +++ b/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/GenericPlayerTaskFactoryTest.cpp @@ -36,6 +36,7 @@ #include "tasks/generic/EnoughData.h" #include "tasks/generic/Eos.h" #include "tasks/generic/FinishSetupSource.h" +#include "tasks/generic/FirstFrameReceived.h" #include "tasks/generic/Flush.h" #include "tasks/generic/GenericPlayerTaskFactory.h" #include "tasks/generic/HandleBusMessage.h" @@ -295,6 +296,13 @@ TEST_F(GenericPlayerTaskFactoryTest, ShouldCreateUnderflow) EXPECT_NO_THROW(dynamic_cast(*task)); } +TEST_F(GenericPlayerTaskFactoryTest, ShouldCreateFirstFrameReceived) +{ + auto task = m_sut.createFirstFrameReceived(m_context, m_gstPlayer, firebolt::rialto::MediaSourceType::VIDEO); + EXPECT_NE(task, nullptr); + EXPECT_NO_THROW(dynamic_cast(*task)); +} + TEST_F(GenericPlayerTaskFactoryTest, ShouldCreateSetPlaybackRate) { auto task = m_sut.createSetPlaybackRate(m_context, 1.25); diff --git a/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/SetupElementTest.cpp b/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/SetupElementTest.cpp index 6f2fa12df..9d16f1953 100644 --- a/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/SetupElementTest.cpp +++ b/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/SetupElementTest.cpp @@ -170,6 +170,15 @@ TEST_F(SetupElementTest, shouldReportVideoUnderflow) triggerVideoUnderflowCallback(); } +TEST_F(SetupElementTest, shouldReportFirstVideoFrame) +{ + shouldSetupVideoDecoderElementWithFirstVideoFrameCallback(); + triggerSetupElement(); + + shouldSetFirstVideoFrameCallback(); + triggerFirstVideoFrameCallback(); +} + TEST_F(SetupElementTest, shouldReportAudioUnderflow) { shouldSetupAudioDecoderElementOnly(); diff --git a/tests/unittests/media/server/mocks/gstplayer/GenericPlayerTaskFactoryMock.h b/tests/unittests/media/server/mocks/gstplayer/GenericPlayerTaskFactoryMock.h index 2cb7f0a35..2226cfb3c 100644 --- a/tests/unittests/media/server/mocks/gstplayer/GenericPlayerTaskFactoryMock.h +++ b/tests/unittests/media/server/mocks/gstplayer/GenericPlayerTaskFactoryMock.h @@ -47,6 +47,9 @@ class GenericPlayerTaskFactoryMock : public IGenericPlayerTaskFactory (GenericPlayerContext & context, IGstGenericPlayerPrivate &player, const firebolt::rialto::MediaSourceType &type), (const, override)); + MOCK_METHOD(std::unique_ptr, createFirstFrameReceived, + (GenericPlayerContext & context, IGstGenericPlayerPrivate &player, MediaSourceType sourceType), + (const, override)); MOCK_METHOD(std::unique_ptr, createFinishSetupSource, (GenericPlayerContext & context, IGstGenericPlayerPrivate &player), (const, override)); MOCK_METHOD(std::unique_ptr, createHandleBusMessage, diff --git a/tests/unittests/media/server/mocks/gstplayer/GstGenericPlayerClientMock.h b/tests/unittests/media/server/mocks/gstplayer/GstGenericPlayerClientMock.h index a40fac73a..83cceacda 100644 --- a/tests/unittests/media/server/mocks/gstplayer/GstGenericPlayerClientMock.h +++ b/tests/unittests/media/server/mocks/gstplayer/GstGenericPlayerClientMock.h @@ -39,6 +39,7 @@ class GstGenericPlayerClientMock : public IGstGenericPlayerClient MOCK_METHOD(void, invalidateActiveRequests, (const MediaSourceType &type), (override)); MOCK_METHOD(void, notifyQos, (MediaSourceType mediaSourceType, const QosInfo &qosInfo), (override)); MOCK_METHOD(void, notifyBufferUnderflow, (MediaSourceType mediaSourceType), (override)); + MOCK_METHOD(void, notifyFirstFrameReceived, (MediaSourceType mediaSourceType), (override)); MOCK_METHOD(void, notifyPlaybackError, (MediaSourceType mediaSourceType, PlaybackError error), (override)); MOCK_METHOD(void, notifySourceFlushed, (MediaSourceType mediaSourceType), (override)); MOCK_METHOD(void, notifyPlaybackInfo, (const PlaybackInfo &playbackInfo), (override)); diff --git a/tests/unittests/media/server/mocks/gstplayer/GstGenericPlayerPrivateMock.h b/tests/unittests/media/server/mocks/gstplayer/GstGenericPlayerPrivateMock.h index e28c70d8d..9fbf62ebc 100644 --- a/tests/unittests/media/server/mocks/gstplayer/GstGenericPlayerPrivateMock.h +++ b/tests/unittests/media/server/mocks/gstplayer/GstGenericPlayerPrivateMock.h @@ -36,6 +36,7 @@ class GstGenericPlayerPrivateMock : public IGstGenericPlayerPrivate MOCK_METHOD(void, scheduleEnoughData, (GstAppSrc * src), (override)); MOCK_METHOD(void, scheduleAudioUnderflow, (), (override)); MOCK_METHOD(void, scheduleVideoUnderflow, (), (override)); + MOCK_METHOD(void, scheduleFirstVideoFrameReceived, (), (override)); MOCK_METHOD(void, scheduleAllSourcesAttached, (), (override)); MOCK_METHOD(bool, setVideoSinkRectangle, (), (override)); MOCK_METHOD(bool, setImmediateOutput, (), (override)); From d615a276d553c205adbc26919f4ed9964da449c8 Mon Sep 17 00:00:00 2001 From: Sasa Mudri Date: Mon, 11 May 2026 10:23:03 +0200 Subject: [PATCH 2/7] Group signals for first frame callback and update related connection. --- media/server/gstplayer/include/Utils.h | 4 ++-- media/server/gstplayer/source/Utils.cpp | 10 +++++----- .../source/tasks/generic/SetupElement.cpp | 19 +++++++++++++------ 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/media/server/gstplayer/include/Utils.h b/media/server/gstplayer/include/Utils.h index ef4c8bb14..c768bd624 100644 --- a/media/server/gstplayer/include/Utils.h +++ b/media/server/gstplayer/include/Utils.h @@ -41,8 +41,8 @@ bool isAudio(const firebolt::rialto::wrappers::IGstWrapper &gstWrapper, GstEleme bool isVideo(const firebolt::rialto::wrappers::IGstWrapper &gstWrapper, GstElement *element); std::optional getUnderflowSignalName(const firebolt::rialto::wrappers::IGlibWrapper &glibWrapper, GstElement *element); -std::optional getFirstVideoFrameSignalName(const firebolt::rialto::wrappers::IGlibWrapper &glibWrapper, - GstElement *element); +std::optional getFirstFrameSignalName(const firebolt::rialto::wrappers::IGlibWrapper &glibWrapper, + GstElement *element); GstCaps *createCapsFromMediaSource(const std::shared_ptr &gstWrapper, const std::shared_ptr &glibWrapper, const std::unique_ptr &source); diff --git a/media/server/gstplayer/source/Utils.cpp b/media/server/gstplayer/source/Utils.cpp index 85b986d64..fd93a2323 100644 --- a/media/server/gstplayer/source/Utils.cpp +++ b/media/server/gstplayer/source/Utils.cpp @@ -29,7 +29,7 @@ namespace { const char *underflowSignals[]{"buffer-underflow-callback", "vidsink-underflow-callback", "underrun-callback"}; -const char *firstVideoFrameSignals[]{"first-video-frame-callback"}; +const char *firstFrameSignals[]{"first-video-frame-callback"}; bool isType(const firebolt::rialto::wrappers::IGstWrapper &gstWrapper, GstElement *element, GstElementFactoryListType type) { @@ -126,8 +126,8 @@ std::optional getUnderflowSignalName(const firebolt::rialto::wrappe return std::nullopt; } -std::optional getFirstVideoFrameSignalName(const firebolt::rialto::wrappers::IGlibWrapper &glibWrapper, - GstElement *element) +std::optional getFirstFrameSignalName(const firebolt::rialto::wrappers::IGlibWrapper &glibWrapper, + GstElement *element) { GType type = glibWrapper.gObjectType(element); guint nsignals{0}; @@ -137,11 +137,11 @@ std::optional getFirstVideoFrameSignalName(const firebolt::rialto:: { GSignalQuery query; glibWrapper.gSignalQuery(signals[i], &query); - const auto signalNameIt = std::find_if(std::begin(firstVideoFrameSignals), std::end(firstVideoFrameSignals), + const auto signalNameIt = std::find_if(std::begin(firstFrameSignals), std::end(firstFrameSignals), [&](const auto *signalName) { return strcmp(signalName, query.signal_name) == 0; }); - if (std::end(firstVideoFrameSignals) != signalNameIt) + if (std::end(firstFrameSignals) != signalNameIt) { glibWrapper.gFree(signals); return std::string(*signalNameIt); diff --git a/media/server/gstplayer/source/tasks/generic/SetupElement.cpp b/media/server/gstplayer/source/tasks/generic/SetupElement.cpp index 0b598d2ce..03da51c6a 100644 --- a/media/server/gstplayer/source/tasks/generic/SetupElement.cpp +++ b/media/server/gstplayer/source/tasks/generic/SetupElement.cpp @@ -283,13 +283,20 @@ void SetupElement::execute() const } } - std::optional firstVideoFrameSignalName = getFirstVideoFrameSignalName(*m_glibWrapper, m_element); - if (firstVideoFrameSignalName && isVideo(*m_gstWrapper, m_element)) + std::optional firstFrameSignalName = getFirstFrameSignalName(*m_glibWrapper, m_element); + if (firstFrameSignalName) { - RIALTO_SERVER_LOG_INFO("Connecting first video frame callback for signal: %s", - firstVideoFrameSignalName.value().c_str()); - m_glibWrapper->gSignalConnect(m_element, firstVideoFrameSignalName.value().c_str(), - G_CALLBACK(firstVideoFrameCallback), &m_player); + if (isAudio(*m_gstWrapper, m_element)) + { + // TODO + } + else if (isVideo(*m_gstWrapper, m_element)) + { + RIALTO_SERVER_LOG_INFO("Connecting first video frame callback for signal: %s", + firstFrameSignalName.value().c_str()); + m_glibWrapper->gSignalConnect(m_element, firstFrameSignalName.value().c_str(), + G_CALLBACK(firstVideoFrameCallback), &m_player); + } } } From bc2f6e465869da0da6ecb786ca24f4b4d1fdd241 Mon Sep 17 00:00:00 2001 From: Sasa Mudri Date: Mon, 11 May 2026 14:15:20 +0200 Subject: [PATCH 3/7] Added UTs and additional fixes. --- .../include/tasks/IGenericPlayerTaskFactory.h | 2 +- .../source/tasks/generic/SetupElement.cpp | 6 +--- .../ipc/mediaPipelineIpc/CallbackTest.cpp | 29 +++++++++++++++++++ 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/media/server/gstplayer/include/tasks/IGenericPlayerTaskFactory.h b/media/server/gstplayer/include/tasks/IGenericPlayerTaskFactory.h index f33864276..86f89c876 100644 --- a/media/server/gstplayer/include/tasks/IGenericPlayerTaskFactory.h +++ b/media/server/gstplayer/include/tasks/IGenericPlayerTaskFactory.h @@ -377,7 +377,7 @@ class IGenericPlayerTaskFactory bool underflowEnabled, MediaSourceType sourceType) const = 0; /** - * @brief Creates an FirstFrameReceived task. + * @brief Creates a FirstFrameReceived task. * * @param[in] context : The GstGenericPlayer context * @param[in] player : The GstPlayer instance diff --git a/media/server/gstplayer/source/tasks/generic/SetupElement.cpp b/media/server/gstplayer/source/tasks/generic/SetupElement.cpp index 03da51c6a..2f2e05657 100644 --- a/media/server/gstplayer/source/tasks/generic/SetupElement.cpp +++ b/media/server/gstplayer/source/tasks/generic/SetupElement.cpp @@ -286,11 +286,7 @@ void SetupElement::execute() const std::optional firstFrameSignalName = getFirstFrameSignalName(*m_glibWrapper, m_element); if (firstFrameSignalName) { - if (isAudio(*m_gstWrapper, m_element)) - { - // TODO - } - else if (isVideo(*m_gstWrapper, m_element)) + if (isVideo(*m_gstWrapper, m_element)) { RIALTO_SERVER_LOG_INFO("Connecting first video frame callback for signal: %s", firstFrameSignalName.value().c_str()); diff --git a/tests/unittests/media/client/ipc/mediaPipelineIpc/CallbackTest.cpp b/tests/unittests/media/client/ipc/mediaPipelineIpc/CallbackTest.cpp index de1090470..d37f2c4a7 100644 --- a/tests/unittests/media/client/ipc/mediaPipelineIpc/CallbackTest.cpp +++ b/tests/unittests/media/client/ipc/mediaPipelineIpc/CallbackTest.cpp @@ -134,6 +134,35 @@ TEST_F(RialtoClientMediaPipelineIpcCallbackTest, InvalidSessionIdQos) m_qosCb(updateQosEvent); } +/** + * Test that a first frame received notification over IPC is forwarded to the client. + */ +TEST_F(RialtoClientMediaPipelineIpcCallbackTest, NotifyFirstFrameReceived) +{ + auto firstFrameReceivedEvent = std::make_shared(); + firstFrameReceivedEvent->set_session_id(m_sessionId); + firstFrameReceivedEvent->set_source_id(m_sourceId); + + EXPECT_CALL(*m_eventThreadMock, addImpl(_)).WillOnce(Invoke([](std::function &&func) { func(); })); + EXPECT_CALL(*m_clientMock, notifyFirstFrameReceived(m_sourceId)); + + m_firstFrameReceivedCb(firstFrameReceivedEvent); +} + +/** + * Test that if the session id of the event is not the same as the playback session the event will be ignored. + */ +TEST_F(RialtoClientMediaPipelineIpcCallbackTest, InvalidSessionIdFirstFrameReceived) +{ + auto firstFrameReceivedEvent = std::make_shared(); + firstFrameReceivedEvent->set_session_id(-1); + firstFrameReceivedEvent->set_source_id(m_sourceId); + + EXPECT_CALL(*m_eventThreadMock, addImpl(_)).WillOnce(Invoke([](std::function &&func) { func(); })); + + m_firstFrameReceivedCb(firstFrameReceivedEvent); +} + /** * Test that a playback error notification over IPC is forwarded to the client. */ From 5cde290fa4d9cc60b9bae4d59f2a0a816d337ff1 Mon Sep 17 00:00:00 2001 From: Sasa Mudri Date: Wed, 13 May 2026 16:18:57 +0200 Subject: [PATCH 4/7] Added more UTs and CTs to provide expected coverage. --- .../source/tasks/generic/SetupElement.cpp | 6 +- tests/componenttests/client/CMakeLists.txt | 1 + .../client/stubs/MediaPipelineModuleStub.cpp | 10 + .../client/stubs/MediaPipelineModuleStub.h | 1 + .../tests/base/MediaPipelineTestMethods.cpp | 12 + .../tests/base/MediaPipelineTestMethods.h | 2 + .../tests/mse/FirstFrameNotificationTest.cpp | 86 +++++ .../server/tests/CMakeLists.txt | 1 + .../FirstFrameNotificationTest.cpp | 304 ++++++++++++++++++ .../main/mediaPipeline/CallbackTest.cpp | 12 + .../media/server/gstplayer/CMakeLists.txt | 1 + .../GstGenericPlayerPrivateTest.cpp | 10 + .../tasksTests/FirstFrameReceivedTest.cpp | 57 ++++ .../MediaPipelineModuleServiceTests.cpp | 8 + ...MediaPipelineModuleServiceTestsFixture.cpp | 18 ++ .../MediaPipelineModuleServiceTestsFixture.h | 2 + .../main/mediaPipeline/CallbackTest.cpp | 26 ++ 17 files changed, 555 insertions(+), 2 deletions(-) create mode 100644 tests/componenttests/client/tests/mse/FirstFrameNotificationTest.cpp create mode 100644 tests/componenttests/server/tests/mediaPipeline/FirstFrameNotificationTest.cpp create mode 100644 tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/FirstFrameReceivedTest.cpp diff --git a/media/server/gstplayer/source/tasks/generic/SetupElement.cpp b/media/server/gstplayer/source/tasks/generic/SetupElement.cpp index 2f2e05657..3cedc8a7d 100644 --- a/media/server/gstplayer/source/tasks/generic/SetupElement.cpp +++ b/media/server/gstplayer/source/tasks/generic/SetupElement.cpp @@ -64,8 +64,10 @@ void videoUnderflowCallback(GstElement *object, guint fifoDepth, gpointer queueD /** * @brief Callback for first video frame event from sink. Called by the Gstreamer thread. * - * @param[in] object : the object that emitted the signal - * @param[in] self : The pointer to IGstGenericPlayerPrivate + * @param[in] object : the object that emitted the signal + * @param[in] fifoDepth : the fifo depth (may be 0) + * @param[in] queueDepth : the queue depth (may be NULL) + * @param[in] self : The pointer to IGstGenericPlayerPrivate */ void firstVideoFrameCallback(GstElement *object, guint fifoDepth, gpointer queueDepth, gpointer self) { diff --git a/tests/componenttests/client/CMakeLists.txt b/tests/componenttests/client/CMakeLists.txt index 35ac239f3..bfc933f06 100644 --- a/tests/componenttests/client/CMakeLists.txt +++ b/tests/componenttests/client/CMakeLists.txt @@ -44,6 +44,7 @@ add_gtests ( tests/mse/CreateMediaPipelineFailuresTest.cpp tests/mse/DualVideoPlaybackTest.cpp tests/mse/FlushTest.cpp + tests/mse/FirstFrameNotificationTest.cpp tests/mse/MediaPipelineCapabilitiesTest.cpp tests/mse/MuteTest.cpp tests/mse/PipelinePropertyTest.cpp diff --git a/tests/componenttests/client/stubs/MediaPipelineModuleStub.cpp b/tests/componenttests/client/stubs/MediaPipelineModuleStub.cpp index c6b7115d0..435992758 100644 --- a/tests/componenttests/client/stubs/MediaPipelineModuleStub.cpp +++ b/tests/componenttests/client/stubs/MediaPipelineModuleStub.cpp @@ -105,6 +105,16 @@ void MediaPipelineModuleStub::notifyBufferUnderflowEvent(int sessionId, int32_t getClient()->sendEvent(event); } +void MediaPipelineModuleStub::notifyFirstFrameReceivedEvent(int sessionId, int32_t sourceId) +{ + waitForClientConnect(); + + auto event = std::make_shared(); + event->set_session_id(sessionId); + event->set_source_id(sourceId); + getClient()->sendEvent(event); +} + void MediaPipelineModuleStub::notifyPlaybackErrorEvent(int sessionId, int32_t sourceId, PlaybackError error) { waitForClientConnect(); diff --git a/tests/componenttests/client/stubs/MediaPipelineModuleStub.h b/tests/componenttests/client/stubs/MediaPipelineModuleStub.h index a939070e8..3cd343254 100644 --- a/tests/componenttests/client/stubs/MediaPipelineModuleStub.h +++ b/tests/componenttests/client/stubs/MediaPipelineModuleStub.h @@ -40,6 +40,7 @@ class MediaPipelineModuleStub void notifyPositionChangeEvent(int sessionId, int64_t position); void notifyQosEvent(int sessionId, int32_t sourceId, const ::firebolt::rialto::QosInfo &qosInfo); void notifyBufferUnderflowEvent(int sessionId, int32_t sourceId); + void notifyFirstFrameReceivedEvent(int sessionId, int32_t sourceId); void notifyPlaybackErrorEvent(int sessionId, int32_t sourceId, PlaybackError error); void notifySourceFlushed(int sessionId, int32_t sourceId); void notifyPlaybackInfo(int sessionId, const firebolt::rialto::PlaybackInfo &playbackInfo); diff --git a/tests/componenttests/client/tests/base/MediaPipelineTestMethods.cpp b/tests/componenttests/client/tests/base/MediaPipelineTestMethods.cpp index 00693bbdf..63658d247 100644 --- a/tests/componenttests/client/tests/base/MediaPipelineTestMethods.cpp +++ b/tests/componenttests/client/tests/base/MediaPipelineTestMethods.cpp @@ -1375,6 +1375,18 @@ void MediaPipelineTestMethods::sendNotifyBufferUnderflowVideo() waitEvent(); } +void MediaPipelineTestMethods::shouldNotifyFirstFrameReceivedVideo() +{ + EXPECT_CALL(*m_mediaPipelineClientMock, notifyFirstFrameReceived(kVideoSourceId)) + .WillOnce(Invoke(this, &MediaPipelineTestMethods::notifyEvent)); +} + +void MediaPipelineTestMethods::sendNotifyFirstFrameReceivedVideo() +{ + getServerStub()->notifyFirstFrameReceivedEvent(kSessionId, kVideoSourceId); + waitEvent(); +} + void MediaPipelineTestMethods::shouldNotifyPlaybackErrorAudio() { EXPECT_CALL(*m_mediaPipelineClientMock, notifyPlaybackError(kAudioSourceId, PlaybackError::DECRYPTION)) diff --git a/tests/componenttests/client/tests/base/MediaPipelineTestMethods.h b/tests/componenttests/client/tests/base/MediaPipelineTestMethods.h index ebe103cc3..f73f23f9d 100644 --- a/tests/componenttests/client/tests/base/MediaPipelineTestMethods.h +++ b/tests/componenttests/client/tests/base/MediaPipelineTestMethods.h @@ -173,6 +173,7 @@ class MediaPipelineTestMethods void shouldNotifyQosVideo(); void shouldNotifyBufferUnderflowAudio(); void shouldNotifyBufferUnderflowVideo(); + void shouldNotifyFirstFrameReceivedVideo(); void shouldNotifyPlaybackErrorAudio(); void shouldNotifyPlaybackErrorVideo(); void shouldNotifySourceFlushed(); @@ -301,6 +302,7 @@ class MediaPipelineTestMethods void sendNotifyQosVideo(); void sendNotifyBufferUnderflowAudio(); void sendNotifyBufferUnderflowVideo(); + void sendNotifyFirstFrameReceivedVideo(); void sendNotifyPlaybackErrorAudio(); void sendNotifyPlaybackErrorVideo(); void sendNotifySourceFlushed(); diff --git a/tests/componenttests/client/tests/mse/FirstFrameNotificationTest.cpp b/tests/componenttests/client/tests/mse/FirstFrameNotificationTest.cpp new file mode 100644 index 000000000..319948300 --- /dev/null +++ b/tests/componenttests/client/tests/mse/FirstFrameNotificationTest.cpp @@ -0,0 +1,86 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2026 Sky UK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ClientComponentTest.h" +#include + +namespace firebolt::rialto::client::ct +{ +class FirstFrameNotificationTest : public ClientComponentTest +{ +public: + FirstFrameNotificationTest() : ClientComponentTest() + { + ClientComponentTest::startApplicationRunning(); + MediaPipelineTestMethods::startAudioVideoMediaSessionPrerollPaused(); + + MediaPipelineTestMethods::shouldPlay(); + MediaPipelineTestMethods::play(); + MediaPipelineTestMethods::shouldNotifyPlaybackStatePlaying(); + MediaPipelineTestMethods::sendNotifyPlaybackStatePlaying(); + } + + ~FirstFrameNotificationTest() + { + MediaPipelineTestMethods::endAudioVideoMediaSession(); + ClientComponentTest::stopApplication(); + } +}; + +/* + * Component Test: First frame notification + * Test Objective: + * Test the first frame notification for video source. + * + * Sequence Diagrams: + * First frame notification + * + * Test Setup: + * Language: C++ + * Testing Framework: Google Test + * Components: MediaPipeline + * + * Test Initialize: + * Create memory region for the shared buffer. + * Create a server that handles Control IPC requests. + * Initalise the control state to running for this test application. + * Initalise a audio video media session playing. + * + * Test Steps: + * Step 1: Notify first frame received for video + * Server notifies the client first frame received with source id video. + * Expect that the first frame notification is propagated to the client. + * + * Test Teardown: + * Terminate the media session. + * Memory region created for the shared buffer is closed. + * Server is terminated. + * + * Expected Results: + * First frame received notification is propagated to the application. + * + * Code: + */ +TEST_F(FirstFrameNotificationTest, notification) +{ + // Step 1: Notify first frame received for video + MediaPipelineTestMethods::shouldNotifyFirstFrameReceivedVideo(); + MediaPipelineTestMethods::sendNotifyFirstFrameReceivedVideo(); +} +} // namespace firebolt::rialto::client::ct diff --git a/tests/componenttests/server/tests/CMakeLists.txt b/tests/componenttests/server/tests/CMakeLists.txt index 72a5a04e7..9898b805d 100644 --- a/tests/componenttests/server/tests/CMakeLists.txt +++ b/tests/componenttests/server/tests/CMakeLists.txt @@ -46,6 +46,7 @@ add_gtests ( mediaPipeline/DualVideoPlaybackTest.cpp mediaPipeline/EncryptedPlaybackTest.cpp mediaPipeline/FailureTests.cpp + mediaPipeline/FirstFrameNotificationTest.cpp mediaPipeline/FlushTest.cpp mediaPipeline/HaveDataFailureTest.cpp mediaPipeline/MuteTest.cpp diff --git a/tests/componenttests/server/tests/mediaPipeline/FirstFrameNotificationTest.cpp b/tests/componenttests/server/tests/mediaPipeline/FirstFrameNotificationTest.cpp new file mode 100644 index 000000000..82daedd6c --- /dev/null +++ b/tests/componenttests/server/tests/mediaPipeline/FirstFrameNotificationTest.cpp @@ -0,0 +1,304 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2026 Sky UK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ExpectMessage.h" +#include "Matchers.h" +#include "MediaPipelineTest.h" +#include + +using testing::_; +using testing::Invoke; +using testing::Return; +using testing::StrEq; + +namespace +{ +constexpr unsigned kFramesToPush{1}; +const std::string kElementName{"Decoder"}; +constexpr gulong kSignalId{123}; +} // namespace + +namespace firebolt::rialto::server::ct +{ +class FirstFrameNotificationTest : public MediaPipelineTest +{ +public: + FirstFrameNotificationTest() + { + m_elementFactory = gst_element_factory_find("fakesrc"); + m_videoDecoder = gst_element_factory_create(m_elementFactory, nullptr); + EXPECT_CALL(*m_gstWrapperMock, gstElementGetFactory(_)).WillRepeatedly(Return(m_elementFactory)); + } + + ~FirstFrameNotificationTest() override + { + gst_object_unref(m_videoDecoder); + gst_object_unref(m_elementFactory); + } + + void setupElementsCommon() + { + EXPECT_CALL(*m_glibWrapperMock, gTypeName(_)).WillRepeatedly(Return(kElementName.c_str())); + EXPECT_CALL(*m_glibWrapperMock, gStrHasPrefix(_, StrEq("amlhalasink"))).WillRepeatedly(Return(FALSE)); + EXPECT_CALL(*m_glibWrapperMock, gStrHasPrefix(_, StrEq("brcmaudiosink"))).WillRepeatedly(Return(FALSE)); + EXPECT_CALL(*m_glibWrapperMock, gStrHasPrefix(_, StrEq("rialtotexttracksink"))).WillRepeatedly(Return(FALSE)); + EXPECT_CALL(*m_gstWrapperMock, gstIsBaseParse(_)).WillRepeatedly(Return(FALSE)); + EXPECT_CALL(*m_glibWrapperMock, gSignalListIds(_, _)) + .WillRepeatedly(Invoke( + [&](GType itype, guint *n_ids) + { + *n_ids = 1; + return m_signals; + })); + EXPECT_CALL(*m_glibWrapperMock, gSignalQuery(m_signals[0], _)) + .WillRepeatedly(Invoke([&](guint signal_id, GSignalQuery *query) + { query->signal_name = "first-video-frame-callback"; })); + EXPECT_CALL(*m_glibWrapperMock, gFree(m_signals)).Times(2); + } + + void willSetupVideoDecoder() + { + EXPECT_CALL(*m_gstWrapperMock, gstObjectRef(m_videoDecoder)).WillOnce(Return(m_videoDecoder)); + + EXPECT_CALL(*m_gstWrapperMock, gstElementFactoryListIsType(m_elementFactory, GST_ELEMENT_FACTORY_TYPE_DECODER)) + .WillOnce(Return(TRUE)); + EXPECT_CALL(*m_gstWrapperMock, + gstElementFactoryListIsType(m_elementFactory, GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO)) + .WillOnce(Return(TRUE)); + EXPECT_CALL(*m_gstWrapperMock, + gstElementFactoryListIsType(m_elementFactory, + GST_ELEMENT_FACTORY_TYPE_SINK | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO)) + .WillOnce(Return(FALSE)) + .RetiresOnSaturation(); + EXPECT_CALL(*m_gstWrapperMock, + gstElementFactoryListIsType(m_elementFactory, GST_ELEMENT_FACTORY_TYPE_DECODER | + GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO)) + .WillOnce(Return(FALSE)) + .RetiresOnSaturation(); + EXPECT_CALL(*m_gstWrapperMock, + gstElementFactoryListIsType(m_elementFactory, + GST_ELEMENT_FACTORY_TYPE_SINK | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO)) + .WillOnce(Return(FALSE)) + .RetiresOnSaturation(); + EXPECT_CALL(*m_gstWrapperMock, + gstElementFactoryListIsType(m_elementFactory, + GST_ELEMENT_FACTORY_TYPE_PARSER | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO)) + .WillOnce(Return(FALSE)) + .RetiresOnSaturation(); + + EXPECT_CALL(*m_glibWrapperMock, gObjectType(m_videoDecoder)).WillRepeatedly(Return(G_TYPE_PARAM)); + EXPECT_CALL(*m_glibWrapperMock, gSignalConnect(_, StrEq("first-video-frame-callback"), _, _)) + .WillOnce(Invoke( + [&](gpointer instance, const gchar *detailed_signal, GCallback c_handler, gpointer data) + { + m_firstVideoFrameCallback = c_handler; + m_firstVideoFrameData = data; + return kSignalId; + })) + .RetiresOnSaturation(); + EXPECT_CALL(*m_gstWrapperMock, gstObjectUnref(m_videoDecoder)) + .WillOnce(Invoke(this, &MediaPipelineTest::workerFinished)); + } + + void setupVideoDecoder() + { + m_gstreamerStub.setupElement(m_videoDecoder); + waitWorker(); + } + + void firstVideoFrameReceived() + { + ExpectMessage expectedFirstFrameReceived{m_clientStub}; + expectedFirstFrameReceived.setFilter([&](const auto &msg) { return msg.source_id() == m_videoSourceId; }); + + ASSERT_TRUE(m_firstVideoFrameCallback); + ASSERT_TRUE(m_firstVideoFrameData); + reinterpret_cast( + m_firstVideoFrameCallback)(m_videoDecoder, 0, nullptr, m_firstVideoFrameData); + + auto receivedFirstFrameReceived{expectedFirstFrameReceived.getMessage()}; + ASSERT_TRUE(receivedFirstFrameReceived); + EXPECT_EQ(receivedFirstFrameReceived->session_id(), m_sessionId); + EXPECT_EQ(receivedFirstFrameReceived->source_id(), m_videoSourceId); + } + +private: + GstElementFactory *m_elementFactory{nullptr}; + GstElement *m_videoDecoder{nullptr}; + guint m_signals[1]{123}; + GCallback m_firstVideoFrameCallback; + gpointer m_firstVideoFrameData{nullptr}; +}; + +/* + * Component Test: First frame notification test + * Test Objective: + * Test if Rialto Server handles gstreamer first frame signals correctly. The notification should be forwarded to + * Rialto Client with FirstFrameReceivedEvent message. + * + * Sequence Diagrams: + * First frame notification + * + * Test Setup: + * Language: C++ + * Testing Framework: Google Test + * Components: MediaPipeline + * + * Test Initialize: + * Set Rialto Server to Active + * Connect Rialto Client Stub + * Map Shared Memory + * + * Test Steps: + * Step 1: Create a new media session + * Send CreateSessionRequest to Rialto Server + * Expect that successful CreateSessionResponse is received + * Save returned session id + * + * Step 2: Load content + * Send LoadRequest to Rialto Server + * Expect that successful LoadResponse is received + * Expect that GstPlayer instance is created. + * Expect that client is notified that the NetworkState has changed to BUFFERING. + * + * Step 3: Setup Video Decoder + * Call SetupElement callback with Video Decoder + * First frame callback should be registered. + * + * Step 4: Attach video source + * Attach the video source. + * Expect that video source is attached. + * Expect that rialto source is setup. + * Expect that all sources are attached. + * Expect that the Playback state has changed to IDLE. + * + * Step 5: Pause + * Pause the content. + * Expect that gstreamer pipeline is paused. + * + * Step 6: Write 1 video frame + * Gstreamer Stub notifies, that it needs video data. + * Expect that server notifies the client that it needs 3 frames of video data. + * Write 1 frame of video data to the shared buffer. + * Send HaveData message. + * Expect that server notifies the client that it needs 3 frames of video data. + * + * Step 7: Notify buffered and Paused + * Expect that server notifies the client that the Network state has changed to BUFFERED. + * Gstreamer Stub notifies, that pipeline state is in PAUSED state. + * Expect that server notifies the client that the Network state has changed to PAUSED. + * + * Step 8: First video frame received + * Rialto Server will receive first video frame signal. + * Rialto Server should send FirstFrameReceivedEvent with video source. + * + * Step 9: End of video stream + * Send video haveData with one frame and EOS status. + * Expect that Gstreamer is notified about end of stream. + * + * Step 10: Notify end of stream + * Simulate, that gst_message_eos is received by Rialto Server. + * Expect that server notifies the client that the Network state has changed to END_OF_STREAM. + * + * Step 11: Remove source + * Remove the video source. + * Expect that video source is removed. + * + * Step 12: Stop + * Stop the playback. + * Expect that stop propagated to the gstreamer pipeline. + * Expect that server notifies the client that the Playback state has changed to STOPPED. + * + * Step 13: Destroy media session + * Send DestroySessionRequest. + * Expect that the session is destroyed on the server. + * + * Test Teardown: + * Memory region created for the shared buffer is unmapped. + * Server is terminated. + * + * Expected Results: + * First frame signal is handled by Rialto Server. + * + * Code: + */ +TEST_F(FirstFrameNotificationTest, firstFrameNotification) +{ + // Step 1: Create a new media session + createSession(); + + // Step 2: Load content + gstPlayerWillBeCreated(); + load(); + + // Step 3: Setup Video Decoder + setupElementsCommon(); + willSetupVideoDecoder(); + setupVideoDecoder(); + + // Step 4: Attach video source + videoSourceWillBeAttached(); + attachVideoSource(); + sourceWillBeSetup(); + setupSource(); + willSetupAndAddSource(&m_videoAppSrc); + willFinishSetupAndAddSource(); + indicateAllSourcesAttached({&m_videoAppSrc}); + + // Step 5: Pause + willPause(); + pause(); + + // Step 6: Write 1 video frame + // Step 7: Notify buffered and Paused + { + ExpectMessage expectedNetworkStateChange{m_clientStub}; + + pushVideoData(kFramesToPush); + + auto receivedNetworkStateChange{expectedNetworkStateChange.getMessage()}; + ASSERT_TRUE(receivedNetworkStateChange); + EXPECT_EQ(receivedNetworkStateChange->session_id(), m_sessionId); + EXPECT_EQ(receivedNetworkStateChange->state(), ::firebolt::rialto::NetworkStateChangeEvent_NetworkState_BUFFERED); + } + willNotifyPaused(); + notifyPaused(); + + // Step 8: First video frame received + firstVideoFrameReceived(); + + // Step 9: End of video stream + willEos(&m_videoAppSrc); + eosVideo(kFramesToPush); + + // Step 10: Notify end of stream + gstNotifyEos(); + + // Step 11: Remove source + removeSource(m_videoSourceId); + + // Step 12: Stop + willStop(); + stop(); + + // Step 13: Destroy media session + gstPlayerWillBeDestructed(); + destroySession(); +} +} // namespace firebolt::rialto::server::ct diff --git a/tests/unittests/media/client/main/mediaPipeline/CallbackTest.cpp b/tests/unittests/media/client/main/mediaPipeline/CallbackTest.cpp index 53b48f0d6..b0c74eef0 100644 --- a/tests/unittests/media/client/main/mediaPipeline/CallbackTest.cpp +++ b/tests/unittests/media/client/main/mediaPipeline/CallbackTest.cpp @@ -98,6 +98,18 @@ TEST_F(RialtoClientMediaPipelineCallbackTest, SourceFlushed) m_mediaPipelineCallback->notifySourceFlushed(sourceId); } +/** + * Test a notification of firstFrameReceived is forwarded to the registered client. + */ +TEST_F(RialtoClientMediaPipelineCallbackTest, FirstFrameReceived) +{ + int32_t sourceId = 1; + + EXPECT_CALL(*m_mediaPipelineClientMock, notifyFirstFrameReceived(sourceId)); + + m_mediaPipelineCallback->notifyFirstFrameReceived(sourceId); +} + /** * Test a notification of playbackInfo is forwarded to the registered client. */ diff --git a/tests/unittests/media/server/gstplayer/CMakeLists.txt b/tests/unittests/media/server/gstplayer/CMakeLists.txt index c2a1c1e02..7ceaaf650 100644 --- a/tests/unittests/media/server/gstplayer/CMakeLists.txt +++ b/tests/unittests/media/server/gstplayer/CMakeLists.txt @@ -35,6 +35,7 @@ add_gtests(RialtoServerGstPlayerUnitTests genericPlayer/tasksTests/EnoughDataTest.cpp genericPlayer/tasksTests/EosTest.cpp genericPlayer/tasksTests/FinishSetupSourceTest.cpp + genericPlayer/tasksTests/FirstFrameReceivedTest.cpp genericPlayer/tasksTests/FlushTest.cpp genericPlayer/tasksTests/GenericPlayerTaskFactoryTest.cpp genericPlayer/tasksTests/HandleBusMessageTest.cpp diff --git a/tests/unittests/media/server/gstplayer/genericPlayer/GstGenericPlayerPrivateTest.cpp b/tests/unittests/media/server/gstplayer/genericPlayer/GstGenericPlayerPrivateTest.cpp index 4544074a8..1ff65a11e 100644 --- a/tests/unittests/media/server/gstplayer/genericPlayer/GstGenericPlayerPrivateTest.cpp +++ b/tests/unittests/media/server/gstplayer/genericPlayer/GstGenericPlayerPrivateTest.cpp @@ -278,6 +278,16 @@ TEST_F(GstGenericPlayerPrivateTest, shouldScheduleVideoUnderflowWithUnderflowDis m_sut->scheduleVideoUnderflow(); } +TEST_F(GstGenericPlayerPrivateTest, shouldScheduleFirstVideoFrameReceived) +{ + std::unique_ptr task{std::make_unique>()}; + EXPECT_CALL(dynamic_cast &>(*task), execute()); + EXPECT_CALL(m_taskFactoryMock, createFirstFrameReceived(_, _, MediaSourceType::VIDEO)) + .WillOnce(Return(ByMove(std::move(task)))); + + m_sut->scheduleFirstVideoFrameReceived(); +} + TEST_F(GstGenericPlayerPrivateTest, shouldNotSetVideoRectangleWhenVideoSinkIsNull) { EXPECT_CALL(*m_glibWrapperMock, gObjectGetStub(_, StrEq(kVideoSinkStr), _)); diff --git a/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/FirstFrameReceivedTest.cpp b/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/FirstFrameReceivedTest.cpp new file mode 100644 index 000000000..1adeffb24 --- /dev/null +++ b/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/FirstFrameReceivedTest.cpp @@ -0,0 +1,57 @@ +/* + * If not stated otherwise in this file or this component's LICENSE file the + * following copyright and licenses apply: + * + * Copyright 2026 Sky UK + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tasks/generic/FirstFrameReceived.h" +#include "GstGenericPlayerClientMock.h" +#include "GstGenericPlayerPrivateMock.h" + +#include +#include + +using testing::StrictMock; + +namespace +{ +constexpr firebolt::rialto::MediaSourceType kSourceType{firebolt::rialto::MediaSourceType::VIDEO}; +} // namespace + +class FirstFrameReceivedTest : public testing::Test +{ +protected: + firebolt::rialto::server::GenericPlayerContext m_context; + StrictMock m_gstPlayer; + StrictMock m_gstPlayerClient; +}; + +TEST_F(FirstFrameReceivedTest, shouldNotifyFirstFrameReceived) +{ + firebolt::rialto::server::tasks::generic::FirstFrameReceived task{m_context, m_gstPlayer, &m_gstPlayerClient, + kSourceType}; + + EXPECT_CALL(m_gstPlayerClient, notifyFirstFrameReceived(kSourceType)); + + task.execute(); +} + +TEST_F(FirstFrameReceivedTest, shouldNotNotifyFirstFrameReceivedWhenClientIsNull) +{ + firebolt::rialto::server::tasks::generic::FirstFrameReceived task{m_context, m_gstPlayer, nullptr, kSourceType}; + + task.execute(); +} diff --git a/tests/unittests/media/server/ipc/mediaPipelineModuleService/MediaPipelineModuleServiceTests.cpp b/tests/unittests/media/server/ipc/mediaPipelineModuleService/MediaPipelineModuleServiceTests.cpp index 472dff026..45a4ff170 100644 --- a/tests/unittests/media/server/ipc/mediaPipelineModuleService/MediaPipelineModuleServiceTests.cpp +++ b/tests/unittests/media/server/ipc/mediaPipelineModuleService/MediaPipelineModuleServiceTests.cpp @@ -361,6 +361,14 @@ TEST_F(MediaPipelineModuleServiceTests, shouldSendPlaybackErrorEvent) sendPlaybackErrorEvent(); } +TEST_F(MediaPipelineModuleServiceTests, shouldSendFirstFrameReceivedEvent) +{ + mediaPipelineServiceWillCreateSession(); + sendCreateSessionRequestAndReceiveResponse(); + mediaClientWillSendFirstFrameReceivedEvent(); + sendFirstFrameReceivedEvent(); +} + TEST_F(MediaPipelineModuleServiceTests, shouldSendSourceFlushedEvent) { mediaPipelineServiceWillCreateSession(); diff --git a/tests/unittests/media/server/ipc/mediaPipelineModuleService/MediaPipelineModuleServiceTestsFixture.cpp b/tests/unittests/media/server/ipc/mediaPipelineModuleService/MediaPipelineModuleServiceTestsFixture.cpp index 27cd7bd40..0df902b1d 100644 --- a/tests/unittests/media/server/ipc/mediaPipelineModuleService/MediaPipelineModuleServiceTestsFixture.cpp +++ b/tests/unittests/media/server/ipc/mediaPipelineModuleService/MediaPipelineModuleServiceTestsFixture.cpp @@ -204,6 +204,13 @@ MATCHER_P2(PlaybackErrorEventMatcher, kExpectedSourceId, kExpectedPlaybackError, return ((kExpectedSourceId == event->source_id()) && (kExpectedPlaybackError == event->error())); } +MATCHER_P(FirstFrameReceivedEventMatcher, kSourceId, "") +{ + std::shared_ptr event = + std::dynamic_pointer_cast(arg); + return (kSourceId == event->source_id()); +} + MATCHER_P(PlaybackStateChangeEventMatcher, kPlaybackState, "") { std::shared_ptr event = @@ -873,6 +880,11 @@ void MediaPipelineModuleServiceTests::mediaClientWillSendPlaybackErrorEvent() EXPECT_CALL(*m_clientMock, sendEvent(PlaybackErrorEventMatcher(kSourceId, convertPlaybackError(kPlaybackError)))); } +void MediaPipelineModuleServiceTests::mediaClientWillSendFirstFrameReceivedEvent() +{ + EXPECT_CALL(*m_clientMock, sendEvent(FirstFrameReceivedEventMatcher(kSourceId))); +} + void MediaPipelineModuleServiceTests::mediaClientWillSendSourceFlushedEvent() { EXPECT_CALL(*m_clientMock, sendEvent(SourceFlushedEventMatcher(kSourceId))); @@ -1591,6 +1603,12 @@ void MediaPipelineModuleServiceTests::sendPlaybackErrorEvent() m_mediaPipelineClient->notifyPlaybackError(kSourceId, kPlaybackError); } +void MediaPipelineModuleServiceTests::sendFirstFrameReceivedEvent() +{ + ASSERT_TRUE(m_mediaPipelineClient); + m_mediaPipelineClient->notifyFirstFrameReceived(kSourceId); +} + void MediaPipelineModuleServiceTests::sendSourceFlushedEvent() { ASSERT_TRUE(m_mediaPipelineClient); diff --git a/tests/unittests/media/server/ipc/mediaPipelineModuleService/MediaPipelineModuleServiceTestsFixture.h b/tests/unittests/media/server/ipc/mediaPipelineModuleService/MediaPipelineModuleServiceTestsFixture.h index fe66ac0e1..19ebe07a0 100644 --- a/tests/unittests/media/server/ipc/mediaPipelineModuleService/MediaPipelineModuleServiceTestsFixture.h +++ b/tests/unittests/media/server/ipc/mediaPipelineModuleService/MediaPipelineModuleServiceTestsFixture.h @@ -129,6 +129,7 @@ class MediaPipelineModuleServiceTests : public testing::Test void mediaClientWillSendPostionChangeEvent(); void mediaClientWillSendQosEvent(); void mediaClientWillSendPlaybackErrorEvent(); + void mediaClientWillSendFirstFrameReceivedEvent(); void mediaClientWillSendSourceFlushedEvent(); void mediaClientWillSendPlaybackInfoEvent(); @@ -196,6 +197,7 @@ class MediaPipelineModuleServiceTests : public testing::Test void sendPostionChangeEvent(); void sendQosEvent(); void sendPlaybackErrorEvent(); + void sendFirstFrameReceivedEvent(); void sendSourceFlushedEvent(); void sendPlaybackInfoEvent(); void sendRenderFrameRequestAndReceiveResponse(); diff --git a/tests/unittests/media/server/main/mediaPipeline/CallbackTest.cpp b/tests/unittests/media/server/main/mediaPipeline/CallbackTest.cpp index 627f111d5..23ef98e1c 100644 --- a/tests/unittests/media/server/main/mediaPipeline/CallbackTest.cpp +++ b/tests/unittests/media/server/main/mediaPipeline/CallbackTest.cpp @@ -269,3 +269,29 @@ TEST_F(RialtoServerMediaPipelineCallbackTest, notifySourceFlushedFailureSourceId m_gstPlayerCallback->notifySourceFlushed(mediaSourceType); } + +/** + * Test a notification of first frame received is forwarded to the registered client. + */ +TEST_F(RialtoServerMediaPipelineCallbackTest, notifyFirstFrameReceived) +{ + auto mediaSourceType = firebolt::rialto::MediaSourceType::VIDEO; + int sourceId = attachSource(mediaSourceType, "video/mp4"); + + mainThreadWillEnqueueTask(); + EXPECT_CALL(*m_mediaPipelineClientMock, notifyFirstFrameReceived(sourceId)); + + m_gstPlayerCallback->notifyFirstFrameReceived(mediaSourceType); +} + +/** + * Test a notification of first frame received fails when sourceid cannot be found. + */ +TEST_F(RialtoServerMediaPipelineCallbackTest, notifyFirstFrameReceivedFailureSourceIdNotFound) +{ + auto mediaSourceType = firebolt::rialto::MediaSourceType::VIDEO; + + mainThreadWillEnqueueTask(); + + m_gstPlayerCallback->notifyFirstFrameReceived(mediaSourceType); +} From 1547793940a90c4c77b153ebc058bf3524b82f11 Mon Sep 17 00:00:00 2001 From: Sasa Mudri Date: Thu, 14 May 2026 16:56:52 +0200 Subject: [PATCH 5/7] Adding fixes for reported issues by github copilot. --- media/server/gstplayer/source/tasks/generic/SetupElement.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media/server/gstplayer/source/tasks/generic/SetupElement.cpp b/media/server/gstplayer/source/tasks/generic/SetupElement.cpp index 3cedc8a7d..e2e7efd74 100644 --- a/media/server/gstplayer/source/tasks/generic/SetupElement.cpp +++ b/media/server/gstplayer/source/tasks/generic/SetupElement.cpp @@ -62,7 +62,7 @@ void videoUnderflowCallback(GstElement *object, guint fifoDepth, gpointer queueD } /** - * @brief Callback for first video frame event from sink. Called by the Gstreamer thread. + * @brief Callback for first video frame event from the emitting video element. Called by the Gstreamer thread. * * @param[in] object : the object that emitted the signal * @param[in] fifoDepth : the fifo depth (may be 0) From ea67730007965e9e573f07e249107173c357d219 Mon Sep 17 00:00:00 2001 From: N Date: Mon, 1 Jun 2026 22:09:43 +0530 Subject: [PATCH 6/7] audio_ff on top of video_ff --- .../gstplayer/include/GenericPlayerContext.h | 15 ++ .../gstplayer/include/GstGenericPlayer.h | 4 + .../include/IGstGenericPlayerPrivate.h | 23 +++ .../gstplayer/source/GstGenericPlayer.cpp | 52 +++++++ media/server/gstplayer/source/Utils.cpp | 2 +- .../source/tasks/generic/AttachSource.cpp | 3 + .../source/tasks/generic/SetupElement.cpp | 60 ++++++++ .../gstplayer/source/tasks/generic/Stop.cpp | 2 + .../changes/audio-first-frame/.openspec.yaml | 2 + openspec/changes/audio-first-frame/design.md | 86 +++++++++++ .../changes/audio-first-frame/proposal.md | 26 ++++ .../specs/audio-first-frame/spec.md | 47 +++++++ openspec/changes/audio-first-frame/tasks.md | 33 +++++ .../tests/base/MediaPipelineTestMethods.cpp | 12 ++ .../tests/base/MediaPipelineTestMethods.h | 2 + .../tests/mse/FirstFrameNotificationTest.cpp | 10 +- .../FirstFrameNotificationTest.cpp | 133 +++++++++++++++++- .../GstGenericPlayerPrivateTest.cpp | 10 ++ .../common/GenericTasksTestsBase.cpp | 37 +++++ .../common/GenericTasksTestsBase.h | 4 + .../common/GenericTasksTestsContext.h | 1 + .../tasksTests/FirstFrameReceivedTest.cpp | 11 ++ .../tasksTests/SetupElementTest.cpp | 9 ++ .../gstplayer/GstGenericPlayerPrivateMock.h | 4 + 24 files changed, 583 insertions(+), 5 deletions(-) create mode 100644 openspec/changes/audio-first-frame/.openspec.yaml create mode 100644 openspec/changes/audio-first-frame/design.md create mode 100644 openspec/changes/audio-first-frame/proposal.md create mode 100644 openspec/changes/audio-first-frame/specs/audio-first-frame/spec.md create mode 100644 openspec/changes/audio-first-frame/tasks.md diff --git a/media/server/gstplayer/include/GenericPlayerContext.h b/media/server/gstplayer/include/GenericPlayerContext.h index af72f88a6..8f2f9ce86 100644 --- a/media/server/gstplayer/include/GenericPlayerContext.h +++ b/media/server/gstplayer/include/GenericPlayerContext.h @@ -276,6 +276,21 @@ struct GenericPlayerContext * @brief Profiler for player pipeline */ std::unique_ptr gstProfiler; + + /** + * @brief True when first audio frame has already been scheduled for the current audio source lifecycle. + */ + bool firstAudioFrameReceived{false}; + + /** + * @brief Fallback probe id for first audio frame detection on sink pad. + */ + gulong audioFirstFrameProbeId{0}; + + /** + * @brief Fallback sink pad that owns audio first frame probe. + */ + GstPad *audioFirstFrameProbePad{nullptr}; }; } // namespace firebolt::rialto::server diff --git a/media/server/gstplayer/include/GstGenericPlayer.h b/media/server/gstplayer/include/GstGenericPlayer.h index bdd3d4a39..8cf70be38 100644 --- a/media/server/gstplayer/include/GstGenericPlayer.h +++ b/media/server/gstplayer/include/GstGenericPlayer.h @@ -155,6 +155,10 @@ class GstGenericPlayer : public IGstGenericPlayer, public IGstGenericPlayerPriva void scheduleAudioUnderflow() override; void scheduleVideoUnderflow() override; void scheduleFirstVideoFrameReceived() override; + void scheduleFirstAudioFrameReceived() override; + void setAudioFirstFrameFallbackProbe(GstPad *pad, gulong id) override; + void clearAudioFirstFrameFallbackProbe() override; + void clearAudioFirstFrameFallbackProbeState() override; void scheduleAllSourcesAttached() override; bool setVideoSinkRectangle() override; bool setImmediateOutput() override; diff --git a/media/server/gstplayer/include/IGstGenericPlayerPrivate.h b/media/server/gstplayer/include/IGstGenericPlayerPrivate.h index 139078515..68c8452f2 100644 --- a/media/server/gstplayer/include/IGstGenericPlayerPrivate.h +++ b/media/server/gstplayer/include/IGstGenericPlayerPrivate.h @@ -66,6 +66,29 @@ class IGstGenericPlayerPrivate */ virtual void scheduleFirstVideoFrameReceived() = 0; + /** + * @brief Schedules first audio frame received task. Called by the worker thread. + */ + virtual void scheduleFirstAudioFrameReceived() = 0; + + /** + * @brief Stores audio first-frame fallback probe state. + * + * @param[in] pad : sink pad with installed probe + * @param[in] id : probe id + */ + virtual void setAudioFirstFrameFallbackProbe(GstPad *pad, gulong id) = 0; + + /** + * @brief Removes and clears audio first-frame fallback probe state. + */ + virtual void clearAudioFirstFrameFallbackProbe() = 0; + + /** + * @brief Clears audio first-frame fallback probe state without removing the probe. + */ + virtual void clearAudioFirstFrameFallbackProbeState() = 0; + /** * @brief Schedules all sources attached task. Called by the worker thread. */ diff --git a/media/server/gstplayer/source/GstGenericPlayer.cpp b/media/server/gstplayer/source/GstGenericPlayer.cpp index c6634e929..2acf06555 100644 --- a/media/server/gstplayer/source/GstGenericPlayer.cpp +++ b/media/server/gstplayer/source/GstGenericPlayer.cpp @@ -281,6 +281,8 @@ void GstGenericPlayer::resetWorkerThread() void GstGenericPlayer::termPipeline() { + clearAudioFirstFrameFallbackProbe(); + if (m_finishSourceSetupTimer && m_finishSourceSetupTimer->isActive()) { m_finishSourceSetupTimer->cancel(); @@ -1682,11 +1684,61 @@ void GstGenericPlayer::scheduleFirstVideoFrameReceived() } } +void GstGenericPlayer::scheduleFirstAudioFrameReceived() +{ + if (m_context.firstAudioFrameReceived) + { + return; + } + + m_context.firstAudioFrameReceived = true; + + if (m_workerThread) + { + m_workerThread->enqueueTask(m_taskFactory->createFirstFrameReceived(m_context, *this, MediaSourceType::AUDIO)); + } +} + +void GstGenericPlayer::setAudioFirstFrameFallbackProbe(GstPad *pad, gulong id) +{ + clearAudioFirstFrameFallbackProbe(); + + m_context.audioFirstFrameProbePad = pad; + m_context.audioFirstFrameProbeId = id; +} + +void GstGenericPlayer::clearAudioFirstFrameFallbackProbe() +{ + if (m_context.audioFirstFrameProbePad && m_context.audioFirstFrameProbeId != 0) + { + m_gstWrapper->gstPadRemoveProbe(m_context.audioFirstFrameProbePad, m_context.audioFirstFrameProbeId); + } + + clearAudioFirstFrameFallbackProbeState(); +} + +void GstGenericPlayer::clearAudioFirstFrameFallbackProbeState() +{ + if (m_context.audioFirstFrameProbePad) + { + m_gstWrapper->gstObjectUnref(m_context.audioFirstFrameProbePad); + m_context.audioFirstFrameProbePad = nullptr; + } + + m_context.audioFirstFrameProbeId = 0; +} + void GstGenericPlayer::scheduleAllSourcesAttached() { allSourcesAttached(); } + if (mediaSourceType == MediaSourceType::AUDIO) + { + m_context.firstAudioFrameReceived = false; + clearAudioFirstFrameFallbackProbe(); + } + void GstGenericPlayer::cancelUnderflow(firebolt::rialto::MediaSourceType mediaSource) { auto elem = m_context.streamInfo.find(mediaSource); diff --git a/media/server/gstplayer/source/Utils.cpp b/media/server/gstplayer/source/Utils.cpp index fd93a2323..9d8ecdcd5 100644 --- a/media/server/gstplayer/source/Utils.cpp +++ b/media/server/gstplayer/source/Utils.cpp @@ -29,7 +29,7 @@ namespace { const char *underflowSignals[]{"buffer-underflow-callback", "vidsink-underflow-callback", "underrun-callback"}; -const char *firstFrameSignals[]{"first-video-frame-callback"}; +const char *firstFrameSignals[]{"first-video-frame-callback", "first-audio-frame", "first-audio-frame-callback"}; bool isType(const firebolt::rialto::wrappers::IGstWrapper &gstWrapper, GstElement *element, GstElementFactoryListType type) { diff --git a/media/server/gstplayer/source/tasks/generic/AttachSource.cpp b/media/server/gstplayer/source/tasks/generic/AttachSource.cpp index 8ca2b2405..cd5ed8efc 100644 --- a/media/server/gstplayer/source/tasks/generic/AttachSource.cpp +++ b/media/server/gstplayer/source/tasks/generic/AttachSource.cpp @@ -82,6 +82,7 @@ void AttachSource::addSource() const GstElement *appSrc = nullptr; if (m_attachedSource->getType() == MediaSourceType::AUDIO) { + m_context.firstAudioFrameReceived = false; RIALTO_SERVER_LOG_MIL("Adding Audio appsrc with caps %s", capsStr); appSrc = m_gstWrapper->gstElementFactoryMake("appsrc", "audsrc"); profilerInfo = "audsrc"; @@ -124,6 +125,8 @@ void AttachSource::addSource() const void AttachSource::reattachAudioSource() const { + m_context.firstAudioFrameReceived = false; + if (!m_player.reattachSource(m_attachedSource)) { RIALTO_SERVER_LOG_ERROR("Reattaching source failed!"); diff --git a/media/server/gstplayer/source/tasks/generic/SetupElement.cpp b/media/server/gstplayer/source/tasks/generic/SetupElement.cpp index e2e7efd74..bf48bec26 100644 --- a/media/server/gstplayer/source/tasks/generic/SetupElement.cpp +++ b/media/server/gstplayer/source/tasks/generic/SetupElement.cpp @@ -76,6 +76,39 @@ void firstVideoFrameCallback(GstElement *object, guint fifoDepth, gpointer queue player->scheduleFirstVideoFrameReceived(); } +/** + * @brief Callback for first audio frame event from the emitting audio element. Called by the Gstreamer thread. + * + * @param[in] object : the object that emitted the signal + * @param[in] fifoDepth : the fifo depth (may be 0) + * @param[in] queueDepth : the queue depth (may be NULL) + * @param[in] self : The pointer to IGstGenericPlayerPrivate + */ +void firstAudioFrameCallback(GstElement *object, guint fifoDepth, gpointer queueDepth, gpointer self) +{ + firebolt::rialto::server::IGstGenericPlayerPrivate *player = + static_cast(self); + player->clearAudioFirstFrameFallbackProbe(); + player->scheduleFirstAudioFrameReceived(); +} + +/** + * @brief Fallback probe callback for first audio frame on sink pad. + */ +GstPadProbeReturn firstAudioFrameProbeCallback(GstPad *pad, GstPadProbeInfo *info, gpointer self) +{ + if (!(info->type & GST_PAD_PROBE_TYPE_BUFFER) || !GST_PAD_PROBE_INFO_BUFFER(info)) + { + return GST_PAD_PROBE_OK; + } + + firebolt::rialto::server::IGstGenericPlayerPrivate *player = + static_cast(self); + player->clearAudioFirstFrameFallbackProbeState(); + player->scheduleFirstAudioFrameReceived(); + return GST_PAD_PROBE_REMOVE; +} + /** * @brief Callback for a autovideosink when a child has been added to the sink. * @@ -295,6 +328,33 @@ void SetupElement::execute() const m_glibWrapper->gSignalConnect(m_element, firstFrameSignalName.value().c_str(), G_CALLBACK(firstVideoFrameCallback), &m_player); } + else if (isAudio(*m_gstWrapper, m_element)) + { + RIALTO_SERVER_LOG_INFO("Connecting first audio frame callback for signal: %s", + firstFrameSignalName.value().c_str()); + m_glibWrapper->gSignalConnect(m_element, firstFrameSignalName.value().c_str(), + G_CALLBACK(firstAudioFrameCallback), &m_player); + } + } + else if (isAudioSink(*m_gstWrapper, m_element)) + { + GstPad *sinkPad = m_gstWrapper->gstElementGetStaticPad(m_element, "sink"); + if (sinkPad) + { + gulong probeId = + m_gstWrapper->gstPadAddProbe(sinkPad, GST_PAD_PROBE_TYPE_BUFFER, firstAudioFrameProbeCallback, + &m_player, nullptr); + + if (probeId != 0) + { + RIALTO_SERVER_LOG_INFO("Installed first audio frame fallback probe on sink"); + m_player.setAudioFirstFrameFallbackProbe(sinkPad, probeId); + } + else + { + m_gstWrapper->gstObjectUnref(sinkPad); + } + } } } diff --git a/media/server/gstplayer/source/tasks/generic/Stop.cpp b/media/server/gstplayer/source/tasks/generic/Stop.cpp index bc566d6a7..7bfaaeaf4 100644 --- a/media/server/gstplayer/source/tasks/generic/Stop.cpp +++ b/media/server/gstplayer/source/tasks/generic/Stop.cpp @@ -37,6 +37,8 @@ Stop::~Stop() void Stop::execute() const { RIALTO_SERVER_LOG_DEBUG("Executing Stop"); + m_context.firstAudioFrameReceived = false; + m_player.clearAudioFirstFrameFallbackProbe(); m_player.stopPositionReportingAndCheckAudioUnderflowTimer(); m_player.stopNotifyPlaybackInfoTimer(); m_player.changePipelineState(GST_STATE_NULL); diff --git a/openspec/changes/audio-first-frame/.openspec.yaml b/openspec/changes/audio-first-frame/.openspec.yaml new file mode 100644 index 000000000..a2168c37b --- /dev/null +++ b/openspec/changes/audio-first-frame/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-06-01 diff --git a/openspec/changes/audio-first-frame/design.md b/openspec/changes/audio-first-frame/design.md new file mode 100644 index 000000000..253af7f33 --- /dev/null +++ b/openspec/changes/audio-first-frame/design.md @@ -0,0 +1,86 @@ +## Context + +Rialto already supports first-frame notification for video and already has the downstream plumbing needed to propagate a first-frame event from gstplayer through the worker thread, server, IPC transport, and client callback. Audio playback currently lacks equivalent detection, which creates an observability gap even though the existing notification path is capable of carrying an audio first-frame event once one is detected. + +This change touches multiple layers: gstplayer element setup, first-frame scheduling, server-side forwarding, IPC transport, and client delivery. The design therefore focuses on how audio first-frame detection is introduced without changing the public callback or protobuf contract and without regressing the existing video path. + +## Goals / Non-Goals + +**Goals:** +- Add first-audio-frame detection in gstplayer using capability-based discovery instead of platform branching. +- Reuse the existing first-frame task scheduling and propagation path by routing audio notifications with `MediaSourceType::AUDIO`. +- Support audio sink implementations that do not expose callback-based first-frame detection by using a bounded fallback probe. +- Preserve existing public client callbacks and existing `FirstFrameReceivedEvent` IPC transport. +- Keep video first-frame behavior unchanged. + +**Non-Goals:** +- Redesign the current first-frame architecture. +- Introduce platform- or vendor-specific branches for audio first-frame support. +- Add a new public callback or a new protobuf event type for audio first-frame notification. +- Change behavior outside the first-frame detection and propagation path. + +## Decisions + +### Use role-specific capability detection during element setup + +Audio first-frame support will be discovered when gstplayer configures pipeline elements. Audio decoder elements will be checked for `first-audio-frame`, while audio sink elements will be checked for `first-audio-frame-callback`. + +This keeps detection aligned with the existing setup flow and avoids hard-coding platform names or element allowlists. It also lets decoder and sink behavior diverge cleanly where platform implementations expose different signaling mechanisms. + +Alternatives considered: +- Use platform-specific branching: rejected because it scales poorly across vendors and hard-codes knowledge that should remain capability based. +- Use a probe for all audio elements: rejected because native callback/signal support is cheaper and more precise when available. + +### Use sink-pad probing only as a fallback for audio sinks + +If an audio sink does not expose `first-audio-frame-callback`, Rialto will install a sink pad probe and treat the first valid audio buffer as the first-frame trigger. The probe will be removed immediately after the first valid trigger and also removed during teardown, flush, reset, or pipeline stop. + +This provides coverage for sink implementations that cannot emit a callback while keeping probe usage narrow and bounded. Restricting the fallback to audio sinks avoids adding speculative probe behavior to decoder elements where the expected mechanism is explicit capability detection. + +Alternatives considered: +- No fallback path: rejected because platforms without sink callback capability would remain unsupported. +- Probe both decoders and sinks: rejected because it broadens runtime overhead and increases ambiguity in which stage constitutes first-frame detection. + +### Reuse the existing first-frame scheduling and propagation path + +Once audio first-frame is detected, the implementation will schedule the existing first-frame handling flow and identify the source using `MediaSourceType::AUDIO`. Server-side forwarding, IPC serialization, and client dispatch will continue to use the existing `notifyFirstFrameReceived(...)` and `FirstFrameReceivedEvent` path. + +This minimizes surface area, keeps the client contract stable, and avoids introducing a parallel audio-specific notification stack. + +Alternatives considered: +- Add a dedicated audio-first-frame event type: rejected because it duplicates the current pipeline without adding functional value. +- Dispatch directly from gstplayer to clients: rejected because it bypasses existing threading and session/source mapping behavior. + +### Guard emission so exactly one first-frame event is sent per audio source lifecycle + +Both callback-based and probe-based detection may observe the same source lifecycle. The implementation will keep a one-shot guard for the relevant audio source/session so only one first-frame event is scheduled and propagated. Cleanup paths will remove probe state and clear any temporary tracking during teardown and reset operations. + +This preserves deterministic client behavior and prevents duplicate telemetry or callback delivery. + +Alternatives considered: +- Allow duplicate low-level detections and deduplicate later: rejected because duplicates would still traverse more of the pipeline and complicate server/client behavior. + +## Risks / Trade-offs + +- Duplicate detection from callback and probe paths -> Mitigation: keep a one-shot guard at the first scheduling point and remove fallback probes immediately after the first valid trigger. +- Incorrect trigger on non-audio or non-buffer sink activity -> Mitigation: only treat the first valid audio buffer as a trigger and ignore unrelated pad activity. +- Regression in existing video behavior -> Mitigation: leave the video detection path unchanged and run existing first-frame non-regression coverage alongside new audio tests. +- Threading issues if detection performs too much work on the GStreamer thread -> Mitigation: keep callbacks and probes minimal and schedule work onto the existing worker-thread path. +- Platform variability in element support -> Mitigation: prefer capability-based detection and limit fallback logic to the specific sink case where callback support is absent. + +## Migration Plan + +1. Implement audio capability detection during gstplayer setup for decoders and sinks. +2. Add sink fallback probe handling, including install, one-shot trigger, and cleanup paths. +3. Route detected audio first-frame events through the existing scheduling and propagation flow using `MediaSourceType::AUDIO`. +4. Add or update unit tests for setup-element detection behavior, fallback probing, and single-emission guarantees. +5. Add or update component coverage for end-to-end server and client delivery of audio first-frame notifications. +6. Validate that existing video first-frame tests remain unchanged. + +Rollback is low risk because the change is additive. If regressions are found, audio detection and fallback wiring can be removed while leaving the existing video path and notification contract intact. + +## Open Questions + +- Which exact gstplayer setup abstraction should own the audio sink probe lifecycle so cleanup is guaranteed across all teardown paths? +- Does any supported platform expose both decoder and sink audio first-frame signals simultaneously, and if so, where should the one-shot guard live to keep behavior deterministic? +- Are there existing test fixtures for sink pad probe behavior that can be extended, or does this change require new unit helpers for probe installation and cleanup validation? \ No newline at end of file diff --git a/openspec/changes/audio-first-frame/proposal.md b/openspec/changes/audio-first-frame/proposal.md new file mode 100644 index 000000000..c49fa4f40 --- /dev/null +++ b/openspec/changes/audio-first-frame/proposal.md @@ -0,0 +1,26 @@ +## Why + +Rialto already reports first-frame receipt for video, but it does not provide equivalent signaling for audio. Adding audio first-frame support closes that observability gap now that the existing first-frame pipeline already covers the worker-thread, server, IPC, and client callback path needed to carry the event end to end. + +## What Changes + +- Extend gstplayer setup to detect first-audio-frame support using element capabilities instead of platform-specific branching. +- Use `first-audio-frame` for audio decoder elements and `first-audio-frame-callback` for audio sink elements when those capabilities are available. +- Add a sink-only fallback probe when an audio sink does not expose callback support, and remove it immediately after the first valid audio buffer is observed. +- Reuse the existing first-frame scheduling, server forwarding, IPC event, and client callback path for audio by routing the event with `MediaSourceType::AUDIO`. +- Preserve the existing public callback and protobuf contract so the change remains additive and non-breaking. + +## Capabilities + +### New Capabilities +- `audio-first-frame`: Detect and propagate the first rendered audio frame through the existing first-frame notification pipeline. + +### Modified Capabilities + +None. + +## Impact + +Affected areas include gstplayer element setup and first-frame detection, worker-thread task scheduling for first-frame handling, server-side forwarding via `notifyFirstFrameReceived(...)`, IPC transport using `FirstFrameReceivedEvent`, and client-side forwarding to `notifyFirstFrameReceived(sourceId)`. + +This change does not introduce a new public callback or protobuf message, but it does require unit and component coverage for audio capability detection, sink probe fallback, one-shot event emission, and video non-regression. diff --git a/openspec/changes/audio-first-frame/specs/audio-first-frame/spec.md b/openspec/changes/audio-first-frame/specs/audio-first-frame/spec.md new file mode 100644 index 000000000..5585e9102 --- /dev/null +++ b/openspec/changes/audio-first-frame/specs/audio-first-frame/spec.md @@ -0,0 +1,47 @@ +## ADDED Requirements + +### Requirement: Audio decoder first-frame detection +The system SHALL detect audio first-frame support on audio decoder elements by checking for the `first-audio-frame` capability during element setup. + +#### Scenario: Audio decoder exposes first-frame capability +- **WHEN** an audio decoder element exposes `first-audio-frame` during setup +- **THEN** the system connects that capability for the audio source +- **THEN** the first detected audio frame is scheduled through the existing first-frame handling path + +### Requirement: Audio sink first-frame fallback +The system SHALL detect audio first-frame support on audio sink elements by using `first-audio-frame-callback` when available and SHALL install a sink pad probe only when that callback capability is unavailable. + +#### Scenario: Audio sink exposes callback capability +- **WHEN** an audio sink element exposes `first-audio-frame-callback` during setup +- **THEN** the system uses that callback capability for first-frame detection +- **THEN** no fallback sink pad probe is installed for that sink + +#### Scenario: Audio sink does not expose callback capability +- **WHEN** an audio sink element does not expose `first-audio-frame-callback` during setup +- **THEN** the system installs a sink pad probe for that sink +- **THEN** the first valid audio buffer observed by the probe triggers first-frame handling +- **THEN** the probe is removed immediately after the first valid audio buffer is observed + +### Requirement: Single audio first-frame notification +The system MUST emit exactly one first-frame notification for each relevant audio source lifecycle even if both capability-based and probe-based detection paths become active. + +#### Scenario: Multiple detection paths observe the same first frame +- **WHEN** capability-based detection and fallback probe detection both observe the same audio source lifecycle +- **THEN** the system emits only one first-frame notification for that audio source + +#### Scenario: Audio source ends before first frame +- **WHEN** an audio source is flushed, reset, torn down, or the pipeline stops before the first audio frame is emitted +- **THEN** the system removes any installed fallback probe without emitting a duplicate or stale first-frame notification + +### Requirement: Existing first-frame contract reuse +The system SHALL propagate audio first-frame notifications through the existing first-frame pipeline using `MediaSourceType::AUDIO` without introducing a new public callback or protobuf event. + +#### Scenario: Audio first frame reaches the client +- **WHEN** the system detects the first frame for an audio source +- **THEN** it forwards the event through the existing worker-thread, server, IPC, and client callback path +- **THEN** the client receives the existing `notifyFirstFrameReceived(sourceId)` callback for the audio source + +#### Scenario: Existing protocol remains unchanged +- **WHEN** audio first-frame support is added +- **THEN** the system continues to use the existing `FirstFrameReceivedEvent` transport shape +- **THEN** no new public callback name or protobuf message is required \ No newline at end of file diff --git a/openspec/changes/audio-first-frame/tasks.md b/openspec/changes/audio-first-frame/tasks.md new file mode 100644 index 000000000..081a555c8 --- /dev/null +++ b/openspec/changes/audio-first-frame/tasks.md @@ -0,0 +1,33 @@ +## 1. Gstplayer Detection Wiring + +- [x] 1.1 Update signal lookup to include audio first-frame signal names in media/server/gstplayer/source/Utils.cpp. +- [x] 1.2 Add audio first-frame callback connection in media/server/gstplayer/source/tasks/generic/SetupElement.cpp. +- [x] 1.3 Add audio first-frame scheduler API in media/server/gstplayer/include/IGstGenericPlayerPrivate.h and media/server/gstplayer/include/GstGenericPlayer.h. +- [x] 1.4 Implement audio first-frame scheduling with MediaSourceType::AUDIO in media/server/gstplayer/source/GstGenericPlayer.cpp. + +## 2. Audio Sink Fallback Probe + +- [x] 2.1 Implement sink pad probe installation for audio sinks without callback capability in media/server/gstplayer/source/tasks/generic/SetupElement.cpp. +- [x] 2.2 Add probe state storage and lifecycle hooks in media/server/gstplayer/source/GstGenericPlayer.cpp and media/server/gstplayer/include/GstGenericPlayer.h. +- [x] 2.3 Remove probe on first trigger and terminal paths (flush/reset/stop/teardown) in media/server/gstplayer/source/GstGenericPlayer.cpp. + +## 3. One-Shot Emission Guard + +- [x] 3.1 Add a per-audio-source one-shot first-frame guard in media/server/gstplayer/source/GstGenericPlayer.cpp. +- [x] 3.2 Wire guard checks in both callback and probe paths in media/server/gstplayer/source/tasks/generic/SetupElement.cpp. +- [x] 3.3 Extend private-interface mocks for new audio scheduler hooks in tests/unittests/media/server/mocks/gstplayer/GstGenericPlayerPrivateMock.h. + +## 4. End-to-End Notification Reuse + +- [ ] 4.1 Validate audio source-id mapping path in media/server/main/source/MediaPipelineServerInternal.cpp (no API changes expected). +- [ ] 4.2 Validate server IPC event emission remains unchanged in media/server/ipc/source/MediaPipelineClient.cpp and proto/mediapipelinemodule.proto. +- [ ] 4.3 Validate client IPC and callback path remains unchanged in media/client/ipc/source/MediaPipelineIpc.cpp and media/client/main/source/MediaPipeline.cpp. + +## 5. Test Coverage and Validation + +- [x] 5.1 Add setup-task unit coverage for audio first-frame signal hookup in tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/SetupElementTest.cpp and tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.cpp. +- [x] 5.2 Add private-player unit coverage for audio first-frame scheduling in tests/unittests/media/server/gstplayer/genericPlayer/GstGenericPlayerPrivateTest.cpp. +- [x] 5.3 Add/update first-frame task behavior assertions in tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/FirstFrameReceivedTest.cpp. +- [x] 5.4 Add/update server component first-frame flow for audio source in tests/componenttests/server/tests/mediaPipeline/FirstFrameNotificationTest.cpp. +- [x] 5.5 Add/update client component first-frame flow for audio source in tests/componenttests/client/tests/base/MediaPipelineTestMethods.cpp and tests/componenttests/client/tests/mse/FirstFrameNotificationTest.cpp. +- [ ] 5.6 Run openspec validate audio-first-frame --type change --json --no-interactive and execute affected unit/component test targets for non-regression. \ No newline at end of file diff --git a/tests/componenttests/client/tests/base/MediaPipelineTestMethods.cpp b/tests/componenttests/client/tests/base/MediaPipelineTestMethods.cpp index 63658d247..38edbd14f 100644 --- a/tests/componenttests/client/tests/base/MediaPipelineTestMethods.cpp +++ b/tests/componenttests/client/tests/base/MediaPipelineTestMethods.cpp @@ -1381,6 +1381,18 @@ void MediaPipelineTestMethods::shouldNotifyFirstFrameReceivedVideo() .WillOnce(Invoke(this, &MediaPipelineTestMethods::notifyEvent)); } +void MediaPipelineTestMethods::shouldNotifyFirstFrameReceivedAudio() +{ + EXPECT_CALL(*m_mediaPipelineClientMock, notifyFirstFrameReceived(kAudioSourceId)) + .WillOnce(Invoke(this, &MediaPipelineTestMethods::notifyEvent)); +} + +void MediaPipelineTestMethods::sendNotifyFirstFrameReceivedAudio() +{ + getServerStub()->notifyFirstFrameReceivedEvent(kSessionId, kAudioSourceId); + waitEvent(); +} + void MediaPipelineTestMethods::sendNotifyFirstFrameReceivedVideo() { getServerStub()->notifyFirstFrameReceivedEvent(kSessionId, kVideoSourceId); diff --git a/tests/componenttests/client/tests/base/MediaPipelineTestMethods.h b/tests/componenttests/client/tests/base/MediaPipelineTestMethods.h index f73f23f9d..838f987e6 100644 --- a/tests/componenttests/client/tests/base/MediaPipelineTestMethods.h +++ b/tests/componenttests/client/tests/base/MediaPipelineTestMethods.h @@ -173,6 +173,7 @@ class MediaPipelineTestMethods void shouldNotifyQosVideo(); void shouldNotifyBufferUnderflowAudio(); void shouldNotifyBufferUnderflowVideo(); + void shouldNotifyFirstFrameReceivedAudio(); void shouldNotifyFirstFrameReceivedVideo(); void shouldNotifyPlaybackErrorAudio(); void shouldNotifyPlaybackErrorVideo(); @@ -302,6 +303,7 @@ class MediaPipelineTestMethods void sendNotifyQosVideo(); void sendNotifyBufferUnderflowAudio(); void sendNotifyBufferUnderflowVideo(); + void sendNotifyFirstFrameReceivedAudio(); void sendNotifyFirstFrameReceivedVideo(); void sendNotifyPlaybackErrorAudio(); void sendNotifyPlaybackErrorVideo(); diff --git a/tests/componenttests/client/tests/mse/FirstFrameNotificationTest.cpp b/tests/componenttests/client/tests/mse/FirstFrameNotificationTest.cpp index 319948300..15d6bd1d1 100644 --- a/tests/componenttests/client/tests/mse/FirstFrameNotificationTest.cpp +++ b/tests/componenttests/client/tests/mse/FirstFrameNotificationTest.cpp @@ -46,7 +46,7 @@ class FirstFrameNotificationTest : public ClientComponentTest /* * Component Test: First frame notification * Test Objective: - * Test the first frame notification for video source. + * Test the first frame notification for video and audio sources. * * Sequence Diagrams: * First frame notification @@ -67,6 +67,10 @@ class FirstFrameNotificationTest : public ClientComponentTest * Server notifies the client first frame received with source id video. * Expect that the first frame notification is propagated to the client. * + * Step 2: Notify first frame received for audio + * Server notifies the client first frame received with source id audio. + * Expect that the first frame notification is propagated to the client. + * * Test Teardown: * Terminate the media session. * Memory region created for the shared buffer is closed. @@ -82,5 +86,9 @@ TEST_F(FirstFrameNotificationTest, notification) // Step 1: Notify first frame received for video MediaPipelineTestMethods::shouldNotifyFirstFrameReceivedVideo(); MediaPipelineTestMethods::sendNotifyFirstFrameReceivedVideo(); + + // Step 2: Notify first frame received for audio + MediaPipelineTestMethods::shouldNotifyFirstFrameReceivedAudio(); + MediaPipelineTestMethods::sendNotifyFirstFrameReceivedAudio(); } } // namespace firebolt::rialto::client::ct diff --git a/tests/componenttests/server/tests/mediaPipeline/FirstFrameNotificationTest.cpp b/tests/componenttests/server/tests/mediaPipeline/FirstFrameNotificationTest.cpp index 82daedd6c..70306ef08 100644 --- a/tests/componenttests/server/tests/mediaPipeline/FirstFrameNotificationTest.cpp +++ b/tests/componenttests/server/tests/mediaPipeline/FirstFrameNotificationTest.cpp @@ -43,16 +43,18 @@ class FirstFrameNotificationTest : public MediaPipelineTest { m_elementFactory = gst_element_factory_find("fakesrc"); m_videoDecoder = gst_element_factory_create(m_elementFactory, nullptr); + m_audioDecoder = gst_element_factory_create(m_elementFactory, nullptr); EXPECT_CALL(*m_gstWrapperMock, gstElementGetFactory(_)).WillRepeatedly(Return(m_elementFactory)); } ~FirstFrameNotificationTest() override { + gst_object_unref(m_audioDecoder); gst_object_unref(m_videoDecoder); gst_object_unref(m_elementFactory); } - void setupElementsCommon() + void setupElementsCommon(const char *signalName) { EXPECT_CALL(*m_glibWrapperMock, gTypeName(_)).WillRepeatedly(Return(kElementName.c_str())); EXPECT_CALL(*m_glibWrapperMock, gStrHasPrefix(_, StrEq("amlhalasink"))).WillRepeatedly(Return(FALSE)); @@ -68,7 +70,7 @@ class FirstFrameNotificationTest : public MediaPipelineTest })); EXPECT_CALL(*m_glibWrapperMock, gSignalQuery(m_signals[0], _)) .WillRepeatedly(Invoke([&](guint signal_id, GSignalQuery *query) - { query->signal_name = "first-video-frame-callback"; })); + { query->signal_name = signalName; })); EXPECT_CALL(*m_glibWrapperMock, gFree(m_signals)).Times(2); } @@ -138,12 +140,72 @@ class FirstFrameNotificationTest : public MediaPipelineTest EXPECT_EQ(receivedFirstFrameReceived->source_id(), m_videoSourceId); } + void willSetupAudioDecoder() + { + EXPECT_CALL(*m_gstWrapperMock, gstObjectRef(m_audioDecoder)).WillOnce(Return(m_audioDecoder)); + + EXPECT_CALL(*m_gstWrapperMock, gstElementFactoryListIsType(m_elementFactory, GST_ELEMENT_FACTORY_TYPE_DECODER)) + .WillOnce(Return(TRUE)); + EXPECT_CALL(*m_gstWrapperMock, + gstElementFactoryListIsType(m_elementFactory, GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO)) + .WillOnce(Return(FALSE)); + EXPECT_CALL(*m_gstWrapperMock, + gstElementFactoryListIsType(m_elementFactory, + GST_ELEMENT_FACTORY_TYPE_DECODER | + GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO)) + .WillOnce(Return(TRUE)) + .RetiresOnSaturation(); + EXPECT_CALL(*m_gstWrapperMock, + gstElementFactoryListIsType(m_elementFactory, + GST_ELEMENT_FACTORY_TYPE_SINK | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO)) + .WillOnce(Return(FALSE)) + .RetiresOnSaturation(); + + EXPECT_CALL(*m_glibWrapperMock, gObjectType(m_audioDecoder)).WillRepeatedly(Return(G_TYPE_PARAM)); + EXPECT_CALL(*m_glibWrapperMock, gSignalConnect(_, StrEq("first-audio-frame"), _, _)) + .WillOnce(Invoke( + [&](gpointer instance, const gchar *detailed_signal, GCallback c_handler, gpointer data) + { + m_firstFrameCallback = c_handler; + m_firstFrameData = data; + return kSignalId; + })) + .RetiresOnSaturation(); + EXPECT_CALL(*m_gstWrapperMock, gstObjectUnref(m_audioDecoder)) + .WillOnce(Invoke(this, &MediaPipelineTest::workerFinished)); + } + + void setupAudioDecoder() + { + m_gstreamerStub.setupElement(m_audioDecoder); + waitWorker(); + } + + void firstAudioFrameReceived() + { + ExpectMessage expectedFirstFrameReceived{m_clientStub}; + expectedFirstFrameReceived.setFilter([&](const auto &msg) { return msg.source_id() == m_audioSourceId; }); + + ASSERT_TRUE(m_firstFrameCallback); + ASSERT_TRUE(m_firstFrameData); + reinterpret_cast( + m_firstFrameCallback)(m_audioDecoder, 0, nullptr, m_firstFrameData); + + auto receivedFirstFrameReceived{expectedFirstFrameReceived.getMessage()}; + ASSERT_TRUE(receivedFirstFrameReceived); + EXPECT_EQ(receivedFirstFrameReceived->session_id(), m_sessionId); + EXPECT_EQ(receivedFirstFrameReceived->source_id(), m_audioSourceId); + } + private: GstElementFactory *m_elementFactory{nullptr}; GstElement *m_videoDecoder{nullptr}; + GstElement *m_audioDecoder{nullptr}; guint m_signals[1]{123}; GCallback m_firstVideoFrameCallback; gpointer m_firstVideoFrameData{nullptr}; + GCallback m_firstFrameCallback; + gpointer m_firstFrameData{nullptr}; }; /* @@ -248,7 +310,7 @@ TEST_F(FirstFrameNotificationTest, firstFrameNotification) load(); // Step 3: Setup Video Decoder - setupElementsCommon(); + setupElementsCommon("first-video-frame-callback"); willSetupVideoDecoder(); setupVideoDecoder(); @@ -301,4 +363,69 @@ TEST_F(FirstFrameNotificationTest, firstFrameNotification) gstPlayerWillBeDestructed(); destroySession(); } + +TEST_F(FirstFrameNotificationTest, firstAudioFrameNotification) +{ + // Step 1: Create a new media session + createSession(); + + // Step 2: Load content + gstPlayerWillBeCreated(); + load(); + + // Step 3: Setup Audio Decoder + setupElementsCommon("first-audio-frame"); + willSetupAudioDecoder(); + setupAudioDecoder(); + + // Step 4: Attach audio source + audioSourceWillBeAttached(); + attachAudioSource(); + sourceWillBeSetup(); + setupSource(); + willSetupAndAddSource(&m_audioAppSrc); + willFinishSetupAndAddSource(); + indicateAllSourcesAttached({&m_audioAppSrc}); + + // Step 5: Pause + willPause(); + pause(); + + // Step 6: Write 1 audio frame + // Step 7: Notify buffered and Paused + { + ExpectMessage expectedNetworkStateChange{m_clientStub}; + + pushAudioData(kFramesToPush); + + auto receivedNetworkStateChange{expectedNetworkStateChange.getMessage()}; + ASSERT_TRUE(receivedNetworkStateChange); + EXPECT_EQ(receivedNetworkStateChange->session_id(), m_sessionId); + EXPECT_EQ(receivedNetworkStateChange->state(), + ::firebolt::rialto::NetworkStateChangeEvent_NetworkState_BUFFERED); + } + willNotifyPaused(); + notifyPaused(); + + // Step 8: First audio frame received + firstAudioFrameReceived(); + + // Step 9: End of audio stream + willEos(&m_audioAppSrc); + eosAudio(kFramesToPush); + + // Step 10: Notify end of stream + gstNotifyEos(); + + // Step 11: Remove source + removeSource(m_audioSourceId); + + // Step 12: Stop + willStop(); + stop(); + + // Step 13: Destroy media session + gstPlayerWillBeDestructed(); + destroySession(); +} } // namespace firebolt::rialto::server::ct diff --git a/tests/unittests/media/server/gstplayer/genericPlayer/GstGenericPlayerPrivateTest.cpp b/tests/unittests/media/server/gstplayer/genericPlayer/GstGenericPlayerPrivateTest.cpp index 1ff65a11e..4b47ddda2 100644 --- a/tests/unittests/media/server/gstplayer/genericPlayer/GstGenericPlayerPrivateTest.cpp +++ b/tests/unittests/media/server/gstplayer/genericPlayer/GstGenericPlayerPrivateTest.cpp @@ -288,6 +288,16 @@ TEST_F(GstGenericPlayerPrivateTest, shouldScheduleFirstVideoFrameReceived) m_sut->scheduleFirstVideoFrameReceived(); } +TEST_F(GstGenericPlayerPrivateTest, shouldScheduleFirstAudioFrameReceived) +{ + std::unique_ptr task{std::make_unique>()}; + EXPECT_CALL(dynamic_cast &>(*task), execute()); + EXPECT_CALL(m_taskFactoryMock, createFirstFrameReceived(_, _, MediaSourceType::AUDIO)) + .WillOnce(Return(ByMove(std::move(task)))); + + m_sut->scheduleFirstAudioFrameReceived(); +} + TEST_F(GstGenericPlayerPrivateTest, shouldNotSetVideoRectangleWhenVideoSinkIsNull) { EXPECT_CALL(*m_glibWrapperMock, gObjectGetStub(_, StrEq(kVideoSinkStr), _)); diff --git a/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.cpp b/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.cpp index c9d971378..ae84595c4 100644 --- a/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.cpp +++ b/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.cpp @@ -433,6 +433,19 @@ void GenericTasksTestsBase::expectFirstVideoFrameSignalConnection() })); } +void GenericTasksTestsBase::expectFirstAudioFrameSignalConnection(const char *signalName) +{ + EXPECT_CALL(*testContext->m_glibWrapper, gSignalQuery(testContext->m_signals[0], _)) + .WillOnce(Invoke([&](guint signal_id, GSignalQuery *query) { query->signal_name = signalName; })); + EXPECT_CALL(*testContext->m_glibWrapper, gSignalConnect(_, StrEq(signalName), _, _)) + .WillOnce(Invoke( + [&](gpointer instance, const gchar *detailed_signal, GCallback c_handler, gpointer data) + { + testContext->m_firstAudioFrameCallback = c_handler; + return kSignalId; + })); +} + void GenericTasksTestsBase::expectAudioUnderflowSignalConnection() { EXPECT_CALL(*testContext->m_glibWrapper, gObjectType(testContext->m_element)).WillRepeatedly(Return(G_TYPE_PARAM)); @@ -618,6 +631,7 @@ void GenericTasksTestsBase::expectSetupAudioSinkElement() GST_ELEMENT_FACTORY_TYPE_SINK | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO)) .WillOnce(Return(TRUE)); + expectFirstAudioFrameSignalConnection("first-audio-frame-callback"); expectAudioUnderflowSignalConnection(); EXPECT_CALL(*testContext->m_gstWrapper, gstObjectUnref(_)); @@ -649,6 +663,7 @@ void GenericTasksTestsBase::expectSetupAudioDecoderElement() GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO)) .WillOnce(Return(TRUE)); + expectFirstAudioFrameSignalConnection("first-audio-frame"); expectAudioUnderflowSignalConnection(); EXPECT_CALL(*testContext->m_gstWrapper, gstObjectUnref(_)); @@ -761,6 +776,14 @@ void GenericTasksTestsBase::shouldSetupVideoDecoderElementWithFirstVideoFrameCal expectSetupVideoDecoderElementWithFirstVideoFrameCallback(); } +void GenericTasksTestsBase::shouldSetupAudioDecoderElementWithFirstAudioFrameCallback() +{ + EXPECT_CALL(*testContext->m_glibWrapper, gTypeName(G_OBJECT_TYPE(testContext->m_element))) + .WillOnce(Return(kElementTypeName.c_str())); + + expectSetupAudioDecoderElement(); +} + void GenericTasksTestsBase::shouldSetupVideoElementWithPendingGeometry() { testContext->m_context.pendingGeometry = kRectangle; @@ -1075,6 +1098,13 @@ void GenericTasksTestsBase::shouldSetFirstVideoFrameCallback() EXPECT_CALL(testContext->m_gstPlayer, scheduleFirstVideoFrameReceived()); } +void GenericTasksTestsBase::shouldSetFirstAudioFrameCallback() +{ + ASSERT_TRUE(testContext->m_firstAudioFrameCallback); + EXPECT_CALL(testContext->m_gstPlayer, clearAudioFirstFrameFallbackProbe()); + EXPECT_CALL(testContext->m_gstPlayer, scheduleFirstAudioFrameReceived()); +} + void GenericTasksTestsBase::shouldSetupBaseParse() { EXPECT_CALL(*testContext->m_gstWrapper, gstBaseParseSetPtsInterpolation(_, FALSE)); @@ -1093,6 +1123,12 @@ void GenericTasksTestsBase::triggerFirstVideoFrameCallback() testContext->m_firstVideoFrameCallback)(testContext->m_element, 0, nullptr, &testContext->m_gstPlayer); } +void GenericTasksTestsBase::triggerFirstAudioFrameCallback() +{ + reinterpret_cast( + testContext->m_firstAudioFrameCallback)(testContext->m_element, 0, nullptr, &testContext->m_gstPlayer); +} + void GenericTasksTestsBase::shouldSetAudioUnderflowCallback() { ASSERT_TRUE(testContext->m_audioUnderflowCallback); @@ -2408,6 +2444,7 @@ void GenericTasksTestsBase::shouldStopGstPlayer() videoStreamIt->second.isDataNeeded = true; audioStreamIt->second.isDataNeeded = true; + EXPECT_CALL(testContext->m_gstPlayer, clearAudioFirstFrameFallbackProbe()); EXPECT_CALL(testContext->m_gstPlayer, stopPositionReportingAndCheckAudioUnderflowTimer()); EXPECT_CALL(testContext->m_gstPlayer, stopNotifyPlaybackInfoTimer()); EXPECT_CALL(testContext->m_gstPlayer, changePipelineState(GST_STATE_NULL)).WillOnce(Return(GST_STATE_CHANGE_SUCCESS)); diff --git a/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.h b/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.h index 4aca17fa7..16f40e3e8 100644 --- a/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.h +++ b/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsBase.h @@ -79,6 +79,7 @@ class GenericTasksTestsBase : public ::testing::Test void shouldSetupVideoSinkElementOnly(); void shouldSetupVideoDecoderElementOnly(); void shouldSetupVideoDecoderElementWithFirstVideoFrameCallback(); + void shouldSetupAudioDecoderElementWithFirstAudioFrameCallback(); void shouldSetupVideoElementWithPendingGeometry(); void shouldSetupVideoElementWithPendingImmediateOutput(); void shouldSetupAudioSinkElementWithPendingLowLatency(); @@ -101,10 +102,12 @@ class GenericTasksTestsBase : public ::testing::Test void shouldSetupAudioDecoderElementOnly(); void shouldSetVideoUnderflowCallback(); void shouldSetFirstVideoFrameCallback(); + void shouldSetFirstAudioFrameCallback(); void shouldSetupBaseParse(); void triggerSetupElement(); void triggerVideoUnderflowCallback(); void triggerFirstVideoFrameCallback(); + void triggerFirstAudioFrameCallback(); void shouldSetAudioUnderflowCallback(); void triggerAudioUnderflowCallback(); void shouldAddFirstAutoVideoSinkChild(); @@ -436,6 +439,7 @@ class GenericTasksTestsBase : public ::testing::Test // SetupElement helper methods void expectVideoUnderflowSignalConnection(); void expectFirstVideoFrameSignalConnection(); + void expectFirstAudioFrameSignalConnection(const char *signalName); void expectAudioUnderflowSignalConnection(); void expectSetupVideoSinkElement(); void expectSetupVideoDecoderElement(); diff --git a/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsContext.h b/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsContext.h index c39eb5303..0e8158407 100644 --- a/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsContext.h +++ b/tests/unittests/media/server/gstplayer/genericPlayer/common/GenericTasksTestsContext.h @@ -100,6 +100,7 @@ class GenericTasksTestsContext GCallback m_audioUnderflowCallback; GCallback m_videoUnderflowCallback; GCallback m_firstVideoFrameCallback; + GCallback m_firstAudioFrameCallback; GCallback m_childAddedCallback; GCallback m_childRemovedCallback; gchar m_capsStr{}; diff --git a/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/FirstFrameReceivedTest.cpp b/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/FirstFrameReceivedTest.cpp index 1adeffb24..b86b3c6a7 100644 --- a/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/FirstFrameReceivedTest.cpp +++ b/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/FirstFrameReceivedTest.cpp @@ -29,6 +29,7 @@ using testing::StrictMock; namespace { constexpr firebolt::rialto::MediaSourceType kSourceType{firebolt::rialto::MediaSourceType::VIDEO}; +constexpr firebolt::rialto::MediaSourceType kAudioSourceType{firebolt::rialto::MediaSourceType::AUDIO}; } // namespace class FirstFrameReceivedTest : public testing::Test @@ -55,3 +56,13 @@ TEST_F(FirstFrameReceivedTest, shouldNotNotifyFirstFrameReceivedWhenClientIsNull task.execute(); } + +TEST_F(FirstFrameReceivedTest, shouldNotifyFirstAudioFrameReceived) +{ + firebolt::rialto::server::tasks::generic::FirstFrameReceived task{m_context, m_gstPlayer, &m_gstPlayerClient, + kAudioSourceType}; + + EXPECT_CALL(m_gstPlayerClient, notifyFirstFrameReceived(kAudioSourceType)); + + task.execute(); +} diff --git a/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/SetupElementTest.cpp b/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/SetupElementTest.cpp index 9d16f1953..a88a05941 100644 --- a/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/SetupElementTest.cpp +++ b/tests/unittests/media/server/gstplayer/genericPlayer/tasksTests/SetupElementTest.cpp @@ -179,6 +179,15 @@ TEST_F(SetupElementTest, shouldReportFirstVideoFrame) triggerFirstVideoFrameCallback(); } +TEST_F(SetupElementTest, shouldReportFirstAudioFrame) +{ + shouldSetupAudioDecoderElementWithFirstAudioFrameCallback(); + triggerSetupElement(); + + shouldSetFirstAudioFrameCallback(); + triggerFirstAudioFrameCallback(); +} + TEST_F(SetupElementTest, shouldReportAudioUnderflow) { shouldSetupAudioDecoderElementOnly(); diff --git a/tests/unittests/media/server/mocks/gstplayer/GstGenericPlayerPrivateMock.h b/tests/unittests/media/server/mocks/gstplayer/GstGenericPlayerPrivateMock.h index 9fbf62ebc..d5b41596d 100644 --- a/tests/unittests/media/server/mocks/gstplayer/GstGenericPlayerPrivateMock.h +++ b/tests/unittests/media/server/mocks/gstplayer/GstGenericPlayerPrivateMock.h @@ -37,6 +37,10 @@ class GstGenericPlayerPrivateMock : public IGstGenericPlayerPrivate MOCK_METHOD(void, scheduleAudioUnderflow, (), (override)); MOCK_METHOD(void, scheduleVideoUnderflow, (), (override)); MOCK_METHOD(void, scheduleFirstVideoFrameReceived, (), (override)); + MOCK_METHOD(void, scheduleFirstAudioFrameReceived, (), (override)); + MOCK_METHOD(void, setAudioFirstFrameFallbackProbe, (GstPad * pad, gulong id), (override)); + MOCK_METHOD(void, clearAudioFirstFrameFallbackProbe, (), (override)); + MOCK_METHOD(void, clearAudioFirstFrameFallbackProbeState, (), (override)); MOCK_METHOD(void, scheduleAllSourcesAttached, (), (override)); MOCK_METHOD(bool, setVideoSinkRectangle, (), (override)); MOCK_METHOD(bool, setImmediateOutput, (), (override)); From 60dd7df4ed74e0e1541b9352fa8630cc0127e7d4 Mon Sep 17 00:00:00 2001 From: balasaraswathy-n Date: Mon, 1 Jun 2026 22:43:17 +0530 Subject: [PATCH 7/7] Update GstGenericPlayer.cpp --- media/server/gstplayer/source/GstGenericPlayer.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/media/server/gstplayer/source/GstGenericPlayer.cpp b/media/server/gstplayer/source/GstGenericPlayer.cpp index 2acf06555..71b2869ef 100644 --- a/media/server/gstplayer/source/GstGenericPlayer.cpp +++ b/media/server/gstplayer/source/GstGenericPlayer.cpp @@ -504,6 +504,11 @@ GstElement *GstGenericPlayer::getSink(const MediaSourceType &mediaSourceType) co void GstGenericPlayer::setSourceFlushed(const MediaSourceType &mediaSourceType) { + if (mediaSourceType == MediaSourceType::AUDIO) + { + m_context.firstAudioFrameReceived = false; + clearAudioFirstFrameFallbackProbe(); + } m_flushWatcher->setFlushed(mediaSourceType); } @@ -1733,11 +1738,6 @@ void GstGenericPlayer::scheduleAllSourcesAttached() allSourcesAttached(); } - if (mediaSourceType == MediaSourceType::AUDIO) - { - m_context.firstAudioFrameReceived = false; - clearAudioFirstFrameFallbackProbe(); - } void GstGenericPlayer::cancelUnderflow(firebolt::rialto::MediaSourceType mediaSource) {