From a47692450ceb24d69ae3f425f58170f4def80bd3 Mon Sep 17 00:00:00 2001 From: Atneya Nair Date: Wed, 10 May 2023 17:26:30 -0700 Subject: [PATCH 01/20] Force unsilence record clients on startInput We call startRecording unconditionally in startInput, so we must update the client state to be unsilenced (since we are treating as such). We subsequently re-update the silence state (with the client marked as active to dispatch ops) in updateUidStates_l. This fixes an issue where we call startRecording for a silenced client, then call it again when it moves to unsilenced when the client is active. Since startRecording is ref-counted, this leaves the client in the recording state leading to incorrect appop attributions. Bug: 279905816 Bug: 281485019 Test: Manual verification of repro cases + verbose log analysis (cherry picked from https://googleplex-android-review.googlesource.com/q/commit:e7720b379bfaba648ab6d85c4c2df6f03ec854d3) (cherry picked from https://googleplex-android-review.googlesource.com/q/commit:2951ad10a6641f9b3554d674877ad314e8cc011f) Merged-In: I31d50457ca8adae577407a28d4d4c0e8582bac5d Change-Id: I31d50457ca8adae577407a28d4d4c0e8582bac5d --- .../service/AudioPolicyInterfaceImpl.cpp | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp index 04aeabaa01..140cabb808 100644 --- a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp +++ b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp @@ -793,8 +793,29 @@ Status AudioPolicyService::startInput(int32_t portIdAidl) Mutex::Autolock _l(mLock); + ALOGW_IF(client->silenced, "startInput on silenced input for port %d, uid %d. Unsilencing.", + portIdAidl, + client->attributionSource.uid); + + if (client->active) { + ALOGE("Client should never be active before startInput. Uid %d port %d", + client->attributionSource.uid, portId); + finishRecording(client->attributionSource, client->attributes.source); + return binderStatusFromStatusT(INVALID_OPERATION); + } + + // Force the possibly silenced client to be unsilenced since we just called + // startRecording (i.e. we have assumed it is unsilenced). + // At this point in time, the client is inactive, so no calls to appops are sent in + // setAppState_l. + // This ensures existing clients have the same behavior as new clients (starting unsilenced). + // TODO(b/282076713) + setAppState_l(client, APP_STATE_TOP); + client->active = true; client->startTimeNs = systemTime(); + // This call updates the silenced state, and since we are active, appropriately notifies appops + // if we silence the track. updateUidStates_l(); status_t status; From 6d96fc6472755c9a548db4bb39d7d569d47e9f8d Mon Sep 17 00:00:00 2001 From: jhenrique09 Date: Tue, 24 Mar 2020 16:36:07 -0300 Subject: [PATCH 02/20] [2/2] av: Remove restrictions for system audio record * Give freedom to screen recorder apps Change-Id: I726bde4f44bba6fc8cd771ae90c8864b26cdd919 Signed-off-by: Edwiin Kusuma Jaya --- .../libaaudio/src/utility/AAudioUtilities.cpp | 4 +-- .../managerdefinitions/src/AudioPolicyMix.cpp | 25 +------------------ 2 files changed, 3 insertions(+), 26 deletions(-) diff --git a/media/libaaudio/src/utility/AAudioUtilities.cpp b/media/libaaudio/src/utility/AAudioUtilities.cpp index 872faca58b..7075150ae6 100644 --- a/media/libaaudio/src/utility/AAudioUtilities.cpp +++ b/media/libaaudio/src/utility/AAudioUtilities.cpp @@ -239,7 +239,7 @@ audio_flags_mask_t AAudioConvert_allowCapturePolicyToAudioFlagsMask( aaudio_spatialization_behavior_t spatializationBehavior, bool isContentSpatialized) { audio_flags_mask_t flagsMask = AUDIO_FLAG_NONE; - switch (policy) { + /*switch (policy) { case AAUDIO_UNSPECIFIED: case AAUDIO_ALLOW_CAPTURE_BY_ALL: // flagsMask is not modified @@ -254,7 +254,7 @@ audio_flags_mask_t AAudioConvert_allowCapturePolicyToAudioFlagsMask( default: ALOGE("%s() 0x%08X unrecognized capture policy", __func__, policy); // flagsMask is not modified - } + }*/ switch (spatializationBehavior) { case AAUDIO_UNSPECIFIED: diff --git a/services/audiopolicy/common/managerdefinitions/src/AudioPolicyMix.cpp b/services/audiopolicy/common/managerdefinitions/src/AudioPolicyMix.cpp index e142bef750..0797769c04 100644 --- a/services/audiopolicy/common/managerdefinitions/src/AudioPolicyMix.cpp +++ b/services/audiopolicy/common/managerdefinitions/src/AudioPolicyMix.cpp @@ -206,30 +206,7 @@ AudioPolicyMixCollection::MixMatchStatus AudioPolicyMixCollection::mixMatch( const audio_config_base_t& config, uid_t uid) { if (mix->mMixType == MIX_TYPE_PLAYERS) { - // Loopback render mixes are created from a public API and thus restricted - // to non sensible audio that have not opted out. - if (is_mix_loopback_render(mix->mRouteFlags)) { - if (!(attributes.usage == AUDIO_USAGE_UNKNOWN || - attributes.usage == AUDIO_USAGE_MEDIA || - attributes.usage == AUDIO_USAGE_GAME || - attributes.usage == AUDIO_USAGE_VOICE_COMMUNICATION)) { - return MixMatchStatus::NO_MATCH; - } - auto hasFlag = [](auto flags, auto flag) { return (flags & flag) == flag; }; - if (hasFlag(attributes.flags, AUDIO_FLAG_NO_SYSTEM_CAPTURE)) { - return MixMatchStatus::NO_MATCH; - } - - if (attributes.usage == AUDIO_USAGE_VOICE_COMMUNICATION) { - if (!mix->mVoiceCommunicationCaptureAllowed) { - return MixMatchStatus::NO_MATCH; - } - } else if (!mix->mAllowPrivilegedMediaPlaybackCapture && - hasFlag(attributes.flags, AUDIO_FLAG_NO_MEDIA_PROJECTION)) { - return MixMatchStatus::NO_MATCH; - } - } - + // Permit match only if requested format and mix format are PCM and can be format // adapted by the mixer, or are the same (compressed) format. if (!is_mix_loopback(mix->mRouteFlags) && From 75ad0f99283db1f4834228473162f53ad09c0d5f Mon Sep 17 00:00:00 2001 From: Mingming Yin Date: Thu, 17 Jul 2014 18:21:21 -0700 Subject: [PATCH 03/20] audiopolicy: Disable sonification on WFD sink * Aligned with CAF as of QSSI12, and removed unnecessary devices.isEmpty() check. Original patch: https://source.codeaurora.org/quic/la/platform/frameworks/av/commit/services/audiopolicy?h=LA.BR64.1.1.1-01010-8x16.0&id=9cc970fc512bedfa6a2cf03457d93609adc6eb85 Change-Id: I9bad6a294ddd7aee72f6f6a314666b892b730c8e Signed-off-by: Edwiin Kusuma Jaya --- services/audiopolicy/enginedefault/src/Engine.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/audiopolicy/enginedefault/src/Engine.cpp b/services/audiopolicy/enginedefault/src/Engine.cpp index 52b9826c2f..d0087b2d8c 100644 --- a/services/audiopolicy/enginedefault/src/Engine.cpp +++ b/services/audiopolicy/enginedefault/src/Engine.cpp @@ -401,6 +401,10 @@ DeviceVector Engine::getDevicesForStrategyInt(legacy_strategy strategy, devices2 = availableOutputDevices.getDevicesFromType( AUDIO_DEVICE_OUT_ANLG_DOCK_HEADSET); } + if ((devices2.isEmpty()) && (strategy != STRATEGY_SONIFICATION)) { + // no sonification on WFD sink + devices2 = availableOutputDevices.getDevicesFromType(AUDIO_DEVICE_OUT_PROXY); + } if (devices2.isEmpty()) { devices2 = availableOutputDevices.getFirstDevicesFromTypes({ AUDIO_DEVICE_OUT_DGTL_DOCK_HEADSET, AUDIO_DEVICE_OUT_SPEAKER}); From dd9e10547bc135641f37546ca679a5bd7a550f8b Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Wed, 9 Jan 2019 01:13:10 +0200 Subject: [PATCH 04/20] Revert "stagefright: remove Miracast sender code" This reverts commit d0a98fa05f0f6719b93d000c4638230af06e0b99. Change-Id: I0554b92c290c1ebbd1a40fc2edb43573a97d4f6a Signed-off-by: DennySPb Signed-off-by: Kujou Yuko Signed-off-by: Edwiin Kusuma Jaya --- include/media/IHDCP.h | 1 + media/libmedia/Android.bp | 1 + media/libmedia/IHDCP.cpp | 359 ++++ media/libmedia/IMediaPlayerService.cpp | 17 + media/libmedia/include/media/IHDCP.h | 120 ++ .../include/media/IMediaPlayerService.h | 2 + media/libmediaplayerservice/Android.bp | 7 + media/libmediaplayerservice/HDCP.cpp | 175 ++ media/libmediaplayerservice/HDCP.h | 66 + .../MediaPlayerService.cpp | 17 +- .../MediaPlayerService.h | 1 + media/libmediaplayerservice/RemoteDisplay.cpp | 66 + media/libmediaplayerservice/RemoteDisplay.h | 59 + media/libstagefright/wifi-display/Android.bp | 51 + .../wifi-display/MediaSender.cpp | 519 +++++ .../libstagefright/wifi-display/MediaSender.h | 132 ++ .../wifi-display/Parameters.cpp | 92 + .../libstagefright/wifi-display/Parameters.h | 41 + .../wifi-display/VideoFormats.cpp | 550 ++++++ .../wifi-display/VideoFormats.h | 125 ++ .../libstagefright/wifi-display/rtp/RTPBase.h | 49 + .../wifi-display/rtp/RTPSender.cpp | 808 ++++++++ .../wifi-display/rtp/RTPSender.h | 119 ++ .../wifi-display/source/Converter.cpp | 821 ++++++++ .../wifi-display/source/Converter.h | 157 ++ .../wifi-display/source/MediaPuller.cpp | 224 +++ .../wifi-display/source/MediaPuller.h | 68 + .../wifi-display/source/PlaybackSession.cpp | 1112 +++++++++++ .../wifi-display/source/PlaybackSession.h | 176 ++ .../wifi-display/source/RepeaterSource.cpp | 219 +++ .../wifi-display/source/RepeaterSource.h | 67 + .../wifi-display/source/TSPacketizer.cpp | 1055 ++++++++++ .../wifi-display/source/TSPacketizer.h | 94 + .../wifi-display/source/WifiDisplaySource.cpp | 1737 +++++++++++++++++ .../wifi-display/source/WifiDisplaySource.h | 278 +++ 35 files changed, 9380 insertions(+), 5 deletions(-) create mode 120000 include/media/IHDCP.h create mode 100644 media/libmedia/IHDCP.cpp create mode 100644 media/libmedia/include/media/IHDCP.h create mode 100644 media/libmediaplayerservice/HDCP.cpp create mode 100644 media/libmediaplayerservice/HDCP.h create mode 100644 media/libmediaplayerservice/RemoteDisplay.cpp create mode 100644 media/libmediaplayerservice/RemoteDisplay.h create mode 100644 media/libstagefright/wifi-display/Android.bp create mode 100644 media/libstagefright/wifi-display/MediaSender.cpp create mode 100644 media/libstagefright/wifi-display/MediaSender.h create mode 100644 media/libstagefright/wifi-display/Parameters.cpp create mode 100644 media/libstagefright/wifi-display/Parameters.h create mode 100644 media/libstagefright/wifi-display/VideoFormats.cpp create mode 100644 media/libstagefright/wifi-display/VideoFormats.h create mode 100644 media/libstagefright/wifi-display/rtp/RTPBase.h create mode 100644 media/libstagefright/wifi-display/rtp/RTPSender.cpp create mode 100644 media/libstagefright/wifi-display/rtp/RTPSender.h create mode 100644 media/libstagefright/wifi-display/source/Converter.cpp create mode 100644 media/libstagefright/wifi-display/source/Converter.h create mode 100644 media/libstagefright/wifi-display/source/MediaPuller.cpp create mode 100644 media/libstagefright/wifi-display/source/MediaPuller.h create mode 100644 media/libstagefright/wifi-display/source/PlaybackSession.cpp create mode 100644 media/libstagefright/wifi-display/source/PlaybackSession.h create mode 100644 media/libstagefright/wifi-display/source/RepeaterSource.cpp create mode 100644 media/libstagefright/wifi-display/source/RepeaterSource.h create mode 100644 media/libstagefright/wifi-display/source/TSPacketizer.cpp create mode 100644 media/libstagefright/wifi-display/source/TSPacketizer.h create mode 100644 media/libstagefright/wifi-display/source/WifiDisplaySource.cpp create mode 100644 media/libstagefright/wifi-display/source/WifiDisplaySource.h diff --git a/include/media/IHDCP.h b/include/media/IHDCP.h new file mode 120000 index 0000000000..9d4568eafd --- /dev/null +++ b/include/media/IHDCP.h @@ -0,0 +1 @@ +../../media/libmedia/include/media/IHDCP.h \ No newline at end of file diff --git a/media/libmedia/Android.bp b/media/libmedia/Android.bp index 2dd5784c3f..11b6d56aa4 100644 --- a/media/libmedia/Android.bp +++ b/media/libmedia/Android.bp @@ -307,6 +307,7 @@ cc_library { srcs: [ ":mediaextractorservice_aidl", "IDataSource.cpp", + "IHDCP.cpp", "BufferingSettings.cpp", "mediaplayer.cpp", "IMediaHTTPConnection.cpp", diff --git a/media/libmedia/IHDCP.cpp b/media/libmedia/IHDCP.cpp new file mode 100644 index 0000000000..a46017ff9d --- /dev/null +++ b/media/libmedia/IHDCP.cpp @@ -0,0 +1,359 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "IHDCP" +#include + +#include +#include +#include +#include + +namespace android { + +enum { + OBSERVER_NOTIFY = IBinder::FIRST_CALL_TRANSACTION, + HDCP_SET_OBSERVER, + HDCP_INIT_ASYNC, + HDCP_SHUTDOWN_ASYNC, + HDCP_GET_CAPS, + HDCP_ENCRYPT, + HDCP_ENCRYPT_NATIVE, + HDCP_DECRYPT, +}; + +struct BpHDCPObserver : public BpInterface { + explicit BpHDCPObserver(const sp &impl) + : BpInterface(impl) { + } + + virtual void notify( + int msg, int ext1, int ext2, const Parcel *obj) { + Parcel data, reply; + data.writeInterfaceToken(IHDCPObserver::getInterfaceDescriptor()); + data.writeInt32(msg); + data.writeInt32(ext1); + data.writeInt32(ext2); + if (obj && obj->dataSize() > 0) { + data.appendFrom(const_cast(obj), 0, obj->dataSize()); + } + remote()->transact(OBSERVER_NOTIFY, data, &reply, IBinder::FLAG_ONEWAY); + } +}; + +IMPLEMENT_META_INTERFACE(HDCPObserver, "android.hardware.IHDCPObserver"); + +struct BpHDCP : public BpInterface { + explicit BpHDCP(const sp &impl) + : BpInterface(impl) { + } + + virtual status_t setObserver(const sp &observer) { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + data.writeStrongBinder(IInterface::asBinder(observer)); + remote()->transact(HDCP_SET_OBSERVER, data, &reply); + return reply.readInt32(); + } + + virtual status_t initAsync(const char *host, unsigned port) { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + data.writeCString(host); + data.writeInt32(port); + remote()->transact(HDCP_INIT_ASYNC, data, &reply); + return reply.readInt32(); + } + + virtual status_t shutdownAsync() { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + remote()->transact(HDCP_SHUTDOWN_ASYNC, data, &reply); + return reply.readInt32(); + } + + virtual uint32_t getCaps() { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + remote()->transact(HDCP_GET_CAPS, data, &reply); + return reply.readInt32(); + } + + virtual status_t encrypt( + const void *inData, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + data.writeInt32(size); + data.write(inData, size); + data.writeInt32(streamCTR); + remote()->transact(HDCP_ENCRYPT, data, &reply); + + status_t err = reply.readInt32(); + + if (err != OK) { + *outInputCTR = 0; + + return err; + } + + *outInputCTR = reply.readInt64(); + reply.read(outData, size); + + return err; + } + + virtual status_t encryptNative( + const sp &graphicBuffer, + size_t offset, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + data.write(*graphicBuffer); + data.writeInt32(offset); + data.writeInt32(size); + data.writeInt32(streamCTR); + remote()->transact(HDCP_ENCRYPT_NATIVE, data, &reply); + + status_t err = reply.readInt32(); + + if (err != OK) { + *outInputCTR = 0; + return err; + } + + *outInputCTR = reply.readInt64(); + reply.read(outData, size); + + return err; + } + + virtual status_t decrypt( + const void *inData, size_t size, + uint32_t streamCTR, uint64_t inputCTR, + void *outData) { + Parcel data, reply; + data.writeInterfaceToken(IHDCP::getInterfaceDescriptor()); + data.writeInt32(size); + data.write(inData, size); + data.writeInt32(streamCTR); + data.writeInt64(inputCTR); + remote()->transact(HDCP_DECRYPT, data, &reply); + + status_t err = reply.readInt32(); + + if (err != OK) { + return err; + } + + reply.read(outData, size); + + return err; + } +}; + +IMPLEMENT_META_INTERFACE(HDCP, "android.hardware.IHDCP"); + +status_t BnHDCPObserver::onTransact( + uint32_t code, const Parcel &data, Parcel *reply, uint32_t flags) { + switch (code) { + case OBSERVER_NOTIFY: + { + CHECK_INTERFACE(IHDCPObserver, data, reply); + + int msg = data.readInt32(); + int ext1 = data.readInt32(); + int ext2 = data.readInt32(); + + Parcel obj; + if (data.dataAvail() > 0) { + obj.appendFrom( + const_cast(&data), + data.dataPosition(), + data.dataAvail()); + } + + notify(msg, ext1, ext2, &obj); + + return OK; + } + + default: + return BBinder::onTransact(code, data, reply, flags); + } +} + +status_t BnHDCP::onTransact( + uint32_t code, const Parcel &data, Parcel *reply, uint32_t flags) { + switch (code) { + case HDCP_SET_OBSERVER: + { + CHECK_INTERFACE(IHDCP, data, reply); + + sp observer = + interface_cast(data.readStrongBinder()); + + reply->writeInt32(setObserver(observer)); + return OK; + } + + case HDCP_INIT_ASYNC: + { + CHECK_INTERFACE(IHDCP, data, reply); + + const char *host = data.readCString(); + unsigned port = data.readInt32(); + + reply->writeInt32(initAsync(host, port)); + return OK; + } + + case HDCP_SHUTDOWN_ASYNC: + { + CHECK_INTERFACE(IHDCP, data, reply); + + reply->writeInt32(shutdownAsync()); + return OK; + } + + case HDCP_GET_CAPS: + { + CHECK_INTERFACE(IHDCP, data, reply); + + reply->writeInt32(getCaps()); + return OK; + } + + case HDCP_ENCRYPT: + { + CHECK_INTERFACE(IHDCP, data, reply); + + size_t size = data.readInt32(); + void *inData = NULL; + // watch out for overflow + if (size <= SIZE_MAX / 2) { + inData = malloc(2 * size); + } + if (inData == NULL) { + reply->writeInt32(ERROR_OUT_OF_RANGE); + return OK; + } + + void *outData = (uint8_t *)inData + size; + + status_t err = data.read(inData, size); + if (err != OK) { + free(inData); + reply->writeInt32(err); + return OK; + } + + uint32_t streamCTR = data.readInt32(); + uint64_t inputCTR; + err = encrypt(inData, size, streamCTR, &inputCTR, outData); + + reply->writeInt32(err); + + if (err == OK) { + reply->writeInt64(inputCTR); + reply->write(outData, size); + } + + free(inData); + inData = outData = NULL; + + return OK; + } + + case HDCP_ENCRYPT_NATIVE: + { + CHECK_INTERFACE(IHDCP, data, reply); + + sp graphicBuffer = new GraphicBuffer(); + data.read(*graphicBuffer); + size_t offset = data.readInt32(); + size_t size = data.readInt32(); + uint32_t streamCTR = data.readInt32(); + void *outData = NULL; + uint64_t inputCTR; + + status_t err = ERROR_OUT_OF_RANGE; + + outData = malloc(size); + + if (outData != NULL) { + err = encryptNative(graphicBuffer, offset, size, + streamCTR, &inputCTR, outData); + } + + reply->writeInt32(err); + + if (err == OK) { + reply->writeInt64(inputCTR); + reply->write(outData, size); + } + + free(outData); + outData = NULL; + + return OK; + } + + case HDCP_DECRYPT: + { + CHECK_INTERFACE(IHDCP, data, reply); + + size_t size = data.readInt32(); + size_t bufSize = 2 * size; + + // watch out for overflow + void *inData = NULL; + if (bufSize > size) { + inData = malloc(bufSize); + } + + if (inData == NULL) { + reply->writeInt32(ERROR_OUT_OF_RANGE); + return OK; + } + + void *outData = (uint8_t *)inData + size; + + data.read(inData, size); + + uint32_t streamCTR = data.readInt32(); + uint64_t inputCTR = data.readInt64(); + status_t err = decrypt(inData, size, streamCTR, inputCTR, outData); + + reply->writeInt32(err); + + if (err == OK) { + reply->write(outData, size); + } + + free(inData); + inData = outData = NULL; + + return OK; + } + + default: + return BBinder::onTransact(code, data, reply, flags); + } +} + +} // namespace android diff --git a/media/libmedia/IMediaPlayerService.cpp b/media/libmedia/IMediaPlayerService.cpp index 07c0ac5979..f8326b607a 100644 --- a/media/libmedia/IMediaPlayerService.cpp +++ b/media/libmedia/IMediaPlayerService.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -41,6 +42,7 @@ enum { CREATE = IBinder::FIRST_CALL_TRANSACTION, CREATE_MEDIA_RECORDER, CREATE_METADATA_RETRIEVER, + MAKE_HDCP, ADD_BATTERY_DATA, PULL_BATTERY_DATA, LISTEN_FOR_REMOTE_DISPLAY, @@ -85,6 +87,14 @@ class BpMediaPlayerService: public BpInterface return interface_cast(reply.readStrongBinder()); } + virtual sp makeHDCP(bool createEncryptionModule) { + Parcel data, reply; + data.writeInterfaceToken(IMediaPlayerService::getInterfaceDescriptor()); + data.writeInt32(createEncryptionModule); + remote()->transact(MAKE_HDCP, data, &reply); + return interface_cast(reply.readStrongBinder()); + } + virtual void addBatteryData(uint32_t params) { Parcel data, reply; data.writeInterfaceToken(IMediaPlayerService::getInterfaceDescriptor()); @@ -157,6 +167,13 @@ status_t BnMediaPlayerService::onTransact( reply->writeStrongBinder(IInterface::asBinder(retriever)); return NO_ERROR; } break; + case MAKE_HDCP: { + CHECK_INTERFACE(IMediaPlayerService, data, reply); + bool createEncryptionModule = data.readInt32(); + sp hdcp = makeHDCP(createEncryptionModule); + reply->writeStrongBinder(IInterface::asBinder(hdcp)); + return NO_ERROR; + } break; case ADD_BATTERY_DATA: { CHECK_INTERFACE(IMediaPlayerService, data, reply); uint32_t params = data.readInt32(); diff --git a/media/libmedia/include/media/IHDCP.h b/media/libmedia/include/media/IHDCP.h new file mode 100644 index 0000000000..352561ecbb --- /dev/null +++ b/media/libmedia/include/media/IHDCP.h @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +namespace android { + +struct IHDCPObserver : public IInterface { + DECLARE_META_INTERFACE(HDCPObserver); + + virtual void notify( + int msg, int ext1, int ext2, const Parcel *obj) = 0; + +private: + DISALLOW_EVIL_CONSTRUCTORS(IHDCPObserver); +}; + +struct IHDCP : public IInterface { + DECLARE_META_INTERFACE(HDCP); + + // Called to specify the observer that receives asynchronous notifications + // from the HDCP implementation to signal completion/failure of asynchronous + // operations (such as initialization) or out of band events. + virtual status_t setObserver(const sp &observer) = 0; + + // Request to setup an HDCP session with the specified host listening + // on the specified port. + virtual status_t initAsync(const char *host, unsigned port) = 0; + + // Request to shutdown the active HDCP session. + virtual status_t shutdownAsync() = 0; + + // Returns the capability bitmask of this HDCP session. + // Possible return values (please refer to HDCAPAPI.h): + // HDCP_CAPS_ENCRYPT: mandatory, meaning the HDCP module can encrypt + // from an input byte-array buffer to an output byte-array buffer + // HDCP_CAPS_ENCRYPT_NATIVE: the HDCP module supports encryption from + // a native buffer to an output byte-array buffer. The format of the + // input native buffer is specific to vendor's encoder implementation. + // It is the same format as that used by the encoder when + // "storeMetaDataInBuffers" extension is enabled on its output port. + virtual uint32_t getCaps() = 0; + + // ENCRYPTION only: + // Encrypt data according to the HDCP spec. "size" bytes of data are + // available at "inData" (virtual address), "size" may not be a multiple + // of 128 bits (16 bytes). An equal number of encrypted bytes should be + // written to the buffer at "outData" (virtual address). + // This operation is to be synchronous, i.e. this call does not return + // until outData contains size bytes of encrypted data. + // streamCTR will be assigned by the caller (to 0 for the first PES stream, + // 1 for the second and so on) + // inputCTR _will_be_maintained_by_the_callee_ for each PES stream. + virtual status_t encrypt( + const void *inData, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) = 0; + + // Encrypt data according to the HDCP spec. "size" bytes of data starting + // at location "offset" are available in "buffer" (buffer handle). "size" + // may not be a multiple of 128 bits (16 bytes). An equal number of + // encrypted bytes should be written to the buffer at "outData" (virtual + // address). This operation is to be synchronous, i.e. this call does not + // return until outData contains size bytes of encrypted data. + // streamCTR will be assigned by the caller (to 0 for the first PES stream, + // 1 for the second and so on) + // inputCTR _will_be_maintained_by_the_callee_ for each PES stream. + virtual status_t encryptNative( + const sp &graphicBuffer, + size_t offset, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) = 0; + + // DECRYPTION only: + // Decrypt data according to the HDCP spec. + // "size" bytes of encrypted data are available at "inData" + // (virtual address), "size" may not be a multiple of 128 bits (16 bytes). + // An equal number of decrypted bytes should be written to the buffer + // at "outData" (virtual address). + // This operation is to be synchronous, i.e. this call does not return + // until outData contains size bytes of decrypted data. + // Both streamCTR and inputCTR will be provided by the caller. + virtual status_t decrypt( + const void *inData, size_t size, + uint32_t streamCTR, uint64_t inputCTR, + void *outData) = 0; + +private: + DISALLOW_EVIL_CONSTRUCTORS(IHDCP); +}; + +struct BnHDCPObserver : public BnInterface { + virtual status_t onTransact( + uint32_t code, const Parcel &data, Parcel *reply, + uint32_t flags = 0); +}; + +struct BnHDCP : public BnInterface { + virtual status_t onTransact( + uint32_t code, const Parcel &data, Parcel *reply, + uint32_t flags = 0); +}; + +} // namespace android + + diff --git a/media/libmedia/include/media/IMediaPlayerService.h b/media/libmedia/include/media/IMediaPlayerService.h index 6070673cfa..d35472bc28 100644 --- a/media/libmedia/include/media/IMediaPlayerService.h +++ b/media/libmedia/include/media/IMediaPlayerService.h @@ -34,6 +34,7 @@ namespace android { class IMediaPlayer; +struct IHDCP; class IMediaCodecList; struct IMediaHTTPService; class IMediaRecorder; @@ -54,6 +55,7 @@ class IMediaPlayerService: public IInterface audio_session_t audioSessionId = AUDIO_SESSION_ALLOCATE, const android::content::AttributionSourceState &attributionSource = android::content::AttributionSourceState()) = 0; + virtual sp makeHDCP(bool createEncryptionModule) = 0; virtual sp getCodecList() const = 0; // Connects to a remote display. diff --git a/media/libmediaplayerservice/Android.bp b/media/libmediaplayerservice/Android.bp index 266cb17356..77f1552829 100644 --- a/media/libmediaplayerservice/Android.bp +++ b/media/libmediaplayerservice/Android.bp @@ -22,11 +22,13 @@ cc_library { srcs: [ "ActivityManager.cpp", "DeathNotifier.cpp", + "HDCP.cpp", "MediaPlayerFactory.cpp", "MediaPlayerService.cpp", "MediaRecorderClient.cpp", "MetadataRetrieverClient.cpp", "StagefrightMetadataRetriever.cpp", + "RemoteDisplay.cpp", "StagefrightRecorder.cpp", "TestPlayerStub.cpp", ], @@ -62,6 +64,7 @@ cc_library { "libnetd_client", "libpowermanager", "libstagefright", + "libstagefright_wfd", "libstagefright_foundation", "libstagefright_httplive", "libutils", @@ -92,6 +95,10 @@ cc_library { "libmediautils_headers", ], + include_dirs: [ + "frameworks/av/media/libstagefright/wifi-display", + ], + local_include_dirs: ["include"], export_include_dirs: [ diff --git a/media/libmediaplayerservice/HDCP.cpp b/media/libmediaplayerservice/HDCP.cpp new file mode 100644 index 0000000000..afe39367fb --- /dev/null +++ b/media/libmediaplayerservice/HDCP.cpp @@ -0,0 +1,175 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "HDCP" +#include + +#include "HDCP.h" + +#include + +#include + +namespace android { + +HDCP::HDCP(bool createEncryptionModule) + : mIsEncryptionModule(createEncryptionModule), + mLibHandle(NULL), + mHDCPModule(NULL) { + mLibHandle = dlopen("libstagefright_hdcp.so", RTLD_NOW); + + if (mLibHandle == NULL) { + ALOGE("Unable to locate libstagefright_hdcp.so"); + return; + } + + typedef HDCPModule *(*CreateHDCPModuleFunc)( + void *, HDCPModule::ObserverFunc); + + CreateHDCPModuleFunc createHDCPModule = + mIsEncryptionModule + ? (CreateHDCPModuleFunc)dlsym(mLibHandle, "createHDCPModule") + : (CreateHDCPModuleFunc)dlsym( + mLibHandle, "createHDCPModuleForDecryption"); + + if (createHDCPModule == NULL) { + ALOGE("Unable to find symbol 'createHDCPModule'."); + } else if ((mHDCPModule = createHDCPModule( + this, &HDCP::ObserveWrapper)) == NULL) { + ALOGE("createHDCPModule failed."); + } +} + +HDCP::~HDCP() { + Mutex::Autolock autoLock(mLock); + + if (mHDCPModule != NULL) { + delete mHDCPModule; + mHDCPModule = NULL; + } + + if (mLibHandle != NULL) { + dlclose(mLibHandle); + mLibHandle = NULL; + } +} + +status_t HDCP::setObserver(const sp &observer) { + Mutex::Autolock autoLock(mLock); + + if (mHDCPModule == NULL) { + return NO_INIT; + } + + mObserver = observer; + + return OK; +} + +status_t HDCP::initAsync(const char *host, unsigned port) { + Mutex::Autolock autoLock(mLock); + + if (mHDCPModule == NULL) { + return NO_INIT; + } + + return mHDCPModule->initAsync(host, port); +} + +status_t HDCP::shutdownAsync() { + Mutex::Autolock autoLock(mLock); + + if (mHDCPModule == NULL) { + return NO_INIT; + } + + return mHDCPModule->shutdownAsync(); +} + +uint32_t HDCP::getCaps() { + Mutex::Autolock autoLock(mLock); + + if (mHDCPModule == NULL) { + return NO_INIT; + } + + return mHDCPModule->getCaps(); +} + +status_t HDCP::encrypt( + const void *inData, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) { + Mutex::Autolock autoLock(mLock); + + CHECK(mIsEncryptionModule); + + if (mHDCPModule == NULL) { + *outInputCTR = 0; + + return NO_INIT; + } + + return mHDCPModule->encrypt(inData, size, streamCTR, outInputCTR, outData); +} + +status_t HDCP::encryptNative( + const sp &graphicBuffer, + size_t offset, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData) { + Mutex::Autolock autoLock(mLock); + + CHECK(mIsEncryptionModule); + + if (mHDCPModule == NULL) { + *outInputCTR = 0; + + return NO_INIT; + } + + return mHDCPModule->encryptNative(graphicBuffer->handle, + offset, size, streamCTR, outInputCTR, outData); +} + +status_t HDCP::decrypt( + const void *inData, size_t size, + uint32_t streamCTR, uint64_t outInputCTR, void *outData) { + Mutex::Autolock autoLock(mLock); + + CHECK(!mIsEncryptionModule); + + if (mHDCPModule == NULL) { + return NO_INIT; + } + + return mHDCPModule->decrypt(inData, size, streamCTR, outInputCTR, outData); +} + +// static +void HDCP::ObserveWrapper(void *me, int msg, int ext1, int ext2) { + static_cast(me)->observe(msg, ext1, ext2); +} + +void HDCP::observe(int msg, int ext1, int ext2) { + Mutex::Autolock autoLock(mLock); + + if (mObserver != NULL) { + mObserver->notify(msg, ext1, ext2, NULL /* obj */); + } +} + +} // namespace android + diff --git a/media/libmediaplayerservice/HDCP.h b/media/libmediaplayerservice/HDCP.h new file mode 100644 index 0000000000..83c61b56c0 --- /dev/null +++ b/media/libmediaplayerservice/HDCP.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2012 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HDCP_H_ + +#define HDCP_H_ + +#include +#include + +namespace android { + +struct HDCP : public BnHDCP { + explicit HDCP(bool createEncryptionModule); + virtual ~HDCP(); + + virtual status_t setObserver(const sp &observer); + virtual status_t initAsync(const char *host, unsigned port); + virtual status_t shutdownAsync(); + virtual uint32_t getCaps(); + + virtual status_t encrypt( + const void *inData, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData); + + virtual status_t encryptNative( + const sp &graphicBuffer, + size_t offset, size_t size, uint32_t streamCTR, + uint64_t *outInputCTR, void *outData); + + virtual status_t decrypt( + const void *inData, size_t size, + uint32_t streamCTR, uint64_t outInputCTR, void *outData); + +private: + Mutex mLock; + + bool mIsEncryptionModule; + + void *mLibHandle; + HDCPModule *mHDCPModule; + sp mObserver; + + static void ObserveWrapper(void *me, int msg, int ext1, int ext2); + void observe(int msg, int ext1, int ext2); + + DISALLOW_EVIL_CONSTRUCTORS(HDCP); +}; + +} // namespace android + +#endif // HDCP_H_ + diff --git a/media/libmediaplayerservice/MediaPlayerService.cpp b/media/libmediaplayerservice/MediaPlayerService.cpp index e654bce2e7..df0f1e8a1c 100644 --- a/media/libmediaplayerservice/MediaPlayerService.cpp +++ b/media/libmediaplayerservice/MediaPlayerService.cpp @@ -84,6 +84,8 @@ #include "TestPlayerStub.h" #include +#include "HDCP.h" +#include "RemoteDisplay.h" static const int kDumpLockRetries = 50; static const int kDumpLockSleepUs = 20000; @@ -519,13 +521,18 @@ sp MediaPlayerService::getCodecList() const { return MediaCodecList::getLocalInstance(); } +sp MediaPlayerService::makeHDCP(bool createEncryptionModule) { + return new HDCP(createEncryptionModule); +} + sp MediaPlayerService::listenForRemoteDisplay( - const String16 &/*opPackageName*/, - const sp& /*client*/, - const String8& /*iface*/) { - ALOGE("listenForRemoteDisplay is no longer supported!"); + const String16 &opPackageName, + const sp& client, const String8& iface) { + if (!checkPermission("android.permission.CONTROL_WIFI_DISPLAY")) { + return NULL; + } - return NULL; + return new RemoteDisplay(opPackageName, client, iface.string()); } status_t MediaPlayerService::AudioOutput::dump(int fd, const Vector& args) const diff --git a/media/libmediaplayerservice/MediaPlayerService.h b/media/libmediaplayerservice/MediaPlayerService.h index 86be3fe10c..9fbe6b18bf 100644 --- a/media/libmediaplayerservice/MediaPlayerService.h +++ b/media/libmediaplayerservice/MediaPlayerService.h @@ -248,6 +248,7 @@ class MediaPlayerService : public BnMediaPlayerService const AttributionSourceState& attributionSource); virtual sp getCodecList() const; + virtual sp makeHDCP(bool createEncryptionModule); virtual sp listenForRemoteDisplay(const String16 &opPackageName, const sp& client, const String8& iface); diff --git a/media/libmediaplayerservice/RemoteDisplay.cpp b/media/libmediaplayerservice/RemoteDisplay.cpp new file mode 100644 index 0000000000..0eb4b5dad0 --- /dev/null +++ b/media/libmediaplayerservice/RemoteDisplay.cpp @@ -0,0 +1,66 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "RemoteDisplay.h" + +#include "source/WifiDisplaySource.h" + +#include +#include +#include +#include + +namespace android { + +RemoteDisplay::RemoteDisplay( + const String16 &opPackageName, + const sp &client, + const char *iface) + : mLooper(new ALooper), + mNetSession(new ANetworkSession) { + mLooper->setName("wfd_looper"); + + mSource = new WifiDisplaySource(opPackageName, mNetSession, client); + mLooper->registerHandler(mSource); + + mNetSession->start(); + mLooper->start(); + + mSource->start(iface); +} + +RemoteDisplay::~RemoteDisplay() { +} + +status_t RemoteDisplay::pause() { + return mSource->pause(); +} + +status_t RemoteDisplay::resume() { + return mSource->resume(); +} + +status_t RemoteDisplay::dispose() { + mSource->stop(); + mSource.clear(); + + mLooper->stop(); + mNetSession->stop(); + + return OK; +} + +} // namespace android diff --git a/media/libmediaplayerservice/RemoteDisplay.h b/media/libmediaplayerservice/RemoteDisplay.h new file mode 100644 index 0000000000..d4573e9a39 --- /dev/null +++ b/media/libmediaplayerservice/RemoteDisplay.h @@ -0,0 +1,59 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef REMOTE_DISPLAY_H_ + +#define REMOTE_DISPLAY_H_ + +#include +#include +#include +#include +#include + +namespace android { + +struct ALooper; +struct ANetworkSession; +class IRemoteDisplayClient; +struct WifiDisplaySource; + +struct RemoteDisplay : public BnRemoteDisplay { + RemoteDisplay( + const String16 &opPackageName, + const sp &client, + const char *iface); + + virtual status_t pause(); + virtual status_t resume(); + virtual status_t dispose(); + +protected: + virtual ~RemoteDisplay(); + +private: + sp mNetLooper; + sp mLooper; + sp mNetSession; + sp mSource; + + DISALLOW_EVIL_CONSTRUCTORS(RemoteDisplay); +}; + +} // namespace android + +#endif // REMOTE_DISPLAY_H_ + diff --git a/media/libstagefright/wifi-display/Android.bp b/media/libstagefright/wifi-display/Android.bp new file mode 100644 index 0000000000..fb08c5b072 --- /dev/null +++ b/media/libstagefright/wifi-display/Android.bp @@ -0,0 +1,51 @@ +cc_library_shared { + name: "libstagefright_wfd", + + srcs: [ + "MediaSender.cpp", + "Parameters.cpp", + "rtp/RTPSender.cpp", + "source/Converter.cpp", + "source/MediaPuller.cpp", + "source/PlaybackSession.cpp", + "source/RepeaterSource.cpp", + "source/TSPacketizer.cpp", + "source/WifiDisplaySource.cpp", + "VideoFormats.cpp", + ], + + include_dirs: [ + "frameworks/av/media/libstagefright", + "frameworks/native/include/media/openmax", + "frameworks/native/include/media/hardware", + "frameworks/av/media/libstagefright/mpeg2ts", + ], + + shared_libs: [ + "libbinder", + "libcutils", + "liblog", + "libmedia", + "libstagefright", + "libstagefright_foundation", + "libui", + "libgui", + "libutils", + ], + + cflags: [ + "-Wno-multichar", + "-Werror", + "-Wall", + ], + + sanitize: { + misc_undefined: [ + "signed-integer-overflow", + ], + cfi: true, + diag: { + cfi: true, + }, + }, +} diff --git a/media/libstagefright/wifi-display/MediaSender.cpp b/media/libstagefright/wifi-display/MediaSender.cpp new file mode 100644 index 0000000000..cc412f5f46 --- /dev/null +++ b/media/libstagefright/wifi-display/MediaSender.cpp @@ -0,0 +1,519 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "MediaSender" +#include + +#include "MediaSender.h" + +#include "rtp/RTPSender.h" +#include "source/TSPacketizer.h" + +#include "include/avc_utils.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace android { + +MediaSender::MediaSender( + const sp &netSession, + const sp ¬ify) + : mNetSession(netSession), + mNotify(notify), + mMode(MODE_UNDEFINED), + mGeneration(0), + mPrevTimeUs(-1ll), + mInitDoneCount(0), + mLogFile(NULL) { + // mLogFile = fopen("/data/misc/log.ts", "wb"); +} + +MediaSender::~MediaSender() { + if (mLogFile != NULL) { + fclose(mLogFile); + mLogFile = NULL; + } +} + +status_t MediaSender::setHDCP(const sp &hdcp) { + if (mMode != MODE_UNDEFINED) { + return INVALID_OPERATION; + } + + mHDCP = hdcp; + + return OK; +} + +ssize_t MediaSender::addTrack(const sp &format, uint32_t flags) { + if (mMode != MODE_UNDEFINED) { + return INVALID_OPERATION; + } + + TrackInfo info; + info.mFormat = format; + info.mFlags = flags; + info.mPacketizerTrackIndex = -1; + + AString mime; + CHECK(format->findString("mime", &mime)); + info.mIsAudio = !strncasecmp("audio/", mime.c_str(), 6); + + size_t index = mTrackInfos.size(); + mTrackInfos.push_back(info); + + return index; +} + +status_t MediaSender::initAsync( + ssize_t trackIndex, + const char *remoteHost, + int32_t remoteRTPPort, + RTPSender::TransportMode rtpMode, + int32_t remoteRTCPPort, + RTPSender::TransportMode rtcpMode, + int32_t *localRTPPort) { + if (trackIndex < 0) { + if (mMode != MODE_UNDEFINED) { + return INVALID_OPERATION; + } + + uint32_t flags = 0; + if (mHDCP != NULL) { + // XXX Determine proper HDCP version. + flags |= TSPacketizer::EMIT_HDCP20_DESCRIPTOR; + } + mTSPacketizer = new TSPacketizer(flags); + + status_t err = OK; + for (size_t i = 0; i < mTrackInfos.size(); ++i) { + TrackInfo *info = &mTrackInfos.editItemAt(i); + + ssize_t packetizerTrackIndex = + mTSPacketizer->addTrack(info->mFormat); + + if (packetizerTrackIndex < 0) { + err = packetizerTrackIndex; + break; + } + + info->mPacketizerTrackIndex = packetizerTrackIndex; + } + + if (err == OK) { + sp notify = new AMessage(kWhatSenderNotify, this); + notify->setInt32("generation", mGeneration); + mTSSender = new RTPSender(mNetSession, notify); + looper()->registerHandler(mTSSender); + + err = mTSSender->initAsync( + remoteHost, + remoteRTPPort, + rtpMode, + remoteRTCPPort, + rtcpMode, + localRTPPort); + + if (err != OK) { + looper()->unregisterHandler(mTSSender->id()); + mTSSender.clear(); + } + } + + if (err != OK) { + for (size_t i = 0; i < mTrackInfos.size(); ++i) { + TrackInfo *info = &mTrackInfos.editItemAt(i); + info->mPacketizerTrackIndex = -1; + } + + mTSPacketizer.clear(); + return err; + } + + mMode = MODE_TRANSPORT_STREAM; + mInitDoneCount = 1; + + return OK; + } + + if (mMode == MODE_TRANSPORT_STREAM) { + return INVALID_OPERATION; + } + + if ((size_t)trackIndex >= mTrackInfos.size()) { + return -ERANGE; + } + + TrackInfo *info = &mTrackInfos.editItemAt(trackIndex); + + if (info->mSender != NULL) { + return INVALID_OPERATION; + } + + sp notify = new AMessage(kWhatSenderNotify, this); + notify->setInt32("generation", mGeneration); + notify->setSize("trackIndex", trackIndex); + + info->mSender = new RTPSender(mNetSession, notify); + looper()->registerHandler(info->mSender); + + status_t err = info->mSender->initAsync( + remoteHost, + remoteRTPPort, + rtpMode, + remoteRTCPPort, + rtcpMode, + localRTPPort); + + if (err != OK) { + looper()->unregisterHandler(info->mSender->id()); + info->mSender.clear(); + + return err; + } + + if (mMode == MODE_UNDEFINED) { + mInitDoneCount = mTrackInfos.size(); + } + + mMode = MODE_ELEMENTARY_STREAMS; + + return OK; +} + +status_t MediaSender::queueAccessUnit( + size_t trackIndex, const sp &accessUnit) { + if (mMode == MODE_UNDEFINED) { + return INVALID_OPERATION; + } + + if (trackIndex >= mTrackInfos.size()) { + return -ERANGE; + } + + if (mMode == MODE_TRANSPORT_STREAM) { + TrackInfo *info = &mTrackInfos.editItemAt(trackIndex); + info->mAccessUnits.push_back(accessUnit); + + mTSPacketizer->extractCSDIfNecessary(info->mPacketizerTrackIndex); + + for (;;) { + ssize_t minTrackIndex = -1; + int64_t minTimeUs = -1ll; + + for (size_t i = 0; i < mTrackInfos.size(); ++i) { + const TrackInfo &info = mTrackInfos.itemAt(i); + + if (info.mAccessUnits.empty()) { + minTrackIndex = -1; + minTimeUs = -1ll; + break; + } + + int64_t timeUs; + const sp &accessUnit = *info.mAccessUnits.begin(); + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + if (minTrackIndex < 0 || timeUs < minTimeUs) { + minTrackIndex = i; + minTimeUs = timeUs; + } + } + + if (minTrackIndex < 0) { + return OK; + } + + TrackInfo *info = &mTrackInfos.editItemAt(minTrackIndex); + sp accessUnit = *info->mAccessUnits.begin(); + info->mAccessUnits.erase(info->mAccessUnits.begin()); + + sp tsPackets; + status_t err = packetizeAccessUnit( + minTrackIndex, accessUnit, &tsPackets); + + if (err == OK) { + if (mLogFile != NULL) { + fwrite(tsPackets->data(), 1, tsPackets->size(), mLogFile); + } + + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + tsPackets->meta()->setInt64("timeUs", timeUs); + + err = mTSSender->queueBuffer( + tsPackets, + 33 /* packetType */, + RTPSender::PACKETIZATION_TRANSPORT_STREAM); + } + + if (err != OK) { + return err; + } + } + } + + TrackInfo *info = &mTrackInfos.editItemAt(trackIndex); + + return info->mSender->queueBuffer( + accessUnit, + info->mIsAudio ? 96 : 97 /* packetType */, + info->mIsAudio + ? RTPSender::PACKETIZATION_AAC : RTPSender::PACKETIZATION_H264); +} + +void MediaSender::onMessageReceived(const sp &msg) { + switch (msg->what()) { + case kWhatSenderNotify: + { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + if (generation != mGeneration) { + break; + } + + onSenderNotify(msg); + break; + } + + default: + TRESPASS(); + } +} + +void MediaSender::onSenderNotify(const sp &msg) { + int32_t what; + CHECK(msg->findInt32("what", &what)); + + switch (what) { + case RTPSender::kWhatInitDone: + { + --mInitDoneCount; + + int32_t err; + CHECK(msg->findInt32("err", &err)); + + if (err != OK) { + notifyInitDone(err); + ++mGeneration; + break; + } + + if (mInitDoneCount == 0) { + notifyInitDone(OK); + } + break; + } + + case RTPSender::kWhatError: + { + int32_t err; + CHECK(msg->findInt32("err", &err)); + + notifyError(err); + break; + } + + case kWhatNetworkStall: + { + size_t numBytesQueued; + CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); + + notifyNetworkStall(numBytesQueued); + break; + } + + case kWhatInformSender: + { + int64_t avgLatencyUs; + CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs)); + + int64_t maxLatencyUs; + CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs)); + + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatInformSender); + notify->setInt64("avgLatencyUs", avgLatencyUs); + notify->setInt64("maxLatencyUs", maxLatencyUs); + notify->post(); + break; + } + + default: + TRESPASS(); + } +} + +void MediaSender::notifyInitDone(status_t err) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatInitDone); + notify->setInt32("err", err); + notify->post(); +} + +void MediaSender::notifyError(status_t err) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatError); + notify->setInt32("err", err); + notify->post(); +} + +void MediaSender::notifyNetworkStall(size_t numBytesQueued) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatNetworkStall); + notify->setSize("numBytesQueued", numBytesQueued); + notify->post(); +} + +status_t MediaSender::packetizeAccessUnit( + size_t trackIndex, + sp accessUnit, + sp *tsPackets) { + const TrackInfo &info = mTrackInfos.itemAt(trackIndex); + + uint32_t flags = 0; + + bool isHDCPEncrypted = false; + uint64_t inputCTR; + uint8_t HDCP_private_data[16]; + + bool manuallyPrependSPSPPS = + !info.mIsAudio + && (info.mFlags & FLAG_MANUALLY_PREPEND_SPS_PPS) + && IsIDR(accessUnit); + + if (mHDCP != NULL && !info.mIsAudio) { + isHDCPEncrypted = true; + + if (manuallyPrependSPSPPS) { + accessUnit = mTSPacketizer->prependCSD( + info.mPacketizerTrackIndex, accessUnit); + } + + status_t err; + native_handle_t* handle; + if (accessUnit->meta()->findPointer("handle", (void**)&handle) + && handle != NULL) { + int32_t rangeLength, rangeOffset; + sp notify; + CHECK(accessUnit->meta()->findInt32("rangeOffset", &rangeOffset)); + CHECK(accessUnit->meta()->findInt32("rangeLength", &rangeLength)); + CHECK(accessUnit->meta()->findMessage("notify", ¬ify) + && notify != NULL); + CHECK_GE((int32_t)accessUnit->size(), rangeLength); + + sp grbuf(new GraphicBuffer( + rangeOffset + rangeLength /* width */, 1 /* height */, + HAL_PIXEL_FORMAT_Y8, 1 /* layerCount */, + GRALLOC_USAGE_HW_VIDEO_ENCODER, + rangeOffset + rangeLength /* stride */, handle, + false /* keepOwnership */)); + + err = mHDCP->encryptNative( + grbuf, rangeOffset, rangeLength, + trackIndex /* streamCTR */, + &inputCTR, + accessUnit->data()); + notify->post(); + } else { + err = mHDCP->encrypt( + accessUnit->data(), accessUnit->size(), + trackIndex /* streamCTR */, + &inputCTR, + accessUnit->data()); + } + + if (err != OK) { + ALOGE("Failed to HDCP-encrypt media data (err %d)", + err); + + return err; + } + + HDCP_private_data[0] = 0x00; + + HDCP_private_data[1] = + (((trackIndex >> 30) & 3) << 1) | 1; + + HDCP_private_data[2] = (trackIndex >> 22) & 0xff; + + HDCP_private_data[3] = + (((trackIndex >> 15) & 0x7f) << 1) | 1; + + HDCP_private_data[4] = (trackIndex >> 7) & 0xff; + + HDCP_private_data[5] = + ((trackIndex & 0x7f) << 1) | 1; + + HDCP_private_data[6] = 0x00; + + HDCP_private_data[7] = + (((inputCTR >> 60) & 0x0f) << 1) | 1; + + HDCP_private_data[8] = (inputCTR >> 52) & 0xff; + + HDCP_private_data[9] = + (((inputCTR >> 45) & 0x7f) << 1) | 1; + + HDCP_private_data[10] = (inputCTR >> 37) & 0xff; + + HDCP_private_data[11] = + (((inputCTR >> 30) & 0x7f) << 1) | 1; + + HDCP_private_data[12] = (inputCTR >> 22) & 0xff; + + HDCP_private_data[13] = + (((inputCTR >> 15) & 0x7f) << 1) | 1; + + HDCP_private_data[14] = (inputCTR >> 7) & 0xff; + + HDCP_private_data[15] = + ((inputCTR & 0x7f) << 1) | 1; + + flags |= TSPacketizer::IS_ENCRYPTED; + } else if (manuallyPrependSPSPPS) { + flags |= TSPacketizer::PREPEND_SPS_PPS_TO_IDR_FRAMES; + } + + int64_t timeUs = ALooper::GetNowUs(); + if (mPrevTimeUs < 0ll || mPrevTimeUs + 100000ll <= timeUs) { + flags |= TSPacketizer::EMIT_PCR; + flags |= TSPacketizer::EMIT_PAT_AND_PMT; + + mPrevTimeUs = timeUs; + } + + mTSPacketizer->packetize( + info.mPacketizerTrackIndex, + accessUnit, + tsPackets, + flags, + !isHDCPEncrypted ? NULL : HDCP_private_data, + !isHDCPEncrypted ? 0 : sizeof(HDCP_private_data), + info.mIsAudio ? 2 : 0 /* numStuffingBytes */); + + return OK; +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/MediaSender.h b/media/libstagefright/wifi-display/MediaSender.h new file mode 100644 index 0000000000..04538ea1f6 --- /dev/null +++ b/media/libstagefright/wifi-display/MediaSender.h @@ -0,0 +1,132 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MEDIA_SENDER_H_ + +#define MEDIA_SENDER_H_ + +#include "rtp/RTPSender.h" + +#include +#include +#include +#include + +namespace android { + +struct ABuffer; +struct ANetworkSession; +struct AMessage; +struct IHDCP; +struct TSPacketizer; + +// This class facilitates sending of data from one or more media tracks +// through one or more RTP channels, either providing a 1:1 mapping from +// track to RTP channel or muxing all tracks into a single RTP channel and +// using transport stream encapsulation. +// Optionally the (video) data is encrypted using the provided hdcp object. +struct MediaSender : public AHandler { + enum { + kWhatInitDone, + kWhatError, + kWhatNetworkStall, + kWhatInformSender, + }; + + MediaSender( + const sp &netSession, + const sp ¬ify); + + status_t setHDCP(const sp &hdcp); + + enum FlagBits { + FLAG_MANUALLY_PREPEND_SPS_PPS = 1, + }; + ssize_t addTrack(const sp &format, uint32_t flags); + + // If trackIndex == -1, initialize for transport stream muxing. + status_t initAsync( + ssize_t trackIndex, + const char *remoteHost, + int32_t remoteRTPPort, + RTPSender::TransportMode rtpMode, + int32_t remoteRTCPPort, + RTPSender::TransportMode rtcpMode, + int32_t *localRTPPort); + + status_t queueAccessUnit( + size_t trackIndex, const sp &accessUnit); + +protected: + virtual void onMessageReceived(const sp &msg); + virtual ~MediaSender(); + +private: + enum { + kWhatSenderNotify, + }; + + enum Mode { + MODE_UNDEFINED, + MODE_TRANSPORT_STREAM, + MODE_ELEMENTARY_STREAMS, + }; + + struct TrackInfo { + sp mFormat; + uint32_t mFlags; + sp mSender; + List > mAccessUnits; + ssize_t mPacketizerTrackIndex; + bool mIsAudio; + }; + + sp mNetSession; + sp mNotify; + + sp mHDCP; + + Mode mMode; + int32_t mGeneration; + + Vector mTrackInfos; + + sp mTSPacketizer; + sp mTSSender; + int64_t mPrevTimeUs; + + size_t mInitDoneCount; + + FILE *mLogFile; + + void onSenderNotify(const sp &msg); + + void notifyInitDone(status_t err); + void notifyError(status_t err); + void notifyNetworkStall(size_t numBytesQueued); + + status_t packetizeAccessUnit( + size_t trackIndex, + sp accessUnit, + sp *tsPackets); + + DISALLOW_EVIL_CONSTRUCTORS(MediaSender); +}; + +} // namespace android + +#endif // MEDIA_SENDER_H_ + diff --git a/media/libstagefright/wifi-display/Parameters.cpp b/media/libstagefright/wifi-display/Parameters.cpp new file mode 100644 index 0000000000..d2a61ea464 --- /dev/null +++ b/media/libstagefright/wifi-display/Parameters.cpp @@ -0,0 +1,92 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Parameters.h" + +#include + +namespace android { + +// static +sp Parameters::Parse(const char *data, size_t size) { + sp params = new Parameters; + status_t err = params->parse(data, size); + + if (err != OK) { + return NULL; + } + + return params; +} + +Parameters::Parameters() {} + +Parameters::~Parameters() {} + +status_t Parameters::parse(const char *data, size_t size) { + size_t i = 0; + while (i < size) { + size_t nameStart = i; + while (i < size && data[i] != ':') { + ++i; + } + + if (i == size || i == nameStart) { + return ERROR_MALFORMED; + } + + AString name(&data[nameStart], i - nameStart); + name.trim(); + name.tolower(); + + ++i; + + size_t valueStart = i; + + while (i + 1 < size && (data[i] != '\r' || data[i + 1] != '\n')) { + ++i; + } + + AString value(&data[valueStart], i - valueStart); + value.trim(); + + mDict.add(name, value); + + while (i + 1 < size && data[i] == '\r' && data[i + 1] == '\n') { + i += 2; + } + } + + return OK; +} + +bool Parameters::findParameter(const char *name, AString *value) const { + AString key = name; + key.tolower(); + + ssize_t index = mDict.indexOfKey(key); + + if (index < 0) { + value->clear(); + + return false; + } + + *value = mDict.valueAt(index); + return true; +} + +} // namespace android diff --git a/media/libstagefright/wifi-display/Parameters.h b/media/libstagefright/wifi-display/Parameters.h new file mode 100644 index 0000000000..a5e787e234 --- /dev/null +++ b/media/libstagefright/wifi-display/Parameters.h @@ -0,0 +1,41 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +namespace android { + +struct Parameters : public RefBase { + static sp Parse(const char *data, size_t size); + + bool findParameter(const char *name, AString *value) const; + +protected: + virtual ~Parameters(); + +private: + KeyedVector mDict; + + Parameters(); + status_t parse(const char *data, size_t size); + + DISALLOW_EVIL_CONSTRUCTORS(Parameters); +}; + +} // namespace android diff --git a/media/libstagefright/wifi-display/VideoFormats.cpp b/media/libstagefright/wifi-display/VideoFormats.cpp new file mode 100644 index 0000000000..dbc511caa7 --- /dev/null +++ b/media/libstagefright/wifi-display/VideoFormats.cpp @@ -0,0 +1,550 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "VideoFormats" +#include + +#include "VideoFormats.h" + +#include + +namespace android { + +// static +const VideoFormats::config_t VideoFormats::mResolutionTable[][32] = { + { + // CEA Resolutions + { 640, 480, 60, false, 0, 0}, + { 720, 480, 60, false, 0, 0}, + { 720, 480, 60, true, 0, 0}, + { 720, 576, 50, false, 0, 0}, + { 720, 576, 50, true, 0, 0}, + { 1280, 720, 30, false, 0, 0}, + { 1280, 720, 60, false, 0, 0}, + { 1920, 1080, 30, false, 0, 0}, + { 1920, 1080, 60, false, 0, 0}, + { 1920, 1080, 60, true, 0, 0}, + { 1280, 720, 25, false, 0, 0}, + { 1280, 720, 50, false, 0, 0}, + { 1920, 1080, 25, false, 0, 0}, + { 1920, 1080, 50, false, 0, 0}, + { 1920, 1080, 50, true, 0, 0}, + { 1280, 720, 24, false, 0, 0}, + { 1920, 1080, 24, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + }, + { + // VESA Resolutions + { 800, 600, 30, false, 0, 0}, + { 800, 600, 60, false, 0, 0}, + { 1024, 768, 30, false, 0, 0}, + { 1024, 768, 60, false, 0, 0}, + { 1152, 864, 30, false, 0, 0}, + { 1152, 864, 60, false, 0, 0}, + { 1280, 768, 30, false, 0, 0}, + { 1280, 768, 60, false, 0, 0}, + { 1280, 800, 30, false, 0, 0}, + { 1280, 800, 60, false, 0, 0}, + { 1360, 768, 30, false, 0, 0}, + { 1360, 768, 60, false, 0, 0}, + { 1366, 768, 30, false, 0, 0}, + { 1366, 768, 60, false, 0, 0}, + { 1280, 1024, 30, false, 0, 0}, + { 1280, 1024, 60, false, 0, 0}, + { 1400, 1050, 30, false, 0, 0}, + { 1400, 1050, 60, false, 0, 0}, + { 1440, 900, 30, false, 0, 0}, + { 1440, 900, 60, false, 0, 0}, + { 1600, 900, 30, false, 0, 0}, + { 1600, 900, 60, false, 0, 0}, + { 1600, 1200, 30, false, 0, 0}, + { 1600, 1200, 60, false, 0, 0}, + { 1680, 1024, 30, false, 0, 0}, + { 1680, 1024, 60, false, 0, 0}, + { 1680, 1050, 30, false, 0, 0}, + { 1680, 1050, 60, false, 0, 0}, + { 1920, 1200, 30, false, 0, 0}, + { 1920, 1200, 60, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + }, + { + // HH Resolutions + { 800, 480, 30, false, 0, 0}, + { 800, 480, 60, false, 0, 0}, + { 854, 480, 30, false, 0, 0}, + { 854, 480, 60, false, 0, 0}, + { 864, 480, 30, false, 0, 0}, + { 864, 480, 60, false, 0, 0}, + { 640, 360, 30, false, 0, 0}, + { 640, 360, 60, false, 0, 0}, + { 960, 540, 30, false, 0, 0}, + { 960, 540, 60, false, 0, 0}, + { 848, 480, 30, false, 0, 0}, + { 848, 480, 60, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + { 0, 0, 0, false, 0, 0}, + } +}; + +VideoFormats::VideoFormats() { + memcpy(mConfigs, mResolutionTable, sizeof(mConfigs)); + + for (size_t i = 0; i < kNumResolutionTypes; ++i) { + mResolutionEnabled[i] = 0; + } + + setNativeResolution(RESOLUTION_CEA, 0); // default to 640x480 p60 +} + +void VideoFormats::setNativeResolution(ResolutionType type, size_t index) { + CHECK_LT(type, kNumResolutionTypes); + CHECK(GetConfiguration(type, index, NULL, NULL, NULL, NULL)); + + mNativeType = type; + mNativeIndex = index; + + setResolutionEnabled(type, index); +} + +void VideoFormats::getNativeResolution( + ResolutionType *type, size_t *index) const { + *type = mNativeType; + *index = mNativeIndex; +} + +void VideoFormats::disableAll() { + for (size_t i = 0; i < kNumResolutionTypes; ++i) { + mResolutionEnabled[i] = 0; + for (size_t j = 0; j < 32; j++) { + mConfigs[i][j].profile = mConfigs[i][j].level = 0; + } + } +} + +void VideoFormats::enableAll() { + for (size_t i = 0; i < kNumResolutionTypes; ++i) { + mResolutionEnabled[i] = 0xffffffff; + for (size_t j = 0; j < 32; j++) { + mConfigs[i][j].profile = (1ul << PROFILE_CBP); + mConfigs[i][j].level = (1ul << LEVEL_31); + } + } +} + +void VideoFormats::enableResolutionUpto( + ResolutionType type, size_t index, + ProfileType profile, LevelType level) { + size_t width, height, fps, score; + bool interlaced; + if (!GetConfiguration(type, index, &width, &height, + &fps, &interlaced)) { + ALOGE("Maximum resolution not found!"); + return; + } + score = width * height * fps * (!interlaced + 1); + for (size_t i = 0; i < kNumResolutionTypes; ++i) { + for (size_t j = 0; j < 32; j++) { + if (GetConfiguration((ResolutionType)i, j, + &width, &height, &fps, &interlaced) + && score >= width * height * fps * (!interlaced + 1)) { + setResolutionEnabled((ResolutionType)i, j); + setProfileLevel((ResolutionType)i, j, profile, level); + } + } + } +} + +void VideoFormats::setResolutionEnabled( + ResolutionType type, size_t index, bool enabled) { + CHECK_LT(type, kNumResolutionTypes); + CHECK(GetConfiguration(type, index, NULL, NULL, NULL, NULL)); + + if (enabled) { + mResolutionEnabled[type] |= (1ul << index); + mConfigs[type][index].profile = (1ul << PROFILE_CBP); + mConfigs[type][index].level = (1ul << LEVEL_31); + } else { + mResolutionEnabled[type] &= ~(1ul << index); + mConfigs[type][index].profile = 0; + mConfigs[type][index].level = 0; + } +} + +void VideoFormats::setProfileLevel( + ResolutionType type, size_t index, + ProfileType profile, LevelType level) { + CHECK_LT(type, kNumResolutionTypes); + CHECK(GetConfiguration(type, index, NULL, NULL, NULL, NULL)); + + mConfigs[type][index].profile = (1ul << profile); + mConfigs[type][index].level = (1ul << level); +} + +void VideoFormats::getProfileLevel( + ResolutionType type, size_t index, + ProfileType *profile, LevelType *level) const{ + CHECK_LT(type, kNumResolutionTypes); + CHECK(GetConfiguration(type, index, NULL, NULL, NULL, NULL)); + + int i, bestProfile = -1, bestLevel = -1; + + for (i = 0; i < kNumProfileTypes; ++i) { + if (mConfigs[type][index].profile & (1ul << i)) { + bestProfile = i; + } + } + + for (i = 0; i < kNumLevelTypes; ++i) { + if (mConfigs[type][index].level & (1ul << i)) { + bestLevel = i; + } + } + + if (bestProfile == -1 || bestLevel == -1) { + ALOGE("Profile or level not set for resolution type %d, index %zu", + type, index); + bestProfile = PROFILE_CBP; + bestLevel = LEVEL_31; + } + + *profile = (ProfileType) bestProfile; + *level = (LevelType) bestLevel; +} + +bool VideoFormats::isResolutionEnabled( + ResolutionType type, size_t index) const { + CHECK_LT(type, kNumResolutionTypes); + CHECK(GetConfiguration(type, index, NULL, NULL, NULL, NULL)); + + return mResolutionEnabled[type] & (1ul << index); +} + +// static +bool VideoFormats::GetConfiguration( + ResolutionType type, + size_t index, + size_t *width, size_t *height, size_t *framesPerSecond, + bool *interlaced) { + CHECK_LT(type, kNumResolutionTypes); + + if (index >= 32) { + return false; + } + + const config_t *config = &mResolutionTable[type][index]; + + if (config->width == 0) { + return false; + } + + if (width) { + *width = config->width; + } + + if (height) { + *height = config->height; + } + + if (framesPerSecond) { + *framesPerSecond = config->framesPerSecond; + } + + if (interlaced) { + *interlaced = config->interlaced; + } + + return true; +} + +bool VideoFormats::parseH264Codec(const char *spec) { + unsigned profile, level, res[3]; + + if (sscanf( + spec, + "%02x %02x %08X %08X %08X", + &profile, + &level, + &res[0], + &res[1], + &res[2]) != 5) { + return false; + } + + for (size_t i = 0; i < kNumResolutionTypes; ++i) { + for (size_t j = 0; j < 32; ++j) { + if (res[i] & (1ul << j)){ + mResolutionEnabled[i] |= (1ul << j); + if (profile > mConfigs[i][j].profile) { + // prefer higher profile (even if level is lower) + mConfigs[i][j].profile = profile; + mConfigs[i][j].level = level; + } else if (profile == mConfigs[i][j].profile && + level > mConfigs[i][j].level) { + mConfigs[i][j].level = level; + } + } + } + } + + return true; +} + +// static +bool VideoFormats::GetProfileLevel( + ProfileType profile, LevelType level, unsigned *profileIdc, + unsigned *levelIdc, unsigned *constraintSet) { + CHECK_LT(profile, kNumProfileTypes); + CHECK_LT(level, kNumLevelTypes); + + static const unsigned kProfileIDC[kNumProfileTypes] = { + 66, // PROFILE_CBP + 100, // PROFILE_CHP + }; + + static const unsigned kLevelIDC[kNumLevelTypes] = { + 31, // LEVEL_31 + 32, // LEVEL_32 + 40, // LEVEL_40 + 41, // LEVEL_41 + 42, // LEVEL_42 + }; + + static const unsigned kConstraintSet[kNumProfileTypes] = { + 0xc0, // PROFILE_CBP + 0x0c, // PROFILE_CHP + }; + + if (profileIdc) { + *profileIdc = kProfileIDC[profile]; + } + + if (levelIdc) { + *levelIdc = kLevelIDC[level]; + } + + if (constraintSet) { + *constraintSet = kConstraintSet[profile]; + } + + return true; +} + +bool VideoFormats::parseFormatSpec(const char *spec) { + CHECK_EQ(kNumResolutionTypes, 3); + + disableAll(); + + unsigned native, dummy; + size_t size = strlen(spec); + size_t offset = 0; + + if (sscanf(spec, "%02x %02x ", &native, &dummy) != 2) { + return false; + } + + offset += 6; // skip native and preferred-display-mode-supported + CHECK_LE(offset + 58, size); + while (offset < size) { + parseH264Codec(spec + offset); + offset += 60; // skip H.264-codec + ", " + } + + mNativeIndex = native >> 3; + mNativeType = (ResolutionType)(native & 7); + + bool success; + if (mNativeType >= kNumResolutionTypes) { + success = false; + } else { + success = GetConfiguration( + mNativeType, mNativeIndex, NULL, NULL, NULL, NULL); + } + + if (!success) { + ALOGW("sink advertised an illegal native resolution, fortunately " + "this value is ignored for the time being..."); + } + + return true; +} + +AString VideoFormats::getFormatSpec(bool forM4Message) const { + CHECK_EQ(kNumResolutionTypes, 3); + + // wfd_video_formats: + // 1 byte "native" + // 1 byte "preferred-display-mode-supported" 0 or 1 + // one or more avc codec structures + // 1 byte profile + // 1 byte level + // 4 byte CEA mask + // 4 byte VESA mask + // 4 byte HH mask + // 1 byte latency + // 2 byte min-slice-slice + // 2 byte slice-enc-params + // 1 byte framerate-control-support + // max-hres (none or 2 byte) + // max-vres (none or 2 byte) + + return AStringPrintf( + "%02x 00 %02x %02x %08x %08x %08x 00 0000 0000 00 none none", + forM4Message ? 0x00 : ((mNativeIndex << 3) | mNativeType), + mConfigs[mNativeType][mNativeIndex].profile, + mConfigs[mNativeType][mNativeIndex].level, + mResolutionEnabled[0], + mResolutionEnabled[1], + mResolutionEnabled[2]); +} + +// static +bool VideoFormats::PickBestFormat( + const VideoFormats &sinkSupported, + const VideoFormats &sourceSupported, + ResolutionType *chosenType, + size_t *chosenIndex, + ProfileType *chosenProfile, + LevelType *chosenLevel) { +#if 0 + // Support for the native format is a great idea, the spec includes + // these features, but nobody supports it and the tests don't validate it. + + ResolutionType nativeType; + size_t nativeIndex; + sinkSupported.getNativeResolution(&nativeType, &nativeIndex); + if (sinkSupported.isResolutionEnabled(nativeType, nativeIndex)) { + if (sourceSupported.isResolutionEnabled(nativeType, nativeIndex)) { + ALOGI("Choosing sink's native resolution"); + *chosenType = nativeType; + *chosenIndex = nativeIndex; + return true; + } + } else { + ALOGW("Sink advertised native resolution that it doesn't " + "actually support... ignoring"); + } + + sourceSupported.getNativeResolution(&nativeType, &nativeIndex); + if (sourceSupported.isResolutionEnabled(nativeType, nativeIndex)) { + if (sinkSupported.isResolutionEnabled(nativeType, nativeIndex)) { + ALOGI("Choosing source's native resolution"); + *chosenType = nativeType; + *chosenIndex = nativeIndex; + return true; + } + } else { + ALOGW("Source advertised native resolution that it doesn't " + "actually support... ignoring"); + } +#endif + + bool first = true; + uint32_t bestScore = 0; + size_t bestType = 0; + size_t bestIndex = 0; + for (size_t i = 0; i < kNumResolutionTypes; ++i) { + for (size_t j = 0; j < 32; ++j) { + size_t width, height, framesPerSecond; + bool interlaced; + if (!GetConfiguration( + (ResolutionType)i, + j, + &width, &height, &framesPerSecond, &interlaced)) { + break; + } + + if (!sinkSupported.isResolutionEnabled((ResolutionType)i, j) + || !sourceSupported.isResolutionEnabled( + (ResolutionType)i, j)) { + continue; + } + + ALOGV("type %zu, index %zu, %zu x %zu %c%zu supported", + i, j, width, height, interlaced ? 'i' : 'p', framesPerSecond); + + uint32_t score = width * height * framesPerSecond; + if (!interlaced) { + score *= 2; + } + + if (first || score > bestScore) { + bestScore = score; + bestType = i; + bestIndex = j; + + first = false; + } + } + } + + if (first) { + return false; + } + + *chosenType = (ResolutionType)bestType; + *chosenIndex = bestIndex; + + // Pick the best profile/level supported by both sink and source. + ProfileType srcProfile, sinkProfile; + LevelType srcLevel, sinkLevel; + sourceSupported.getProfileLevel( + (ResolutionType)bestType, bestIndex, + &srcProfile, &srcLevel); + sinkSupported.getProfileLevel( + (ResolutionType)bestType, bestIndex, + &sinkProfile, &sinkLevel); + *chosenProfile = srcProfile < sinkProfile ? srcProfile : sinkProfile; + *chosenLevel = srcLevel < sinkLevel ? srcLevel : sinkLevel; + + return true; +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/VideoFormats.h b/media/libstagefright/wifi-display/VideoFormats.h new file mode 100644 index 0000000000..fd38fd192e --- /dev/null +++ b/media/libstagefright/wifi-display/VideoFormats.h @@ -0,0 +1,125 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef VIDEO_FORMATS_H_ + +#define VIDEO_FORMATS_H_ + +#include + +#include + +namespace android { + +struct AString; + +// This class encapsulates that video resolution capabilities of a wfd source +// or sink as outlined in the wfd specs. Currently three sets of resolutions +// are specified, each of which supports up to 32 resolutions. +// In addition to its capabilities each sink/source also publishes its +// "native" resolution, presumably one that is preferred among all others +// because it wouldn't require any scaling and directly corresponds to the +// display capabilities/pixels. +struct VideoFormats { + VideoFormats(); + + struct config_t { + size_t width, height, framesPerSecond; + bool interlaced; + unsigned char profile, level; + }; + + enum ProfileType { + PROFILE_CBP = 0, + PROFILE_CHP, + kNumProfileTypes, + }; + + enum LevelType { + LEVEL_31 = 0, + LEVEL_32, + LEVEL_40, + LEVEL_41, + LEVEL_42, + kNumLevelTypes, + }; + + enum ResolutionType { + RESOLUTION_CEA, + RESOLUTION_VESA, + RESOLUTION_HH, + kNumResolutionTypes, + }; + + void setNativeResolution(ResolutionType type, size_t index); + void getNativeResolution(ResolutionType *type, size_t *index) const; + + void disableAll(); + void enableAll(); + void enableResolutionUpto( + ResolutionType type, size_t index, + ProfileType profile, LevelType level); + + void setResolutionEnabled( + ResolutionType type, size_t index, bool enabled = true); + + bool isResolutionEnabled(ResolutionType type, size_t index) const; + + void setProfileLevel( + ResolutionType type, size_t index, + ProfileType profile, LevelType level); + + void getProfileLevel( + ResolutionType type, size_t index, + ProfileType *profile, LevelType *level) const; + + static bool GetConfiguration( + ResolutionType type, size_t index, + size_t *width, size_t *height, size_t *framesPerSecond, + bool *interlaced); + + static bool GetProfileLevel( + ProfileType profile, LevelType level, + unsigned *profileIdc, unsigned *levelIdc, + unsigned *constraintSet); + + bool parseFormatSpec(const char *spec); + AString getFormatSpec(bool forM4Message = false) const; + + static bool PickBestFormat( + const VideoFormats &sinkSupported, + const VideoFormats &sourceSupported, + ResolutionType *chosenType, + size_t *chosenIndex, + ProfileType *chosenProfile, + LevelType *chosenLevel); + +private: + bool parseH264Codec(const char *spec); + ResolutionType mNativeType; + size_t mNativeIndex; + + uint32_t mResolutionEnabled[kNumResolutionTypes]; + static const config_t mResolutionTable[kNumResolutionTypes][32]; + config_t mConfigs[kNumResolutionTypes][32]; + + DISALLOW_EVIL_CONSTRUCTORS(VideoFormats); +}; + +} // namespace android + +#endif // VIDEO_FORMATS_H_ + diff --git a/media/libstagefright/wifi-display/rtp/RTPBase.h b/media/libstagefright/wifi-display/rtp/RTPBase.h new file mode 100644 index 0000000000..194f1ee13b --- /dev/null +++ b/media/libstagefright/wifi-display/rtp/RTPBase.h @@ -0,0 +1,49 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RTP_BASE_H_ + +#define RTP_BASE_H_ + +namespace android { + +struct RTPBase { + enum PacketizationMode { + PACKETIZATION_TRANSPORT_STREAM, + PACKETIZATION_H264, + PACKETIZATION_AAC, + PACKETIZATION_NONE, + }; + + enum TransportMode { + TRANSPORT_UNDEFINED, + TRANSPORT_NONE, + TRANSPORT_UDP, + TRANSPORT_TCP, + TRANSPORT_TCP_INTERLEAVED, + }; + + // Really UDP _payload_ size + const unsigned int kMaxUDPPacketSize = 1472; // 1472 good, 1473 bad on Android@Home + + static int32_t PickRandomRTPPort(); +}; + +} // namespace android + +#endif // RTP_BASE_H_ + + diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.cpp b/media/libstagefright/wifi-display/rtp/RTPSender.cpp new file mode 100644 index 0000000000..ca9fdd2bd7 --- /dev/null +++ b/media/libstagefright/wifi-display/rtp/RTPSender.cpp @@ -0,0 +1,808 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "RTPSender" +#include + +#include "RTPSender.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "include/avc_utils.h" + +namespace android { + +RTPSender::RTPSender( + const sp &netSession, + const sp ¬ify) + : mNetSession(netSession), + mNotify(notify), + mRTPMode(TRANSPORT_UNDEFINED), + mRTCPMode(TRANSPORT_UNDEFINED), + mRTPSessionID(0), + mRTCPSessionID(0), + mRTPConnected(false), + mRTCPConnected(false), + mLastNTPTime(0), + mLastRTPTime(0), + mNumRTPSent(0), + mNumRTPOctetsSent(0), + mNumSRsSent(0), + mRTPSeqNo(0), + mHistorySize(0) { +} + +RTPSender::~RTPSender() { + if (mRTCPSessionID != 0) { + mNetSession->destroySession(mRTCPSessionID); + mRTCPSessionID = 0; + } + + if (mRTPSessionID != 0) { + mNetSession->destroySession(mRTPSessionID); + mRTPSessionID = 0; + } +} + +// static +int32_t RTPBase::PickRandomRTPPort() { + // Pick an even integer in range [1024, 65534) + + static const size_t kRange = (65534 - 1024) / 2; + + return (int32_t)(((float)(kRange + 1) * rand()) / RAND_MAX) * 2 + 1024; +} + +status_t RTPSender::initAsync( + const char *remoteHost, + int32_t remoteRTPPort, + TransportMode rtpMode, + int32_t remoteRTCPPort, + TransportMode rtcpMode, + int32_t *outLocalRTPPort) { + if (mRTPMode != TRANSPORT_UNDEFINED + || rtpMode == TRANSPORT_UNDEFINED + || rtpMode == TRANSPORT_NONE + || rtcpMode == TRANSPORT_UNDEFINED) { + return INVALID_OPERATION; + } + + CHECK_NE(rtpMode, TRANSPORT_TCP_INTERLEAVED); + CHECK_NE(rtcpMode, TRANSPORT_TCP_INTERLEAVED); + + if ((rtcpMode == TRANSPORT_NONE && remoteRTCPPort >= 0) + || (rtcpMode != TRANSPORT_NONE && remoteRTCPPort < 0)) { + return INVALID_OPERATION; + } + + sp rtpNotify = new AMessage(kWhatRTPNotify, this); + + sp rtcpNotify; + if (remoteRTCPPort >= 0) { + rtcpNotify = new AMessage(kWhatRTCPNotify, this); + } + + CHECK_EQ(mRTPSessionID, 0); + CHECK_EQ(mRTCPSessionID, 0); + + int32_t localRTPPort; + + for (;;) { + localRTPPort = PickRandomRTPPort(); + + status_t err; + if (rtpMode == TRANSPORT_UDP) { + err = mNetSession->createUDPSession( + localRTPPort, + remoteHost, + remoteRTPPort, + rtpNotify, + &mRTPSessionID); + } else { + CHECK_EQ(rtpMode, TRANSPORT_TCP); + err = mNetSession->createTCPDatagramSession( + localRTPPort, + remoteHost, + remoteRTPPort, + rtpNotify, + &mRTPSessionID); + } + + if (err != OK) { + continue; + } + + if (remoteRTCPPort < 0) { + break; + } + + if (rtcpMode == TRANSPORT_UDP) { + err = mNetSession->createUDPSession( + localRTPPort + 1, + remoteHost, + remoteRTCPPort, + rtcpNotify, + &mRTCPSessionID); + } else { + CHECK_EQ(rtcpMode, TRANSPORT_TCP); + err = mNetSession->createTCPDatagramSession( + localRTPPort + 1, + remoteHost, + remoteRTCPPort, + rtcpNotify, + &mRTCPSessionID); + } + + if (err == OK) { + break; + } + + mNetSession->destroySession(mRTPSessionID); + mRTPSessionID = 0; + } + + if (rtpMode == TRANSPORT_UDP) { + mRTPConnected = true; + } + + if (rtcpMode == TRANSPORT_UDP) { + mRTCPConnected = true; + } + + mRTPMode = rtpMode; + mRTCPMode = rtcpMode; + *outLocalRTPPort = localRTPPort; + + if (mRTPMode == TRANSPORT_UDP + && (mRTCPMode == TRANSPORT_UDP || mRTCPMode == TRANSPORT_NONE)) { + notifyInitDone(OK); + } + + return OK; +} + +status_t RTPSender::queueBuffer( + const sp &buffer, uint8_t packetType, PacketizationMode mode) { + status_t err; + + switch (mode) { + case PACKETIZATION_NONE: + err = queueRawPacket(buffer, packetType); + break; + + case PACKETIZATION_TRANSPORT_STREAM: + err = queueTSPackets(buffer, packetType); + break; + + case PACKETIZATION_H264: + err = queueAVCBuffer(buffer, packetType); + break; + + default: + TRESPASS(); + } + + return err; +} + +status_t RTPSender::queueRawPacket( + const sp &packet, uint8_t packetType) { + CHECK_LE(packet->size(), kMaxUDPPacketSize - 12); + + int64_t timeUs; + CHECK(packet->meta()->findInt64("timeUs", &timeUs)); + + sp udpPacket = new ABuffer(12 + packet->size()); + + udpPacket->setInt32Data(mRTPSeqNo); + + uint8_t *rtp = udpPacket->data(); + rtp[0] = 0x80; + rtp[1] = packetType; + + rtp[2] = (mRTPSeqNo >> 8) & 0xff; + rtp[3] = mRTPSeqNo & 0xff; + ++mRTPSeqNo; + + uint32_t rtpTime = (timeUs * 9) / 100ll; + + rtp[4] = rtpTime >> 24; + rtp[5] = (rtpTime >> 16) & 0xff; + rtp[6] = (rtpTime >> 8) & 0xff; + rtp[7] = rtpTime & 0xff; + + rtp[8] = kSourceID >> 24; + rtp[9] = (kSourceID >> 16) & 0xff; + rtp[10] = (kSourceID >> 8) & 0xff; + rtp[11] = kSourceID & 0xff; + + memcpy(&rtp[12], packet->data(), packet->size()); + + return sendRTPPacket( + udpPacket, + true /* storeInHistory */, + true /* timeValid */, + ALooper::GetNowUs()); +} + +status_t RTPSender::queueTSPackets( + const sp &tsPackets, uint8_t packetType) { + CHECK_EQ(0u, tsPackets->size() % 188); + + int64_t timeUs; + CHECK(tsPackets->meta()->findInt64("timeUs", &timeUs)); + + size_t srcOffset = 0; + while (srcOffset < tsPackets->size()) { + sp udpPacket = + new ABuffer(12 + kMaxNumTSPacketsPerRTPPacket * 188); + + udpPacket->setInt32Data(mRTPSeqNo); + + uint8_t *rtp = udpPacket->data(); + rtp[0] = 0x80; + rtp[1] = packetType; + + rtp[2] = (mRTPSeqNo >> 8) & 0xff; + rtp[3] = mRTPSeqNo & 0xff; + ++mRTPSeqNo; + + int64_t nowUs = ALooper::GetNowUs(); + uint32_t rtpTime = (nowUs * 9) / 100ll; + + rtp[4] = rtpTime >> 24; + rtp[5] = (rtpTime >> 16) & 0xff; + rtp[6] = (rtpTime >> 8) & 0xff; + rtp[7] = rtpTime & 0xff; + + rtp[8] = kSourceID >> 24; + rtp[9] = (kSourceID >> 16) & 0xff; + rtp[10] = (kSourceID >> 8) & 0xff; + rtp[11] = kSourceID & 0xff; + + size_t numTSPackets = (tsPackets->size() - srcOffset) / 188; + if (numTSPackets > kMaxNumTSPacketsPerRTPPacket) { + numTSPackets = kMaxNumTSPacketsPerRTPPacket; + } + + memcpy(&rtp[12], tsPackets->data() + srcOffset, numTSPackets * 188); + + udpPacket->setRange(0, 12 + numTSPackets * 188); + + srcOffset += numTSPackets * 188; + bool isLastPacket = (srcOffset == tsPackets->size()); + + status_t err = sendRTPPacket( + udpPacket, + true /* storeInHistory */, + isLastPacket /* timeValid */, + timeUs); + + if (err != OK) { + return err; + } + } + + return OK; +} + +status_t RTPSender::queueAVCBuffer( + const sp &accessUnit, uint8_t packetType) { + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + uint32_t rtpTime = (timeUs * 9 / 100ll); + + List > packets; + + sp out = new ABuffer(kMaxUDPPacketSize); + size_t outBytesUsed = 12; // Placeholder for RTP header. + + const uint8_t *data = accessUnit->data(); + size_t size = accessUnit->size(); + const uint8_t *nalStart; + size_t nalSize; + while (getNextNALUnit( + &data, &size, &nalStart, &nalSize, + true /* startCodeFollows */) == OK) { + size_t bytesNeeded = nalSize + 2; + if (outBytesUsed == 12) { + ++bytesNeeded; + } + + if (outBytesUsed + bytesNeeded > out->capacity()) { + bool emitSingleNALPacket = false; + + if (outBytesUsed == 12 + && outBytesUsed + nalSize <= out->capacity()) { + // We haven't emitted anything into the current packet yet and + // this NAL unit fits into a single-NAL-unit-packet while + // it wouldn't have fit as part of a STAP-A packet. + + memcpy(out->data() + outBytesUsed, nalStart, nalSize); + outBytesUsed += nalSize; + + emitSingleNALPacket = true; + } + + if (outBytesUsed > 12) { + out->setRange(0, outBytesUsed); + packets.push_back(out); + out = new ABuffer(kMaxUDPPacketSize); + outBytesUsed = 12; // Placeholder for RTP header + } + + if (emitSingleNALPacket) { + continue; + } + } + + if (outBytesUsed + bytesNeeded <= out->capacity()) { + uint8_t *dst = out->data() + outBytesUsed; + + if (outBytesUsed == 12) { + *dst++ = 24; // STAP-A header + } + + *dst++ = (nalSize >> 8) & 0xff; + *dst++ = nalSize & 0xff; + memcpy(dst, nalStart, nalSize); + + outBytesUsed += bytesNeeded; + continue; + } + + // This single NAL unit does not fit into a single RTP packet, + // we need to emit an FU-A. + + CHECK_EQ(outBytesUsed, 12u); + + uint8_t nalType = nalStart[0] & 0x1f; + uint8_t nri = (nalStart[0] >> 5) & 3; + + size_t srcOffset = 1; + while (srcOffset < nalSize) { + size_t copy = out->capacity() - outBytesUsed - 2; + if (copy > nalSize - srcOffset) { + copy = nalSize - srcOffset; + } + + uint8_t *dst = out->data() + outBytesUsed; + dst[0] = (nri << 5) | 28; + + dst[1] = nalType; + + if (srcOffset == 1) { + dst[1] |= 0x80; + } + + if (srcOffset + copy == nalSize) { + dst[1] |= 0x40; + } + + memcpy(&dst[2], nalStart + srcOffset, copy); + srcOffset += copy; + + out->setRange(0, outBytesUsed + copy + 2); + + packets.push_back(out); + out = new ABuffer(kMaxUDPPacketSize); + outBytesUsed = 12; // Placeholder for RTP header + } + } + + if (outBytesUsed > 12) { + out->setRange(0, outBytesUsed); + packets.push_back(out); + } + + while (!packets.empty()) { + sp out = *packets.begin(); + packets.erase(packets.begin()); + + out->setInt32Data(mRTPSeqNo); + + bool last = packets.empty(); + + uint8_t *dst = out->data(); + + dst[0] = 0x80; + + dst[1] = packetType; + if (last) { + dst[1] |= 1 << 7; // M-bit + } + + dst[2] = (mRTPSeqNo >> 8) & 0xff; + dst[3] = mRTPSeqNo & 0xff; + ++mRTPSeqNo; + + dst[4] = rtpTime >> 24; + dst[5] = (rtpTime >> 16) & 0xff; + dst[6] = (rtpTime >> 8) & 0xff; + dst[7] = rtpTime & 0xff; + dst[8] = kSourceID >> 24; + dst[9] = (kSourceID >> 16) & 0xff; + dst[10] = (kSourceID >> 8) & 0xff; + dst[11] = kSourceID & 0xff; + + status_t err = sendRTPPacket(out, true /* storeInHistory */); + + if (err != OK) { + return err; + } + } + + return OK; +} + +status_t RTPSender::sendRTPPacket( + const sp &buffer, bool storeInHistory, + bool timeValid, int64_t timeUs) { + CHECK(mRTPConnected); + + status_t err = mNetSession->sendRequest( + mRTPSessionID, buffer->data(), buffer->size(), + timeValid, timeUs); + + if (err != OK) { + return err; + } + + mLastNTPTime = GetNowNTP(); + mLastRTPTime = U32_AT(buffer->data() + 4); + + ++mNumRTPSent; + mNumRTPOctetsSent += buffer->size() - 12; + + if (storeInHistory) { + if (mHistorySize == kMaxHistorySize) { + mHistory.erase(mHistory.begin()); + } else { + ++mHistorySize; + } + mHistory.push_back(buffer); + } + + return OK; +} + +// static +uint64_t RTPSender::GetNowNTP() { + struct timeval tv; + gettimeofday(&tv, NULL /* timezone */); + + uint64_t nowUs = tv.tv_sec * 1000000ll + tv.tv_usec; + + nowUs += ((70ll * 365 + 17) * 24) * 60 * 60 * 1000000ll; + + uint64_t hi = nowUs / 1000000ll; + uint64_t lo = ((1ll << 32) * (nowUs % 1000000ll)) / 1000000ll; + + return (hi << 32) | lo; +} + +void RTPSender::onMessageReceived(const sp &msg) { + switch (msg->what()) { + case kWhatRTPNotify: + case kWhatRTCPNotify: + onNetNotify(msg->what() == kWhatRTPNotify, msg); + break; + + default: + TRESPASS(); + } +} + +void RTPSender::onNetNotify(bool isRTP, const sp &msg) { + int32_t reason; + CHECK(msg->findInt32("reason", &reason)); + + switch (reason) { + case ANetworkSession::kWhatError: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + int32_t err; + CHECK(msg->findInt32("err", &err)); + + int32_t errorOccuredDuringSend; + CHECK(msg->findInt32("send", &errorOccuredDuringSend)); + + AString detail; + CHECK(msg->findString("detail", &detail)); + + ALOGE("An error occurred during %s in session %d " + "(%d, '%s' (%s)).", + errorOccuredDuringSend ? "send" : "receive", + sessionID, + err, + detail.c_str(), + strerror(-err)); + + mNetSession->destroySession(sessionID); + + if (sessionID == mRTPSessionID) { + mRTPSessionID = 0; + } else if (sessionID == mRTCPSessionID) { + mRTCPSessionID = 0; + } + + if (!mRTPConnected + || (mRTPMode != TRANSPORT_NONE && !mRTCPConnected)) { + // We haven't completed initialization, attach the error + // to the notification instead. + notifyInitDone(err); + break; + } + + notifyError(err); + break; + } + + case ANetworkSession::kWhatDatagram: + { + sp data; + CHECK(msg->findBuffer("data", &data)); + + if (isRTP) { + ALOGW("Huh? Received data on RTP connection..."); + } else { + onRTCPData(data); + } + break; + } + + case ANetworkSession::kWhatConnected: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + if (isRTP) { + CHECK_EQ(mRTPMode, TRANSPORT_TCP); + CHECK_EQ(sessionID, mRTPSessionID); + mRTPConnected = true; + } else { + CHECK_EQ(mRTCPMode, TRANSPORT_TCP); + CHECK_EQ(sessionID, mRTCPSessionID); + mRTCPConnected = true; + } + + if (mRTPConnected + && (mRTCPMode == TRANSPORT_NONE || mRTCPConnected)) { + notifyInitDone(OK); + } + break; + } + + case ANetworkSession::kWhatNetworkStall: + { + size_t numBytesQueued; + CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); + + notifyNetworkStall(numBytesQueued); + break; + } + + default: + TRESPASS(); + } +} + +status_t RTPSender::onRTCPData(const sp &buffer) { + const uint8_t *data = buffer->data(); + size_t size = buffer->size(); + + while (size > 0) { + if (size < 8) { + // Too short to be a valid RTCP header + return ERROR_MALFORMED; + } + + if ((data[0] >> 6) != 2) { + // Unsupported version. + return ERROR_UNSUPPORTED; + } + + if (data[0] & 0x20) { + // Padding present. + + size_t paddingLength = data[size - 1]; + + if (paddingLength + 12 > size) { + // If we removed this much padding we'd end up with something + // that's too short to be a valid RTP header. + return ERROR_MALFORMED; + } + + size -= paddingLength; + } + + size_t headerLength = 4 * (data[2] << 8 | data[3]) + 4; + + if (size < headerLength) { + // Only received a partial packet? + return ERROR_MALFORMED; + } + + switch (data[1]) { + case 200: + case 201: // RR + parseReceiverReport(data, headerLength); + break; + + case 202: // SDES + case 203: + break; + + case 204: // APP + parseAPP(data, headerLength); + break; + + case 205: // TSFB (transport layer specific feedback) + parseTSFB(data, headerLength); + break; + + case 206: // PSFB (payload specific feedback) + // hexdump(data, headerLength); + break; + + default: + { + ALOGW("Unknown RTCP packet type %u of size %zu", + (unsigned)data[1], headerLength); + break; + } + } + + data += headerLength; + size -= headerLength; + } + + return OK; +} + +status_t RTPSender::parseReceiverReport( + const uint8_t *data, size_t /* size */) { + float fractionLost = data[12] / 256.0f; + + ALOGI("lost %.2f %% of packets during report interval.", + 100.0f * fractionLost); + + return OK; +} + +status_t RTPSender::parseTSFB(const uint8_t *data, size_t size) { + if ((data[0] & 0x1f) != 1) { + return ERROR_UNSUPPORTED; // We only support NACK for now. + } + + uint32_t srcId = U32_AT(&data[8]); + if (srcId != kSourceID) { + return ERROR_MALFORMED; + } + + for (size_t i = 12; i < size; i += 4) { + uint16_t seqNo = U16_AT(&data[i]); + uint16_t blp = U16_AT(&data[i + 2]); + + List >::iterator it = mHistory.begin(); + bool foundSeqNo = false; + while (it != mHistory.end()) { + const sp &buffer = *it; + + uint16_t bufferSeqNo = buffer->int32Data() & 0xffff; + + bool retransmit = false; + if (bufferSeqNo == seqNo) { + retransmit = true; + } else if (blp != 0) { + for (size_t i = 0; i < 16; ++i) { + if ((blp & (1 << i)) + && (bufferSeqNo == ((seqNo + i + 1) & 0xffff))) { + blp &= ~(1 << i); + retransmit = true; + } + } + } + + if (retransmit) { + ALOGV("retransmitting seqNo %d", bufferSeqNo); + + CHECK_EQ((status_t)OK, + sendRTPPacket(buffer, false /* storeInHistory */)); + + if (bufferSeqNo == seqNo) { + foundSeqNo = true; + } + + if (foundSeqNo && blp == 0) { + break; + } + } + + ++it; + } + + if (!foundSeqNo || blp != 0) { + ALOGI("Some sequence numbers were no longer available for " + "retransmission (seqNo = %d, foundSeqNo = %d, blp = 0x%04x)", + seqNo, foundSeqNo, blp); + + if (!mHistory.empty()) { + int32_t earliest = (*mHistory.begin())->int32Data() & 0xffff; + int32_t latest = (*--mHistory.end())->int32Data() & 0xffff; + + ALOGI("have seq numbers from %d - %d", earliest, latest); + } + } + } + + return OK; +} + +status_t RTPSender::parseAPP(const uint8_t *data, size_t size) { + static const size_t late_offset = 8; + static const char late_string[] = "late"; + static const size_t avgLatencyUs_offset = late_offset + sizeof(late_string) - 1; + static const size_t maxLatencyUs_offset = avgLatencyUs_offset + sizeof(int64_t); + + if ((size >= (maxLatencyUs_offset + sizeof(int64_t))) + && !memcmp(late_string, &data[late_offset], sizeof(late_string) - 1)) { + int64_t avgLatencyUs = (int64_t)U64_AT(&data[avgLatencyUs_offset]); + int64_t maxLatencyUs = (int64_t)U64_AT(&data[maxLatencyUs_offset]); + + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatInformSender); + notify->setInt64("avgLatencyUs", avgLatencyUs); + notify->setInt64("maxLatencyUs", maxLatencyUs); + notify->post(); + } + + return OK; +} + +void RTPSender::notifyInitDone(status_t err) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatInitDone); + notify->setInt32("err", err); + notify->post(); +} + +void RTPSender::notifyError(status_t err) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatError); + notify->setInt32("err", err); + notify->post(); +} + +void RTPSender::notifyNetworkStall(size_t numBytesQueued) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatNetworkStall); + notify->setSize("numBytesQueued", numBytesQueued); + notify->post(); +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.h b/media/libstagefright/wifi-display/rtp/RTPSender.h new file mode 100644 index 0000000000..bedfd01a8a --- /dev/null +++ b/media/libstagefright/wifi-display/rtp/RTPSender.h @@ -0,0 +1,119 @@ +/* + * Copyright 2013, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RTP_SENDER_H_ + +#define RTP_SENDER_H_ + +#include "RTPBase.h" + +#include + +namespace android { + +struct ABuffer; +struct ANetworkSession; + +// An object of this class facilitates sending of media data over an RTP +// channel. The channel is established over a UDP or TCP connection depending +// on which "TransportMode" was chosen. In addition different RTP packetization +// schemes are supported such as "Transport Stream Packets over RTP", +// or "AVC/H.264 encapsulation as specified in RFC 3984 (non-interleaved mode)" +struct RTPSender : public RTPBase, public AHandler { + enum { + kWhatInitDone, + kWhatError, + kWhatNetworkStall, + kWhatInformSender, + }; + RTPSender( + const sp &netSession, + const sp ¬ify); + + status_t initAsync( + const char *remoteHost, + int32_t remoteRTPPort, + TransportMode rtpMode, + int32_t remoteRTCPPort, + TransportMode rtcpMode, + int32_t *outLocalRTPPort); + + status_t queueBuffer( + const sp &buffer, + uint8_t packetType, + PacketizationMode mode); + +protected: + virtual ~RTPSender(); + virtual void onMessageReceived(const sp &msg); + +private: + enum { + kWhatRTPNotify, + kWhatRTCPNotify, + }; + + const unsigned int kMaxNumTSPacketsPerRTPPacket = (kMaxUDPPacketSize - 12) / 188; + const unsigned int kMaxHistorySize = 1024; + const unsigned int kSourceID = 0xdeadbeef; + + sp mNetSession; + sp mNotify; + TransportMode mRTPMode; + TransportMode mRTCPMode; + int32_t mRTPSessionID; + int32_t mRTCPSessionID; + bool mRTPConnected; + bool mRTCPConnected; + + uint64_t mLastNTPTime; + uint32_t mLastRTPTime; + uint32_t mNumRTPSent; + uint32_t mNumRTPOctetsSent; + uint32_t mNumSRsSent; + + uint32_t mRTPSeqNo; + + List > mHistory; + size_t mHistorySize; + + static uint64_t GetNowNTP(); + + status_t queueRawPacket(const sp &tsPackets, uint8_t packetType); + status_t queueTSPackets(const sp &tsPackets, uint8_t packetType); + status_t queueAVCBuffer(const sp &accessUnit, uint8_t packetType); + + status_t sendRTPPacket( + const sp &packet, bool storeInHistory, + bool timeValid = false, int64_t timeUs = -1ll); + + void onNetNotify(bool isRTP, const sp &msg); + + status_t onRTCPData(const sp &data); + status_t parseReceiverReport(const uint8_t *data, size_t size); + status_t parseTSFB(const uint8_t *data, size_t size); + status_t parseAPP(const uint8_t *data, size_t size); + + void notifyInitDone(status_t err); + void notifyError(status_t err); + void notifyNetworkStall(size_t numBytesQueued); + + DISALLOW_EVIL_CONSTRUCTORS(RTPSender); +}; + +} // namespace android + +#endif // RTP_SENDER_H_ diff --git a/media/libstagefright/wifi-display/source/Converter.cpp b/media/libstagefright/wifi-display/source/Converter.cpp new file mode 100644 index 0000000000..273af18d62 --- /dev/null +++ b/media/libstagefright/wifi-display/source/Converter.cpp @@ -0,0 +1,821 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "Converter" +#include + +#include "Converter.h" + +#include "MediaPuller.h" +#include "include/avc_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace android { + +Converter::Converter( + const sp ¬ify, + const sp &codecLooper, + const sp &outputFormat, + uint32_t flags) + : mNotify(notify), + mCodecLooper(codecLooper), + mOutputFormat(outputFormat), + mFlags(flags), + mIsVideo(false), + mIsH264(false), + mIsPCMAudio(false), + mNeedToManuallyPrependSPSPPS(false), + mDoMoreWorkPending(false) +#if ENABLE_SILENCE_DETECTION + ,mFirstSilentFrameUs(-1ll) + ,mInSilentMode(false) +#endif + ,mPrevVideoBitrate(-1) + ,mNumFramesToDrop(0) + ,mEncodingSuspended(false) + { + AString mime; + CHECK(mOutputFormat->findString("mime", &mime)); + + if (!strncasecmp("video/", mime.c_str(), 6)) { + mIsVideo = true; + + mIsH264 = !strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC); + } else if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_RAW, mime.c_str())) { + mIsPCMAudio = true; + } +} + +void Converter::releaseEncoder() { + if (mEncoder == NULL) { + return; + } + + mEncoder->release(); + mEncoder.clear(); + + mInputBufferQueue.clear(); + mEncoderInputBuffers.clear(); + mEncoderOutputBuffers.clear(); +} + +Converter::~Converter() { + CHECK(mEncoder == NULL); +} + +void Converter::shutdownAsync() { + ALOGV("shutdown"); + (new AMessage(kWhatShutdown, this))->post(); +} + +status_t Converter::init() { + status_t err = initEncoder(); + + if (err != OK) { + releaseEncoder(); + } + + return err; +} + +sp Converter::getGraphicBufferProducer() { + CHECK(mFlags & FLAG_USE_SURFACE_INPUT); + return mGraphicBufferProducer; +} + +size_t Converter::getInputBufferCount() const { + return mEncoderInputBuffers.size(); +} + +sp Converter::getOutputFormat() const { + return mOutputFormat; +} + +bool Converter::needToManuallyPrependSPSPPS() const { + return mNeedToManuallyPrependSPSPPS; +} + +// static +int32_t Converter::GetInt32Property( + const char *propName, int32_t defaultValue) { + char val[PROPERTY_VALUE_MAX]; + if (property_get(propName, val, NULL)) { + char *end; + unsigned long x = strtoul(val, &end, 10); + + if (*end == '\0' && end > val && x > 0) { + return x; + } + } + + return defaultValue; +} + +status_t Converter::initEncoder() { + AString outputMIME; + CHECK(mOutputFormat->findString("mime", &outputMIME)); + + bool isAudio = !strncasecmp(outputMIME.c_str(), "audio/", 6); + + if (!mIsPCMAudio) { + mEncoder = MediaCodec::CreateByType( + mCodecLooper, outputMIME.c_str(), true /* encoder */); + + if (mEncoder == NULL) { + return ERROR_UNSUPPORTED; + } + } + + if (mIsPCMAudio) { + return OK; + } + + int32_t audioBitrate = GetInt32Property("media.wfd.audio-bitrate", 128000); + int32_t videoBitrate = GetInt32Property("media.wfd.video-bitrate", 5000000); + mPrevVideoBitrate = videoBitrate; + + ALOGI("using audio bitrate of %d bps, video bitrate of %d bps", + audioBitrate, videoBitrate); + + if (isAudio) { + mOutputFormat->setInt32("bitrate", audioBitrate); + } else { + mOutputFormat->setInt32("bitrate", videoBitrate); + mOutputFormat->setInt32("bitrate-mode", OMX_Video_ControlRateConstant); + mOutputFormat->setInt32("frame-rate", 30); + mOutputFormat->setInt32("i-frame-interval", 15); // Iframes every 15 secs + + // Configure encoder to use intra macroblock refresh mode + mOutputFormat->setInt32("intra-refresh-mode", OMX_VIDEO_IntraRefreshCyclic); + + int width, height, mbs; + if (!mOutputFormat->findInt32("width", &width) + || !mOutputFormat->findInt32("height", &height)) { + return ERROR_UNSUPPORTED; + } + + // Update macroblocks in a cyclic fashion with 10% of all MBs within + // frame gets updated at one time. It takes about 10 frames to + // completely update a whole video frame. If the frame rate is 30, + // it takes about 333 ms in the best case (if next frame is not an IDR) + // to recover from a lost/corrupted packet. + mbs = (((width + 15) / 16) * ((height + 15) / 16) * 10) / 100; + mOutputFormat->setInt32("intra-refresh-CIR-mbs", mbs); + } + + ALOGV("output format is '%s'", mOutputFormat->debugString(0).c_str()); + + mNeedToManuallyPrependSPSPPS = false; + + status_t err = NO_INIT; + + if (!isAudio) { + sp tmp = mOutputFormat->dup(); + tmp->setInt32("prepend-sps-pps-to-idr-frames", 1); + + err = mEncoder->configure( + tmp, + NULL /* nativeWindow */, + NULL /* crypto */, + MediaCodec::CONFIGURE_FLAG_ENCODE); + + if (err == OK) { + // Encoder supported prepending SPS/PPS, we don't need to emulate + // it. + mOutputFormat = tmp; + } else { + mNeedToManuallyPrependSPSPPS = true; + + ALOGI("We going to manually prepend SPS and PPS to IDR frames."); + } + } + + if (err != OK) { + // We'll get here for audio or if we failed to configure the encoder + // to automatically prepend SPS/PPS in the case of video. + + err = mEncoder->configure( + mOutputFormat, + NULL /* nativeWindow */, + NULL /* crypto */, + MediaCodec::CONFIGURE_FLAG_ENCODE); + } + + if (err != OK) { + return err; + } + + if (mFlags & FLAG_USE_SURFACE_INPUT) { + CHECK(mIsVideo); + + err = mEncoder->createInputSurface(&mGraphicBufferProducer); + + if (err != OK) { + return err; + } + } + + err = mEncoder->start(); + + if (err != OK) { + return err; + } + + err = mEncoder->getInputBuffers(&mEncoderInputBuffers); + + if (err != OK) { + return err; + } + + err = mEncoder->getOutputBuffers(&mEncoderOutputBuffers); + + if (err != OK) { + return err; + } + + if (mFlags & FLAG_USE_SURFACE_INPUT) { + scheduleDoMoreWork(); + } + + return OK; +} + +void Converter::notifyError(status_t err) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatError); + notify->setInt32("err", err); + notify->post(); +} + +// static +bool Converter::IsSilence(const sp &accessUnit) { + const uint8_t *ptr = accessUnit->data(); + const uint8_t *end = ptr + accessUnit->size(); + while (ptr < end) { + if (*ptr != 0) { + return false; + } + ++ptr; + } + + return true; +} + +void Converter::onMessageReceived(const sp &msg) { + switch (msg->what()) { + case kWhatMediaPullerNotify: + { + int32_t what; + CHECK(msg->findInt32("what", &what)); + + if (!mIsPCMAudio && mEncoder == NULL) { + ALOGV("got msg '%s' after encoder shutdown.", + msg->debugString().c_str()); + + if (what == MediaPuller::kWhatAccessUnit) { + sp accessUnit; + CHECK(msg->findBuffer("accessUnit", &accessUnit)); + + accessUnit->setMediaBufferBase(NULL); + } + break; + } + + if (what == MediaPuller::kWhatEOS) { + mInputBufferQueue.push_back(NULL); + + feedEncoderInputBuffers(); + + scheduleDoMoreWork(); + } else { + CHECK_EQ(what, MediaPuller::kWhatAccessUnit); + + sp accessUnit; + CHECK(msg->findBuffer("accessUnit", &accessUnit)); + + if (mNumFramesToDrop > 0 || mEncodingSuspended) { + if (mNumFramesToDrop > 0) { + --mNumFramesToDrop; + ALOGI("dropping frame."); + } + + accessUnit->setMediaBufferBase(NULL); + break; + } + +#if 0 + MediaBuffer *mbuf = + (MediaBuffer *)(accessUnit->getMediaBufferBase()); + if (mbuf != NULL) { + ALOGI("queueing mbuf %p", mbuf); + mbuf->release(); + } +#endif + +#if ENABLE_SILENCE_DETECTION + if (!mIsVideo) { + if (IsSilence(accessUnit)) { + if (mInSilentMode) { + break; + } + + int64_t nowUs = ALooper::GetNowUs(); + + if (mFirstSilentFrameUs < 0ll) { + mFirstSilentFrameUs = nowUs; + } else if (nowUs >= mFirstSilentFrameUs + 10000000ll) { + mInSilentMode = true; + ALOGI("audio in silent mode now."); + break; + } + } else { + if (mInSilentMode) { + ALOGI("audio no longer in silent mode."); + } + mInSilentMode = false; + mFirstSilentFrameUs = -1ll; + } + } +#endif + + mInputBufferQueue.push_back(accessUnit); + + feedEncoderInputBuffers(); + + scheduleDoMoreWork(); + } + break; + } + + case kWhatEncoderActivity: + { +#if 0 + int64_t whenUs; + if (msg->findInt64("whenUs", &whenUs)) { + int64_t nowUs = ALooper::GetNowUs(); + ALOGI("[%s] kWhatEncoderActivity after %lld us", + mIsVideo ? "video" : "audio", nowUs - whenUs); + } +#endif + + mDoMoreWorkPending = false; + + if (mEncoder == NULL) { + break; + } + + status_t err = doMoreWork(); + + if (err != OK) { + notifyError(err); + } else { + scheduleDoMoreWork(); + } + break; + } + + case kWhatRequestIDRFrame: + { + if (mEncoder == NULL) { + break; + } + + if (mIsVideo) { + ALOGV("requesting IDR frame"); + mEncoder->requestIDRFrame(); + } + break; + } + + case kWhatShutdown: + { + ALOGI("shutting down %s encoder", mIsVideo ? "video" : "audio"); + + releaseEncoder(); + + AString mime; + CHECK(mOutputFormat->findString("mime", &mime)); + ALOGI("encoder (%s) shut down.", mime.c_str()); + + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatShutdownCompleted); + notify->post(); + break; + } + + case kWhatDropAFrame: + { + ++mNumFramesToDrop; + break; + } + + case kWhatReleaseOutputBuffer: + { + if (mEncoder != NULL) { + size_t bufferIndex; + CHECK(msg->findInt32("bufferIndex", (int32_t*)&bufferIndex)); + CHECK(bufferIndex < mEncoderOutputBuffers.size()); + mEncoder->releaseOutputBuffer(bufferIndex); + } + break; + } + + case kWhatSuspendEncoding: + { + int32_t suspend; + CHECK(msg->findInt32("suspend", &suspend)); + + mEncodingSuspended = suspend; + + if (mFlags & FLAG_USE_SURFACE_INPUT) { + sp params = new AMessage; + params->setInt32("drop-input-frames",suspend); + mEncoder->setParameters(params); + } + break; + } + + default: + TRESPASS(); + } +} + +void Converter::scheduleDoMoreWork() { + if (mIsPCMAudio) { + // There's no encoder involved in this case. + return; + } + + if (mDoMoreWorkPending) { + return; + } + + mDoMoreWorkPending = true; + +#if 1 + if (mEncoderActivityNotify == NULL) { + mEncoderActivityNotify = new AMessage(kWhatEncoderActivity, this); + } + mEncoder->requestActivityNotification(mEncoderActivityNotify->dup()); +#else + sp notify = new AMessage(kWhatEncoderActivity, this); + notify->setInt64("whenUs", ALooper::GetNowUs()); + mEncoder->requestActivityNotification(notify); +#endif +} + +status_t Converter::feedRawAudioInputBuffers() { + // Split incoming PCM audio into buffers of 6 AUs of 80 audio frames each + // and add a 4 byte header according to the wifi display specs. + + while (!mInputBufferQueue.empty()) { + sp buffer = *mInputBufferQueue.begin(); + mInputBufferQueue.erase(mInputBufferQueue.begin()); + + int16_t *ptr = (int16_t *)buffer->data(); + int16_t *stop = (int16_t *)(buffer->data() + buffer->size()); + while (ptr < stop) { + *ptr = htons(*ptr); + ++ptr; + } + + static const size_t kFrameSize = 2 * sizeof(int16_t); // stereo + static const size_t kFramesPerAU = 80; + static const size_t kNumAUsPerPESPacket = 6; + + if (mPartialAudioAU != NULL) { + size_t bytesMissingForFullAU = + kNumAUsPerPESPacket * kFramesPerAU * kFrameSize + - mPartialAudioAU->size() + 4; + + size_t copy = buffer->size(); + if(copy > bytesMissingForFullAU) { + copy = bytesMissingForFullAU; + } + + memcpy(mPartialAudioAU->data() + mPartialAudioAU->size(), + buffer->data(), + copy); + + mPartialAudioAU->setRange(0, mPartialAudioAU->size() + copy); + + buffer->setRange(buffer->offset() + copy, buffer->size() - copy); + + int64_t timeUs; + CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); + + int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0); + timeUs += copyUs; + buffer->meta()->setInt64("timeUs", timeUs); + + if (bytesMissingForFullAU == copy) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatAccessUnit); + notify->setBuffer("accessUnit", mPartialAudioAU); + notify->post(); + + mPartialAudioAU.clear(); + } + } + + while (buffer->size() > 0) { + sp partialAudioAU = + new ABuffer( + 4 + + kNumAUsPerPESPacket * kFrameSize * kFramesPerAU); + + uint8_t *ptr = partialAudioAU->data(); + ptr[0] = 0xa0; // 10100000b + ptr[1] = kNumAUsPerPESPacket; + ptr[2] = 0; // reserved, audio _emphasis_flag = 0 + + static const unsigned kQuantizationWordLength = 0; // 16-bit + static const unsigned kAudioSamplingFrequency = 2; // 48Khz + static const unsigned kNumberOfAudioChannels = 1; // stereo + + ptr[3] = (kQuantizationWordLength << 6) + | (kAudioSamplingFrequency << 3) + | kNumberOfAudioChannels; + + size_t copy = buffer->size(); + if (copy > partialAudioAU->size() - 4) { + copy = partialAudioAU->size() - 4; + } + + memcpy(&ptr[4], buffer->data(), copy); + + partialAudioAU->setRange(0, 4 + copy); + buffer->setRange(buffer->offset() + copy, buffer->size() - copy); + + int64_t timeUs; + CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); + + partialAudioAU->meta()->setInt64("timeUs", timeUs); + + int64_t copyUs = (int64_t)((copy / kFrameSize) * 1E6 / 48000.0); + timeUs += copyUs; + buffer->meta()->setInt64("timeUs", timeUs); + + if (copy == partialAudioAU->capacity() - 4) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatAccessUnit); + notify->setBuffer("accessUnit", partialAudioAU); + notify->post(); + + partialAudioAU.clear(); + continue; + } + + mPartialAudioAU = partialAudioAU; + } + } + + return OK; +} + +status_t Converter::feedEncoderInputBuffers() { + if (mIsPCMAudio) { + return feedRawAudioInputBuffers(); + } + + while (!mInputBufferQueue.empty() + && !mAvailEncoderInputIndices.empty()) { + sp buffer = *mInputBufferQueue.begin(); + mInputBufferQueue.erase(mInputBufferQueue.begin()); + + size_t bufferIndex = *mAvailEncoderInputIndices.begin(); + mAvailEncoderInputIndices.erase(mAvailEncoderInputIndices.begin()); + + int64_t timeUs = 0ll; + uint32_t flags = 0; + + if (buffer != NULL) { + CHECK(buffer->meta()->findInt64("timeUs", &timeUs)); + + memcpy(mEncoderInputBuffers.itemAt(bufferIndex)->data(), + buffer->data(), + buffer->size()); + + MediaBuffer *mediaBuffer = + (MediaBuffer *)(buffer->getMediaBufferBase()); + if (mediaBuffer != NULL) { + mEncoderInputBuffers.itemAt(bufferIndex)->setMediaBufferBase( + mediaBuffer); + + buffer->setMediaBufferBase(NULL); + } + } else { + flags = MediaCodec::BUFFER_FLAG_EOS; + } + + status_t err = mEncoder->queueInputBuffer( + bufferIndex, 0, (buffer == NULL) ? 0 : buffer->size(), + timeUs, flags); + + if (err != OK) { + return err; + } + } + + return OK; +} + +sp Converter::prependCSD(const sp &accessUnit) const { + CHECK(mCSD0 != NULL); + + sp dup = new ABuffer(accessUnit->size() + mCSD0->size()); + memcpy(dup->data(), mCSD0->data(), mCSD0->size()); + memcpy(dup->data() + mCSD0->size(), accessUnit->data(), accessUnit->size()); + + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + dup->meta()->setInt64("timeUs", timeUs); + + return dup; +} + +status_t Converter::doMoreWork() { + status_t err; + + if (!(mFlags & FLAG_USE_SURFACE_INPUT)) { + for (;;) { + size_t bufferIndex; + err = mEncoder->dequeueInputBuffer(&bufferIndex); + + if (err != OK) { + break; + } + + mAvailEncoderInputIndices.push_back(bufferIndex); + } + + feedEncoderInputBuffers(); + } + + for (;;) { + size_t bufferIndex; + size_t offset; + size_t size; + int64_t timeUs; + uint32_t flags; + native_handle_t* handle = NULL; + err = mEncoder->dequeueOutputBuffer( + &bufferIndex, &offset, &size, &timeUs, &flags); + + if (err != OK) { + if (err == INFO_FORMAT_CHANGED) { + continue; + } else if (err == INFO_OUTPUT_BUFFERS_CHANGED) { + mEncoder->getOutputBuffers(&mEncoderOutputBuffers); + continue; + } + + if (err == -EAGAIN) { + err = OK; + } + break; + } + + if (flags & MediaCodec::BUFFER_FLAG_EOS) { + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatEOS); + notify->post(); + } else { +#if 0 + if (mIsVideo) { + int32_t videoBitrate = GetInt32Property( + "media.wfd.video-bitrate", 5000000); + + setVideoBitrate(videoBitrate); + } +#endif + + sp buffer; + sp outbuf = mEncoderOutputBuffers.itemAt(bufferIndex); + + if (outbuf->meta()->findPointer("handle", (void**)&handle) && + handle != NULL) { + int32_t rangeLength, rangeOffset; + CHECK(outbuf->meta()->findInt32("rangeOffset", &rangeOffset)); + CHECK(outbuf->meta()->findInt32("rangeLength", &rangeLength)); + outbuf->meta()->setPointer("handle", NULL); + + // MediaSender will post the following message when HDCP + // is done, to release the output buffer back to encoder. + sp notify(new AMessage(kWhatReleaseOutputBuffer, this)); + notify->setInt32("bufferIndex", bufferIndex); + + buffer = new ABuffer( + rangeLength > (int32_t)size ? rangeLength : size); + buffer->meta()->setPointer("handle", handle); + buffer->meta()->setInt32("rangeOffset", rangeOffset); + buffer->meta()->setInt32("rangeLength", rangeLength); + buffer->meta()->setMessage("notify", notify); + } else { + buffer = new ABuffer(size); + } + + buffer->meta()->setInt64("timeUs", timeUs); + + ALOGV("[%s] time %lld us (%.2f secs)", + mIsVideo ? "video" : "audio", (long long)timeUs, timeUs / 1E6); + + memcpy(buffer->data(), outbuf->base() + offset, size); + + if (flags & MediaCodec::BUFFER_FLAG_CODECCONFIG) { + if (!handle) { + if (mIsH264) { + mCSD0 = buffer; + } + mOutputFormat->setBuffer("csd-0", buffer); + } + } else { + if (mNeedToManuallyPrependSPSPPS + && mIsH264 + && (mFlags & FLAG_PREPEND_CSD_IF_NECESSARY) + && IsIDR(buffer)) { + buffer = prependCSD(buffer); + } + + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatAccessUnit); + notify->setBuffer("accessUnit", buffer); + notify->post(); + } + } + + if (!handle) { + mEncoder->releaseOutputBuffer(bufferIndex); + } + + if (flags & MediaCodec::BUFFER_FLAG_EOS) { + break; + } + } + + return err; +} + +void Converter::requestIDRFrame() { + (new AMessage(kWhatRequestIDRFrame, this))->post(); +} + +void Converter::dropAFrame() { + // Unsupported in surface input mode. + CHECK(!(mFlags & FLAG_USE_SURFACE_INPUT)); + + (new AMessage(kWhatDropAFrame, this))->post(); +} + +void Converter::suspendEncoding(bool suspend) { + sp msg = new AMessage(kWhatSuspendEncoding, this); + msg->setInt32("suspend", suspend); + msg->post(); +} + +int32_t Converter::getVideoBitrate() const { + return mPrevVideoBitrate; +} + +void Converter::setVideoBitrate(int32_t bitRate) { + if (mIsVideo && mEncoder != NULL && bitRate != mPrevVideoBitrate) { + sp params = new AMessage; + params->setInt32("video-bitrate", bitRate); + + mEncoder->setParameters(params); + + mPrevVideoBitrate = bitRate; + } +} + +} // namespace android diff --git a/media/libstagefright/wifi-display/source/Converter.h b/media/libstagefright/wifi-display/source/Converter.h new file mode 100644 index 0000000000..ad95ab5ce9 --- /dev/null +++ b/media/libstagefright/wifi-display/source/Converter.h @@ -0,0 +1,157 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CONVERTER_H_ + +#define CONVERTER_H_ + +#include + +namespace android { + +struct ABuffer; +class IGraphicBufferProducer; +struct MediaCodec; +class MediaCodecBuffer; + +#define ENABLE_SILENCE_DETECTION 0 + +// Utility class that receives media access units and converts them into +// media access unit of a different format. +// Right now this'll convert raw video into H.264 and raw audio into AAC. +struct Converter : public AHandler { + enum { + kWhatAccessUnit, + kWhatEOS, + kWhatError, + kWhatShutdownCompleted, + }; + + enum FlagBits { + FLAG_USE_SURFACE_INPUT = 1, + FLAG_PREPEND_CSD_IF_NECESSARY = 2, + }; + Converter(const sp ¬ify, + const sp &codecLooper, + const sp &outputFormat, + uint32_t flags = 0); + + status_t init(); + + sp getGraphicBufferProducer(); + + size_t getInputBufferCount() const; + + sp getOutputFormat() const; + bool needToManuallyPrependSPSPPS() const; + + void feedAccessUnit(const sp &accessUnit); + void signalEOS(); + + void requestIDRFrame(); + + void dropAFrame(); + void suspendEncoding(bool suspend); + + void shutdownAsync(); + + int32_t getVideoBitrate() const; + void setVideoBitrate(int32_t bitrate); + + static int32_t GetInt32Property(const char *propName, int32_t defaultValue); + + enum { + // MUST not conflict with private enums below. + kWhatMediaPullerNotify = 'pulN', + }; + +protected: + virtual ~Converter(); + virtual void onMessageReceived(const sp &msg); + +private: + enum { + kWhatDoMoreWork, + kWhatRequestIDRFrame, + kWhatSuspendEncoding, + kWhatShutdown, + kWhatEncoderActivity, + kWhatDropAFrame, + kWhatReleaseOutputBuffer, + }; + + sp mNotify; + sp mCodecLooper; + sp mOutputFormat; + uint32_t mFlags; + bool mIsVideo; + bool mIsH264; + bool mIsPCMAudio; + bool mNeedToManuallyPrependSPSPPS; + + sp mEncoder; + sp mEncoderActivityNotify; + + sp mGraphicBufferProducer; + + Vector > mEncoderInputBuffers; + Vector > mEncoderOutputBuffers; + + List mAvailEncoderInputIndices; + + List > mInputBufferQueue; + + sp mCSD0; + + bool mDoMoreWorkPending; + +#if ENABLE_SILENCE_DETECTION + int64_t mFirstSilentFrameUs; + bool mInSilentMode; +#endif + + sp mPartialAudioAU; + + int32_t mPrevVideoBitrate; + + int32_t mNumFramesToDrop; + bool mEncodingSuspended; + + status_t initEncoder(); + void releaseEncoder(); + + status_t feedEncoderInputBuffers(); + + void scheduleDoMoreWork(); + status_t doMoreWork(); + + void notifyError(status_t err); + + // Packetizes raw PCM audio data available in mInputBufferQueue + // into a format suitable for transport stream inclusion and + // notifies the observer. + status_t feedRawAudioInputBuffers(); + + static bool IsSilence(const sp &accessUnit); + + sp prependCSD(const sp &accessUnit) const; + + DISALLOW_EVIL_CONSTRUCTORS(Converter); +}; + +} // namespace android + +#endif // CONVERTER_H_ diff --git a/media/libstagefright/wifi-display/source/MediaPuller.cpp b/media/libstagefright/wifi-display/source/MediaPuller.cpp new file mode 100644 index 0000000000..ce07a4ec4c --- /dev/null +++ b/media/libstagefright/wifi-display/source/MediaPuller.cpp @@ -0,0 +1,224 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "MediaPuller" +#include + +#include "MediaPuller.h" + +#include +#include +#include +#include +#include +#include + +namespace android { + +MediaPuller::MediaPuller( + const sp &source, const sp ¬ify) + : mSource(source), + mNotify(notify), + mPullGeneration(0), + mIsAudio(false), + mPaused(false) { + sp meta = source->getFormat(); + const char *mime; + CHECK(meta->findCString(kKeyMIMEType, &mime)); + + mIsAudio = !strncasecmp(mime, "audio/", 6); +} + +MediaPuller::~MediaPuller() { +} + +status_t MediaPuller::postSynchronouslyAndReturnError( + const sp &msg) { + sp response; + status_t err = msg->postAndAwaitResponse(&response); + + if (err != OK) { + return err; + } + + if (!response->findInt32("err", &err)) { + err = OK; + } + + return err; +} + +status_t MediaPuller::start() { + return postSynchronouslyAndReturnError(new AMessage(kWhatStart, this)); +} + +void MediaPuller::stopAsync(const sp ¬ify) { + sp msg = new AMessage(kWhatStop, this); + msg->setMessage("notify", notify); + msg->post(); +} + +void MediaPuller::pause() { + (new AMessage(kWhatPause, this))->post(); +} + +void MediaPuller::resume() { + (new AMessage(kWhatResume, this))->post(); +} + +void MediaPuller::onMessageReceived(const sp &msg) { + switch (msg->what()) { + case kWhatStart: + { + status_t err; + if (mIsAudio) { + // This atrocity causes AudioSource to deliver absolute + // systemTime() based timestamps (off by 1 us). + sp params = new MetaData; + params->setInt64(kKeyTime, 1ll); + err = mSource->start(params.get()); + } else { + err = mSource->start(); + if (err != OK) { + ALOGE("source failed to start w/ err %d", err); + } + } + + if (err == OK) { + schedulePull(); + } + + sp response = new AMessage; + response->setInt32("err", err); + + sp replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + response->postReply(replyID); + break; + } + + case kWhatStop: + { + sp meta = mSource->getFormat(); + const char *tmp; + CHECK(meta->findCString(kKeyMIMEType, &tmp)); + AString mime = tmp; + + ALOGI("MediaPuller(%s) stopping.", mime.c_str()); + mSource->stop(); + ALOGI("MediaPuller(%s) stopped.", mime.c_str()); + ++mPullGeneration; + + sp notify; + CHECK(msg->findMessage("notify", ¬ify)); + notify->post(); + break; + } + + case kWhatPull: + { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + + if (generation != mPullGeneration) { + break; + } + + MediaBuffer *mbuf; + status_t err = mSource->read(&mbuf); + + if (mPaused) { + if (err == OK) { + mbuf->release(); + mbuf = NULL; + } + + schedulePull(); + break; + } + + if (err != OK) { + if (err == ERROR_END_OF_STREAM) { + ALOGI("stream ended."); + } else { + ALOGE("error %d reading stream.", err); + } + + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatEOS); + notify->post(); + } else { + int64_t timeUs; + CHECK(mbuf->meta_data()->findInt64(kKeyTime, &timeUs)); + + sp accessUnit = new ABuffer(mbuf->range_length()); + + memcpy(accessUnit->data(), + (const uint8_t *)mbuf->data() + mbuf->range_offset(), + mbuf->range_length()); + + accessUnit->meta()->setInt64("timeUs", timeUs); + + if (mIsAudio) { + mbuf->release(); + mbuf = NULL; + } else { + // video encoder will release MediaBuffer when done + // with underlying data. + accessUnit->setMediaBufferBase(mbuf); + } + + sp notify = mNotify->dup(); + + notify->setInt32("what", kWhatAccessUnit); + notify->setBuffer("accessUnit", accessUnit); + notify->post(); + + if (mbuf != NULL) { + ALOGV("posted mbuf %p", mbuf); + } + + schedulePull(); + } + break; + } + + case kWhatPause: + { + mPaused = true; + break; + } + + case kWhatResume: + { + mPaused = false; + break; + } + + default: + TRESPASS(); + } +} + +void MediaPuller::schedulePull() { + sp msg = new AMessage(kWhatPull, this); + msg->setInt32("generation", mPullGeneration); + msg->post(); +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/source/MediaPuller.h b/media/libstagefright/wifi-display/source/MediaPuller.h new file mode 100644 index 0000000000..1291bb3f9e --- /dev/null +++ b/media/libstagefright/wifi-display/source/MediaPuller.h @@ -0,0 +1,68 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MEDIA_PULLER_H_ + +#define MEDIA_PULLER_H_ + +#include + +namespace android { + +struct MediaSource; + +struct MediaPuller : public AHandler { + enum { + kWhatEOS, + kWhatAccessUnit + }; + + MediaPuller(const sp &source, const sp ¬ify); + + status_t start(); + void stopAsync(const sp ¬ify); + + void pause(); + void resume(); + +protected: + virtual void onMessageReceived(const sp &msg); + virtual ~MediaPuller(); + +private: + enum { + kWhatStart, + kWhatStop, + kWhatPull, + kWhatPause, + kWhatResume, + }; + + sp mSource; + sp mNotify; + int32_t mPullGeneration; + bool mIsAudio; + bool mPaused; + + status_t postSynchronouslyAndReturnError(const sp &msg); + void schedulePull(); + + DISALLOW_EVIL_CONSTRUCTORS(MediaPuller); +}; + +} // namespace android + +#endif // MEDIA_PULLER_H_ diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.cpp b/media/libstagefright/wifi-display/source/PlaybackSession.cpp new file mode 100644 index 0000000000..f1ecca0515 --- /dev/null +++ b/media/libstagefright/wifi-display/source/PlaybackSession.cpp @@ -0,0 +1,1112 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "PlaybackSession" +#include + +#include "PlaybackSession.h" + +#include "Converter.h" +#include "MediaPuller.h" +#include "RepeaterSource.h" +#include "include/avc_utils.h" +#include "WifiDisplaySource.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace android { + +struct WifiDisplaySource::PlaybackSession::Track : public AHandler { + enum { + kWhatStopped, + }; + + Track(const sp ¬ify, + const sp &pullLooper, + const sp &codecLooper, + const sp &mediaPuller, + const sp &converter); + + Track(const sp ¬ify, const sp &format); + + void setRepeaterSource(const sp &source); + + sp getFormat(); + bool isAudio() const; + + const sp &converter() const; + const sp &repeaterSource() const; + + ssize_t mediaSenderTrackIndex() const; + void setMediaSenderTrackIndex(size_t index); + + status_t start(); + void stopAsync(); + + void pause(); + void resume(); + + void queueAccessUnit(const sp &accessUnit); + sp dequeueAccessUnit(); + + bool hasOutputBuffer(int64_t *timeUs) const; + void queueOutputBuffer(const sp &accessUnit); + sp dequeueOutputBuffer(); + +#if SUSPEND_VIDEO_IF_IDLE + bool isSuspended() const; +#endif + + size_t countQueuedOutputBuffers() const { + return mQueuedOutputBuffers.size(); + } + + void requestIDRFrame(); + +protected: + virtual void onMessageReceived(const sp &msg); + virtual ~Track(); + +private: + enum { + kWhatMediaPullerStopped, + }; + + sp mNotify; + sp mPullLooper; + sp mCodecLooper; + sp mMediaPuller; + sp mConverter; + sp mFormat; + bool mStarted; + ssize_t mMediaSenderTrackIndex; + bool mIsAudio; + List > mQueuedAccessUnits; + sp mRepeaterSource; + List > mQueuedOutputBuffers; + int64_t mLastOutputBufferQueuedTimeUs; + + static bool IsAudioFormat(const sp &format); + + DISALLOW_EVIL_CONSTRUCTORS(Track); +}; + +WifiDisplaySource::PlaybackSession::Track::Track( + const sp ¬ify, + const sp &pullLooper, + const sp &codecLooper, + const sp &mediaPuller, + const sp &converter) + : mNotify(notify), + mPullLooper(pullLooper), + mCodecLooper(codecLooper), + mMediaPuller(mediaPuller), + mConverter(converter), + mStarted(false), + mIsAudio(IsAudioFormat(mConverter->getOutputFormat())), + mLastOutputBufferQueuedTimeUs(-1ll) { +} + +WifiDisplaySource::PlaybackSession::Track::Track( + const sp ¬ify, const sp &format) + : mNotify(notify), + mFormat(format), + mStarted(false), + mIsAudio(IsAudioFormat(format)), + mLastOutputBufferQueuedTimeUs(-1ll) { +} + +WifiDisplaySource::PlaybackSession::Track::~Track() { + CHECK(!mStarted); +} + +// static +bool WifiDisplaySource::PlaybackSession::Track::IsAudioFormat( + const sp &format) { + AString mime; + CHECK(format->findString("mime", &mime)); + + return !strncasecmp(mime.c_str(), "audio/", 6); +} + +sp WifiDisplaySource::PlaybackSession::Track::getFormat() { + return mFormat != NULL ? mFormat : mConverter->getOutputFormat(); +} + +bool WifiDisplaySource::PlaybackSession::Track::isAudio() const { + return mIsAudio; +} + +const sp &WifiDisplaySource::PlaybackSession::Track::converter() const { + return mConverter; +} + +const sp & +WifiDisplaySource::PlaybackSession::Track::repeaterSource() const { + return mRepeaterSource; +} + +ssize_t WifiDisplaySource::PlaybackSession::Track::mediaSenderTrackIndex() const { + CHECK_GE(mMediaSenderTrackIndex, 0); + return mMediaSenderTrackIndex; +} + +void WifiDisplaySource::PlaybackSession::Track::setMediaSenderTrackIndex( + size_t index) { + mMediaSenderTrackIndex = index; +} + +status_t WifiDisplaySource::PlaybackSession::Track::start() { + ALOGV("Track::start isAudio=%d", mIsAudio); + + CHECK(!mStarted); + + status_t err = OK; + + if (mMediaPuller != NULL) { + err = mMediaPuller->start(); + } + + if (err == OK) { + mStarted = true; + } + + return err; +} + +void WifiDisplaySource::PlaybackSession::Track::stopAsync() { + ALOGV("Track::stopAsync isAudio=%d", mIsAudio); + + if (mConverter != NULL) { + mConverter->shutdownAsync(); + } + + sp msg = new AMessage(kWhatMediaPullerStopped, this); + + if (mStarted && mMediaPuller != NULL) { + if (mRepeaterSource != NULL) { + // Let's unblock MediaPuller's MediaSource::read(). + mRepeaterSource->wakeUp(); + } + + mMediaPuller->stopAsync(msg); + } else { + mStarted = false; + msg->post(); + } +} + +void WifiDisplaySource::PlaybackSession::Track::pause() { + mMediaPuller->pause(); +} + +void WifiDisplaySource::PlaybackSession::Track::resume() { + mMediaPuller->resume(); +} + +void WifiDisplaySource::PlaybackSession::Track::onMessageReceived( + const sp &msg) { + switch (msg->what()) { + case kWhatMediaPullerStopped: + { + mConverter.clear(); + + mStarted = false; + + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatStopped); + notify->post(); + + ALOGI("kWhatStopped %s posted", mIsAudio ? "audio" : "video"); + break; + } + + default: + TRESPASS(); + } +} + +void WifiDisplaySource::PlaybackSession::Track::queueAccessUnit( + const sp &accessUnit) { + mQueuedAccessUnits.push_back(accessUnit); +} + +sp WifiDisplaySource::PlaybackSession::Track::dequeueAccessUnit() { + if (mQueuedAccessUnits.empty()) { + return NULL; + } + + sp accessUnit = *mQueuedAccessUnits.begin(); + CHECK(accessUnit != NULL); + + mQueuedAccessUnits.erase(mQueuedAccessUnits.begin()); + + return accessUnit; +} + +void WifiDisplaySource::PlaybackSession::Track::setRepeaterSource( + const sp &source) { + mRepeaterSource = source; +} + +void WifiDisplaySource::PlaybackSession::Track::requestIDRFrame() { + if (mIsAudio) { + return; + } + + if (mRepeaterSource != NULL) { + mRepeaterSource->wakeUp(); + } + + mConverter->requestIDRFrame(); +} + +bool WifiDisplaySource::PlaybackSession::Track::hasOutputBuffer( + int64_t *timeUs) const { + *timeUs = 0ll; + + if (mQueuedOutputBuffers.empty()) { + return false; + } + + const sp &outputBuffer = *mQueuedOutputBuffers.begin(); + + CHECK(outputBuffer->meta()->findInt64("timeUs", timeUs)); + + return true; +} + +void WifiDisplaySource::PlaybackSession::Track::queueOutputBuffer( + const sp &accessUnit) { + mQueuedOutputBuffers.push_back(accessUnit); + mLastOutputBufferQueuedTimeUs = ALooper::GetNowUs(); +} + +sp WifiDisplaySource::PlaybackSession::Track::dequeueOutputBuffer() { + CHECK(!mQueuedOutputBuffers.empty()); + + sp outputBuffer = *mQueuedOutputBuffers.begin(); + mQueuedOutputBuffers.erase(mQueuedOutputBuffers.begin()); + + return outputBuffer; +} + +#if SUSPEND_VIDEO_IF_IDLE +bool WifiDisplaySource::PlaybackSession::Track::isSuspended() const { + if (!mQueuedOutputBuffers.empty()) { + return false; + } + + if (mLastOutputBufferQueuedTimeUs < 0ll) { + // We've never seen an output buffer queued, but tracks start + // out live, not suspended. + return false; + } + + // If we've not seen new output data for 60ms or more, we consider + // this track suspended for the time being. + return (ALooper::GetNowUs() - mLastOutputBufferQueuedTimeUs) > 60000ll; +} +#endif + +//////////////////////////////////////////////////////////////////////////////// + +WifiDisplaySource::PlaybackSession::PlaybackSession( + const String16 &opPackageName, + const sp &netSession, + const sp ¬ify, + const in_addr &interfaceAddr, + const sp &hdcp, + const char *path) + : mOpPackageName(opPackageName), + mNetSession(netSession), + mNotify(notify), + mInterfaceAddr(interfaceAddr), + mHDCP(hdcp), + mLocalRTPPort(-1), + mWeAreDead(false), + mPaused(false), + mLastLifesignUs(), + mVideoTrackIndex(-1), + mPrevTimeUs(-1ll), + mPullExtractorPending(false), + mPullExtractorGeneration(0), + mFirstSampleTimeRealUs(-1ll), + mFirstSampleTimeUs(-1ll) { + if (path != NULL) { + mMediaPath.setTo(path); + } +} + +status_t WifiDisplaySource::PlaybackSession::init( + const char *clientIP, + int32_t clientRtp, + RTPSender::TransportMode rtpMode, + int32_t clientRtcp, + RTPSender::TransportMode rtcpMode, + bool enableAudio, + bool usePCMAudio, + bool enableVideo, + VideoFormats::ResolutionType videoResolutionType, + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType) { + sp notify = new AMessage(kWhatMediaSenderNotify, this); + mMediaSender = new MediaSender(mNetSession, notify); + looper()->registerHandler(mMediaSender); + + mMediaSender->setHDCP(mHDCP); + + status_t err = setupPacketizer( + enableAudio, + usePCMAudio, + enableVideo, + videoResolutionType, + videoResolutionIndex, + videoProfileType, + videoLevelType); + + if (err == OK) { + err = mMediaSender->initAsync( + -1 /* trackIndex */, + clientIP, + clientRtp, + rtpMode, + clientRtcp, + rtcpMode, + &mLocalRTPPort); + } + + if (err != OK) { + mLocalRTPPort = -1; + + looper()->unregisterHandler(mMediaSender->id()); + mMediaSender.clear(); + + return err; + } + + updateLiveness(); + + return OK; +} + +WifiDisplaySource::PlaybackSession::~PlaybackSession() { +} + +int32_t WifiDisplaySource::PlaybackSession::getRTPPort() const { + return mLocalRTPPort; +} + +int64_t WifiDisplaySource::PlaybackSession::getLastLifesignUs() const { + return mLastLifesignUs; +} + +void WifiDisplaySource::PlaybackSession::updateLiveness() { + mLastLifesignUs = ALooper::GetNowUs(); +} + +status_t WifiDisplaySource::PlaybackSession::play() { + updateLiveness(); + + (new AMessage(kWhatResume, this))->post(); + + return OK; +} + +status_t WifiDisplaySource::PlaybackSession::onMediaSenderInitialized() { + for (size_t i = 0; i < mTracks.size(); ++i) { + CHECK_EQ((status_t)OK, mTracks.editValueAt(i)->start()); + } + + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatSessionEstablished); + notify->post(); + + return OK; +} + +status_t WifiDisplaySource::PlaybackSession::pause() { + updateLiveness(); + + (new AMessage(kWhatPause, this))->post(); + + return OK; +} + +void WifiDisplaySource::PlaybackSession::destroyAsync() { + ALOGI("destroyAsync"); + + for (size_t i = 0; i < mTracks.size(); ++i) { + mTracks.valueAt(i)->stopAsync(); + } +} + +void WifiDisplaySource::PlaybackSession::onMessageReceived( + const sp &msg) { + switch (msg->what()) { + case kWhatConverterNotify: + { + if (mWeAreDead) { + ALOGV("dropping msg '%s' because we're dead", + msg->debugString().c_str()); + + break; + } + + int32_t what; + CHECK(msg->findInt32("what", &what)); + + size_t trackIndex; + CHECK(msg->findSize("trackIndex", &trackIndex)); + + if (what == Converter::kWhatAccessUnit) { + sp accessUnit; + CHECK(msg->findBuffer("accessUnit", &accessUnit)); + + const sp &track = mTracks.valueFor(trackIndex); + + status_t err = mMediaSender->queueAccessUnit( + track->mediaSenderTrackIndex(), + accessUnit); + + if (err != OK) { + notifySessionDead(); + } + break; + } else if (what == Converter::kWhatEOS) { + CHECK_EQ(what, Converter::kWhatEOS); + + ALOGI("output EOS on track %zu", trackIndex); + + ssize_t index = mTracks.indexOfKey(trackIndex); + CHECK_GE(index, 0); + + const sp &converter = + mTracks.valueAt(index)->converter(); + looper()->unregisterHandler(converter->id()); + + mTracks.removeItemsAt(index); + + if (mTracks.isEmpty()) { + ALOGI("Reached EOS"); + } + } else if (what != Converter::kWhatShutdownCompleted) { + CHECK_EQ(what, Converter::kWhatError); + + status_t err; + CHECK(msg->findInt32("err", &err)); + + ALOGE("converter signaled error %d", err); + + notifySessionDead(); + } + break; + } + + case kWhatMediaSenderNotify: + { + int32_t what; + CHECK(msg->findInt32("what", &what)); + + if (what == MediaSender::kWhatInitDone) { + status_t err; + CHECK(msg->findInt32("err", &err)); + + if (err == OK) { + onMediaSenderInitialized(); + } else { + notifySessionDead(); + } + } else if (what == MediaSender::kWhatError) { + notifySessionDead(); + } else if (what == MediaSender::kWhatNetworkStall) { + size_t numBytesQueued; + CHECK(msg->findSize("numBytesQueued", &numBytesQueued)); + + if (mVideoTrackIndex >= 0) { + const sp &videoTrack = + mTracks.valueFor(mVideoTrackIndex); + + sp converter = videoTrack->converter(); + if (converter != NULL) { + converter->dropAFrame(); + } + } + } else if (what == MediaSender::kWhatInformSender) { + onSinkFeedback(msg); + } else { + TRESPASS(); + } + break; + } + + case kWhatTrackNotify: + { + int32_t what; + CHECK(msg->findInt32("what", &what)); + + size_t trackIndex; + CHECK(msg->findSize("trackIndex", &trackIndex)); + + if (what == Track::kWhatStopped) { + ALOGI("Track %zu stopped", trackIndex); + + sp track = mTracks.valueFor(trackIndex); + looper()->unregisterHandler(track->id()); + mTracks.removeItem(trackIndex); + track.clear(); + + if (!mTracks.isEmpty()) { + ALOGI("not all tracks are stopped yet"); + break; + } + + looper()->unregisterHandler(mMediaSender->id()); + mMediaSender.clear(); + + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatSessionDestroyed); + notify->post(); + } + break; + } + + case kWhatPause: + { + if (mExtractor != NULL) { + ++mPullExtractorGeneration; + mFirstSampleTimeRealUs = -1ll; + mFirstSampleTimeUs = -1ll; + } + + if (mPaused) { + break; + } + + for (size_t i = 0; i < mTracks.size(); ++i) { + mTracks.editValueAt(i)->pause(); + } + + mPaused = true; + break; + } + + case kWhatResume: + { + if (mExtractor != NULL) { + schedulePullExtractor(); + } + + if (!mPaused) { + break; + } + + for (size_t i = 0; i < mTracks.size(); ++i) { + mTracks.editValueAt(i)->resume(); + } + + mPaused = false; + break; + } + + case kWhatPullExtractorSample: + { + int32_t generation; + CHECK(msg->findInt32("generation", &generation)); + + if (generation != mPullExtractorGeneration) { + break; + } + + mPullExtractorPending = false; + + onPullExtractor(); + break; + } + + default: + TRESPASS(); + } +} + +void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp &msg) { + int64_t avgLatencyUs; + CHECK(msg->findInt64("avgLatencyUs", &avgLatencyUs)); + + int64_t maxLatencyUs; + CHECK(msg->findInt64("maxLatencyUs", &maxLatencyUs)); + + ALOGI("sink reports avg. latency of %lld ms (max %lld ms)", + avgLatencyUs / 1000ll, + maxLatencyUs / 1000ll); + + if (mVideoTrackIndex >= 0) { + const sp &videoTrack = mTracks.valueFor(mVideoTrackIndex); + sp converter = videoTrack->converter(); + + if (converter != NULL) { + int32_t videoBitrate = + Converter::GetInt32Property("media.wfd.video-bitrate", -1); + + char val[PROPERTY_VALUE_MAX]; + if (videoBitrate < 0 + && property_get("media.wfd.video-bitrate", val, NULL) + && !strcasecmp("adaptive", val)) { + videoBitrate = converter->getVideoBitrate(); + + if (avgLatencyUs > 300000ll) { + videoBitrate *= 0.6; + } else if (avgLatencyUs < 100000ll) { + videoBitrate *= 1.1; + } + } + + if (videoBitrate > 0) { + if (videoBitrate < 500000) { + videoBitrate = 500000; + } else if (videoBitrate > 10000000) { + videoBitrate = 10000000; + } + + if (videoBitrate != converter->getVideoBitrate()) { + ALOGI("setting video bitrate to %d bps", videoBitrate); + + converter->setVideoBitrate(videoBitrate); + } + } + } + + sp repeaterSource = videoTrack->repeaterSource(); + if (repeaterSource != NULL) { + double rateHz = + Converter::GetInt32Property( + "media.wfd.video-framerate", -1); + + char val[PROPERTY_VALUE_MAX]; + if (rateHz < 0.0 + && property_get("media.wfd.video-framerate", val, NULL) + && !strcasecmp("adaptive", val)) { + rateHz = repeaterSource->getFrameRate(); + + if (avgLatencyUs > 300000ll) { + rateHz *= 0.9; + } else if (avgLatencyUs < 200000ll) { + rateHz *= 1.1; + } + } + + if (rateHz > 0) { + if (rateHz < 5.0) { + rateHz = 5.0; + } else if (rateHz > 30.0) { + rateHz = 30.0; + } + + if (rateHz != repeaterSource->getFrameRate()) { + ALOGI("setting frame rate to %.2f Hz", rateHz); + + repeaterSource->setFrameRate(rateHz); + } + } + } + } +} + +status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer( + bool enableAudio, bool enableVideo) { + mExtractor = new NuMediaExtractor; + + status_t err = mExtractor->setDataSource( + NULL /* httpService */, mMediaPath.c_str()); + + if (err != OK) { + return err; + } + + size_t n = mExtractor->countTracks(); + bool haveAudio = false; + bool haveVideo = false; + for (size_t i = 0; i < n; ++i) { + sp format; + err = mExtractor->getTrackFormat(i, &format); + + if (err != OK) { + continue; + } + + AString mime; + CHECK(format->findString("mime", &mime)); + + bool isAudio = !strncasecmp(mime.c_str(), "audio/", 6); + bool isVideo = !strncasecmp(mime.c_str(), "video/", 6); + + if (isAudio && enableAudio && !haveAudio) { + haveAudio = true; + } else if (isVideo && enableVideo && !haveVideo) { + haveVideo = true; + } else { + continue; + } + + err = mExtractor->selectTrack(i); + + size_t trackIndex = mTracks.size(); + + sp notify = new AMessage(kWhatTrackNotify, this); + notify->setSize("trackIndex", trackIndex); + + sp track = new Track(notify, format); + looper()->registerHandler(track); + + mTracks.add(trackIndex, track); + + mExtractorTrackToInternalTrack.add(i, trackIndex); + + if (isVideo) { + mVideoTrackIndex = trackIndex; + } + + uint32_t flags = MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS; + + ssize_t mediaSenderTrackIndex = + mMediaSender->addTrack(format, flags); + CHECK_GE(mediaSenderTrackIndex, 0); + + track->setMediaSenderTrackIndex(mediaSenderTrackIndex); + + if ((haveAudio || !enableAudio) && (haveVideo || !enableVideo)) { + break; + } + } + + return OK; +} + +void WifiDisplaySource::PlaybackSession::schedulePullExtractor() { + if (mPullExtractorPending) { + return; + } + + int64_t delayUs = 1000000; // default delay is 1 sec + int64_t sampleTimeUs; + status_t err = mExtractor->getSampleTime(&sampleTimeUs); + + if (err == OK) { + int64_t nowUs = ALooper::GetNowUs(); + + if (mFirstSampleTimeRealUs < 0ll) { + mFirstSampleTimeRealUs = nowUs; + mFirstSampleTimeUs = sampleTimeUs; + } + + int64_t whenUs = sampleTimeUs - mFirstSampleTimeUs + mFirstSampleTimeRealUs; + delayUs = whenUs - nowUs; + } else { + ALOGW("could not get sample time (%d)", err); + } + + sp msg = new AMessage(kWhatPullExtractorSample, this); + msg->setInt32("generation", mPullExtractorGeneration); + msg->post(delayUs); + + mPullExtractorPending = true; +} + +void WifiDisplaySource::PlaybackSession::onPullExtractor() { + sp accessUnit = new ABuffer(1024 * 1024); + status_t err = mExtractor->readSampleData(accessUnit); + if (err != OK) { + // EOS. + return; + } + + int64_t timeUs; + CHECK_EQ((status_t)OK, mExtractor->getSampleTime(&timeUs)); + + accessUnit->meta()->setInt64( + "timeUs", mFirstSampleTimeRealUs + timeUs - mFirstSampleTimeUs); + + size_t trackIndex; + CHECK_EQ((status_t)OK, mExtractor->getSampleTrackIndex(&trackIndex)); + + sp msg = new AMessage(kWhatConverterNotify, this); + + msg->setSize( + "trackIndex", mExtractorTrackToInternalTrack.valueFor(trackIndex)); + + msg->setInt32("what", Converter::kWhatAccessUnit); + msg->setBuffer("accessUnit", accessUnit); + msg->post(); + + mExtractor->advance(); + + schedulePullExtractor(); +} + +status_t WifiDisplaySource::PlaybackSession::setupPacketizer( + bool enableAudio, + bool usePCMAudio, + bool enableVideo, + VideoFormats::ResolutionType videoResolutionType, + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType) { + CHECK(enableAudio || enableVideo); + + if (!mMediaPath.empty()) { + return setupMediaPacketizer(enableAudio, enableVideo); + } + + if (enableVideo) { + status_t err = addVideoSource( + videoResolutionType, videoResolutionIndex, videoProfileType, + videoLevelType); + + if (err != OK) { + return err; + } + } + + if (!enableAudio) { + return OK; + } + + return addAudioSource(usePCMAudio); +} + +status_t WifiDisplaySource::PlaybackSession::addSource( + bool isVideo, const sp &source, bool isRepeaterSource, + bool usePCMAudio, unsigned profileIdc, unsigned levelIdc, + unsigned constraintSet, size_t *numInputBuffers) { + CHECK(!usePCMAudio || !isVideo); + CHECK(!isRepeaterSource || isVideo); + CHECK(!profileIdc || isVideo); + CHECK(!levelIdc || isVideo); + CHECK(!constraintSet || isVideo); + + sp pullLooper = new ALooper; + pullLooper->setName("pull_looper"); + + pullLooper->start( + false /* runOnCallingThread */, + false /* canCallJava */, + PRIORITY_AUDIO); + + sp codecLooper = new ALooper; + codecLooper->setName("codec_looper"); + + codecLooper->start( + false /* runOnCallingThread */, + false /* canCallJava */, + PRIORITY_AUDIO); + + size_t trackIndex; + + sp notify; + + trackIndex = mTracks.size(); + + sp format; + status_t err = convertMetaDataToMessage(source->getFormat(), &format); + CHECK_EQ(err, (status_t)OK); + + if (isVideo) { + format->setString("mime", MEDIA_MIMETYPE_VIDEO_AVC); + format->setInt32( + "android._input-metadata-buffer-type", kMetadataBufferTypeANWBuffer); + format->setInt32("android._store-metadata-in-buffers-output", (mHDCP != NULL) + && (mHDCP->getCaps() & HDCPModule::HDCP_CAPS_ENCRYPT_NATIVE)); + format->setInt32( + "color-format", OMX_COLOR_FormatAndroidOpaque); + format->setInt32("profile-idc", profileIdc); + format->setInt32("level-idc", levelIdc); + format->setInt32("constraint-set", constraintSet); + } else { + if (usePCMAudio) { + format->setInt32("pcm-encoding", kAudioEncodingPcm16bit); + format->setString("mime", MEDIA_MIMETYPE_AUDIO_RAW); + } else { + format->setString("mime", MEDIA_MIMETYPE_AUDIO_AAC); + } + } + + notify = new AMessage(kWhatConverterNotify, this); + notify->setSize("trackIndex", trackIndex); + + sp converter = new Converter(notify, codecLooper, format); + + looper()->registerHandler(converter); + + err = converter->init(); + if (err != OK) { + ALOGE("%s converter returned err %d", isVideo ? "video" : "audio", err); + + looper()->unregisterHandler(converter->id()); + return err; + } + + notify = new AMessage(Converter::kWhatMediaPullerNotify, converter); + notify->setSize("trackIndex", trackIndex); + + sp puller = new MediaPuller(source, notify); + pullLooper->registerHandler(puller); + + if (numInputBuffers != NULL) { + *numInputBuffers = converter->getInputBufferCount(); + } + + notify = new AMessage(kWhatTrackNotify, this); + notify->setSize("trackIndex", trackIndex); + + sp track = new Track( + notify, pullLooper, codecLooper, puller, converter); + + if (isRepeaterSource) { + track->setRepeaterSource(static_cast(source.get())); + } + + looper()->registerHandler(track); + + mTracks.add(trackIndex, track); + + if (isVideo) { + mVideoTrackIndex = trackIndex; + } + + uint32_t flags = 0; + if (converter->needToManuallyPrependSPSPPS()) { + flags |= MediaSender::FLAG_MANUALLY_PREPEND_SPS_PPS; + } + + ssize_t mediaSenderTrackIndex = + mMediaSender->addTrack(converter->getOutputFormat(), flags); + CHECK_GE(mediaSenderTrackIndex, 0); + + track->setMediaSenderTrackIndex(mediaSenderTrackIndex); + + return OK; +} + +status_t WifiDisplaySource::PlaybackSession::addVideoSource( + VideoFormats::ResolutionType videoResolutionType, + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType) { + size_t width, height, framesPerSecond; + bool interlaced; + CHECK(VideoFormats::GetConfiguration( + videoResolutionType, + videoResolutionIndex, + &width, + &height, + &framesPerSecond, + &interlaced)); + + unsigned profileIdc, levelIdc, constraintSet; + CHECK(VideoFormats::GetProfileLevel( + videoProfileType, + videoLevelType, + &profileIdc, + &levelIdc, + &constraintSet)); + + sp source = new SurfaceMediaSource(width, height); + + source->setUseAbsoluteTimestamps(); + + sp videoSource = + new RepeaterSource(source, framesPerSecond); + + size_t numInputBuffers; + status_t err = addSource( + true /* isVideo */, videoSource, true /* isRepeaterSource */, + false /* usePCMAudio */, profileIdc, levelIdc, constraintSet, + &numInputBuffers); + + if (err != OK) { + return err; + } + + err = source->setMaxAcquiredBufferCount(numInputBuffers); + CHECK_EQ(err, (status_t)OK); + + mProducer = source->getProducer(); + + return OK; +} + +status_t WifiDisplaySource::PlaybackSession::addAudioSource(bool usePCMAudio) { + sp audioSource = new AudioSource( + AUDIO_SOURCE_REMOTE_SUBMIX, + mOpPackageName, + 48000 /* sampleRate */, + 2 /* channelCount */); + + if (audioSource->initCheck() == OK) { + return addSource( + false /* isVideo */, audioSource, false /* isRepeaterSource */, + usePCMAudio, 0 /* profileIdc */, 0 /* levelIdc */, + 0 /* constraintSet */, NULL /* numInputBuffers */); + } + + ALOGW("Unable to instantiate audio source"); + + return OK; +} + +sp WifiDisplaySource::PlaybackSession::getSurfaceTexture() { + return mProducer; +} + +void WifiDisplaySource::PlaybackSession::requestIDRFrame() { + for (size_t i = 0; i < mTracks.size(); ++i) { + const sp &track = mTracks.valueAt(i); + + track->requestIDRFrame(); + } +} + +void WifiDisplaySource::PlaybackSession::notifySessionDead() { + // Inform WifiDisplaySource of our premature death (wish). + sp notify = mNotify->dup(); + notify->setInt32("what", kWhatSessionDead); + notify->post(); + + mWeAreDead = true; +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.h b/media/libstagefright/wifi-display/source/PlaybackSession.h new file mode 100644 index 0000000000..f6673df541 --- /dev/null +++ b/media/libstagefright/wifi-display/source/PlaybackSession.h @@ -0,0 +1,176 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PLAYBACK_SESSION_H_ + +#define PLAYBACK_SESSION_H_ + +#include "MediaSender.h" +#include "VideoFormats.h" +#include "WifiDisplaySource.h" + +#include + +namespace android { + +struct ABuffer; +struct IHDCP; +class IGraphicBufferProducer; +struct MediaPuller; +struct MediaSource; +struct MediaSender; +struct NuMediaExtractor; + +// Encapsulates the state of an RTP/RTCP session in the context of wifi +// display. +struct WifiDisplaySource::PlaybackSession : public AHandler { + PlaybackSession( + const String16 &opPackageName, + const sp &netSession, + const sp ¬ify, + const struct in_addr &interfaceAddr, + const sp &hdcp, + const char *path = NULL); + + status_t init( + const char *clientIP, + int32_t clientRtp, + RTPSender::TransportMode rtpMode, + int32_t clientRtcp, + RTPSender::TransportMode rtcpMode, + bool enableAudio, + bool usePCMAudio, + bool enableVideo, + VideoFormats::ResolutionType videoResolutionType, + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType); + + void destroyAsync(); + + int32_t getRTPPort() const; + + int64_t getLastLifesignUs() const; + void updateLiveness(); + + status_t play(); + status_t finishPlay(); + status_t pause(); + + sp getSurfaceTexture(); + + void requestIDRFrame(); + + enum { + kWhatSessionDead, + kWhatBinaryData, + kWhatSessionEstablished, + kWhatSessionDestroyed, + }; + +protected: + virtual void onMessageReceived(const sp &msg); + virtual ~PlaybackSession(); + +private: + struct Track; + + enum { + kWhatMediaPullerNotify, + kWhatConverterNotify, + kWhatTrackNotify, + kWhatUpdateSurface, + kWhatPause, + kWhatResume, + kWhatMediaSenderNotify, + kWhatPullExtractorSample, + }; + + String16 mOpPackageName; + + sp mNetSession; + sp mNotify; + in_addr mInterfaceAddr; + sp mHDCP; + AString mMediaPath; + + sp mMediaSender; + int32_t mLocalRTPPort; + + bool mWeAreDead; + bool mPaused; + + int64_t mLastLifesignUs; + + sp mProducer; + + KeyedVector > mTracks; + ssize_t mVideoTrackIndex; + + int64_t mPrevTimeUs; + + sp mExtractor; + KeyedVector mExtractorTrackToInternalTrack; + bool mPullExtractorPending; + int32_t mPullExtractorGeneration; + int64_t mFirstSampleTimeRealUs; + int64_t mFirstSampleTimeUs; + + status_t setupMediaPacketizer(bool enableAudio, bool enableVideo); + + status_t setupPacketizer( + bool enableAudio, + bool usePCMAudio, + bool enableVideo, + VideoFormats::ResolutionType videoResolutionType, + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType); + + status_t addSource( + bool isVideo, + const sp &source, + bool isRepeaterSource, + bool usePCMAudio, + unsigned profileIdc, + unsigned levelIdc, + unsigned contraintSet, + size_t *numInputBuffers); + + status_t addVideoSource( + VideoFormats::ResolutionType videoResolutionType, + size_t videoResolutionIndex, + VideoFormats::ProfileType videoProfileType, + VideoFormats::LevelType videoLevelType); + + status_t addAudioSource(bool usePCMAudio); + + status_t onMediaSenderInitialized(); + + void notifySessionDead(); + + void schedulePullExtractor(); + void onPullExtractor(); + + void onSinkFeedback(const sp &msg); + + DISALLOW_EVIL_CONSTRUCTORS(PlaybackSession); +}; + +} // namespace android + +#endif // PLAYBACK_SESSION_H_ + diff --git a/media/libstagefright/wifi-display/source/RepeaterSource.cpp b/media/libstagefright/wifi-display/source/RepeaterSource.cpp new file mode 100644 index 0000000000..af6b66337d --- /dev/null +++ b/media/libstagefright/wifi-display/source/RepeaterSource.cpp @@ -0,0 +1,219 @@ +//#define LOG_NDEBUG 0 +#define LOG_TAG "RepeaterSource" +#include + +#include "RepeaterSource.h" + +#include +#include +#include +#include +#include + +namespace android { + +RepeaterSource::RepeaterSource(const sp &source, double rateHz) + : mStarted(false), + mSource(source), + mRateHz(rateHz), + mBuffer(NULL), + mResult(OK), + mLastBufferUpdateUs(-1ll), + mStartTimeUs(-1ll), + mFrameCount(0) { +} + +RepeaterSource::~RepeaterSource() { + CHECK(!mStarted); +} + +double RepeaterSource::getFrameRate() const { + return mRateHz; +} + +void RepeaterSource::setFrameRate(double rateHz) { + Mutex::Autolock autoLock(mLock); + + if (rateHz == mRateHz) { + return; + } + + if (mStartTimeUs >= 0ll) { + int64_t nextTimeUs = mStartTimeUs + (mFrameCount * 1000000ll) / mRateHz; + mStartTimeUs = nextTimeUs; + mFrameCount = 0; + } + mRateHz = rateHz; +} + +status_t RepeaterSource::start(MetaData *params) { + CHECK(!mStarted); + + status_t err = mSource->start(params); + + if (err != OK) { + return err; + } + + mBuffer = NULL; + mResult = OK; + mStartTimeUs = -1ll; + mFrameCount = 0; + + mLooper = new ALooper; + mLooper->setName("repeater_looper"); + mLooper->start(); + + mReflector = new AHandlerReflector(this); + mLooper->registerHandler(mReflector); + + postRead(); + + mStarted = true; + + return OK; +} + +status_t RepeaterSource::stop() { + CHECK(mStarted); + + ALOGV("stopping"); + + status_t err = mSource->stop(); + + if (mLooper != NULL) { + mLooper->stop(); + mLooper.clear(); + + mReflector.clear(); + } + + if (mBuffer != NULL) { + ALOGV("releasing mbuf %p", mBuffer); + mBuffer->release(); + mBuffer = NULL; + } + + + ALOGV("stopped"); + + mStarted = false; + + return err; +} + +sp RepeaterSource::getFormat() { + return mSource->getFormat(); +} + +status_t RepeaterSource::read( + MediaBuffer **buffer, const ReadOptions *options) { + int64_t seekTimeUs; + ReadOptions::SeekMode seekMode; + CHECK(options == NULL || !options->getSeekTo(&seekTimeUs, &seekMode)); + + for (;;) { + int64_t bufferTimeUs = -1ll; + + if (mStartTimeUs < 0ll) { + Mutex::Autolock autoLock(mLock); + while ((mLastBufferUpdateUs < 0ll || mBuffer == NULL) + && mResult == OK) { + mCondition.wait(mLock); + } + + ALOGV("now resuming."); + mStartTimeUs = ALooper::GetNowUs(); + bufferTimeUs = mStartTimeUs; + } else { + bufferTimeUs = mStartTimeUs + (mFrameCount * 1000000ll) / mRateHz; + + int64_t nowUs = ALooper::GetNowUs(); + int64_t delayUs = bufferTimeUs - nowUs; + + if (delayUs > 0ll) { + usleep(delayUs); + } + } + + bool stale = false; + + { + Mutex::Autolock autoLock(mLock); + if (mResult != OK) { + CHECK(mBuffer == NULL); + return mResult; + } + +#if SUSPEND_VIDEO_IF_IDLE + int64_t nowUs = ALooper::GetNowUs(); + if (nowUs - mLastBufferUpdateUs > 1000000ll) { + mLastBufferUpdateUs = -1ll; + stale = true; + } else +#endif + { + mBuffer->add_ref(); + *buffer = mBuffer; + (*buffer)->meta_data()->setInt64(kKeyTime, bufferTimeUs); + ++mFrameCount; + } + } + + if (!stale) { + break; + } + + mStartTimeUs = -1ll; + mFrameCount = 0; + ALOGV("now dormant"); + } + + return OK; +} + +void RepeaterSource::postRead() { + (new AMessage(kWhatRead, mReflector))->post(); +} + +void RepeaterSource::onMessageReceived(const sp &msg) { + switch (msg->what()) { + case kWhatRead: + { + MediaBuffer *buffer; + status_t err = mSource->read(&buffer); + + ALOGV("read mbuf %p", buffer); + + Mutex::Autolock autoLock(mLock); + if (mBuffer != NULL) { + mBuffer->release(); + mBuffer = NULL; + } + mBuffer = buffer; + mResult = err; + mLastBufferUpdateUs = ALooper::GetNowUs(); + + mCondition.broadcast(); + + if (err == OK) { + postRead(); + } + break; + } + + default: + TRESPASS(); + } +} + +void RepeaterSource::wakeUp() { + ALOGV("wakeUp"); + Mutex::Autolock autoLock(mLock); + if (mLastBufferUpdateUs < 0ll && mBuffer != NULL) { + mLastBufferUpdateUs = ALooper::GetNowUs(); + mCondition.broadcast(); + } +} + +} // namespace android diff --git a/media/libstagefright/wifi-display/source/RepeaterSource.h b/media/libstagefright/wifi-display/source/RepeaterSource.h new file mode 100644 index 0000000000..8d414fd538 --- /dev/null +++ b/media/libstagefright/wifi-display/source/RepeaterSource.h @@ -0,0 +1,67 @@ +#ifndef REPEATER_SOURCE_H_ + +#define REPEATER_SOURCE_H_ + +#include +#include +#include + +#define SUSPEND_VIDEO_IF_IDLE 0 + +namespace android { + +// This MediaSource delivers frames at a constant rate by repeating buffers +// if necessary. +struct RepeaterSource : public MediaSource { + RepeaterSource(const sp &source, double rateHz); + + virtual status_t start(MetaData *params); + virtual status_t stop(); + virtual sp getFormat(); + + virtual status_t read( + MediaBuffer **buffer, const ReadOptions *options); + + void onMessageReceived(const sp &msg); + + // If RepeaterSource is currently dormant, because SurfaceFlinger didn't + // send updates in a while, this is its wakeup call. + void wakeUp(); + + double getFrameRate() const; + void setFrameRate(double rateHz); + +protected: + virtual ~RepeaterSource(); + +private: + enum { + kWhatRead, + }; + + Mutex mLock; + Condition mCondition; + + bool mStarted; + + sp mSource; + double mRateHz; + + sp mLooper; + sp > mReflector; + + MediaBuffer *mBuffer; + status_t mResult; + int64_t mLastBufferUpdateUs; + + int64_t mStartTimeUs; + int32_t mFrameCount; + + void postRead(); + + DISALLOW_EVIL_CONSTRUCTORS(RepeaterSource); +}; + +} // namespace android + +#endif // REPEATER_SOURCE_H_ diff --git a/media/libstagefright/wifi-display/source/TSPacketizer.cpp b/media/libstagefright/wifi-display/source/TSPacketizer.cpp new file mode 100644 index 0000000000..865ba94e14 --- /dev/null +++ b/media/libstagefright/wifi-display/source/TSPacketizer.cpp @@ -0,0 +1,1055 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "TSPacketizer" +#include + +#include "TSPacketizer.h" +#include "include/avc_utils.h" + +#include +#include +#include +#include +#include +#include + +#include + +namespace android { + +struct TSPacketizer::Track : public RefBase { + Track(const sp &format, + unsigned PID, unsigned streamType, unsigned streamID); + + unsigned PID() const; + unsigned streamType() const; + unsigned streamID() const; + + // Returns the previous value. + unsigned incrementContinuityCounter(); + + bool isAudio() const; + bool isVideo() const; + + bool isH264() const; + bool isAAC() const; + bool lacksADTSHeader() const; + bool isPCMAudio() const; + + sp prependCSD(const sp &accessUnit) const; + sp prependADTSHeader(const sp &accessUnit) const; + + size_t countDescriptors() const; + sp descriptorAt(size_t index) const; + + void finalize(); + void extractCSDIfNecessary(); + +protected: + virtual ~Track(); + +private: + sp mFormat; + + unsigned mPID; + unsigned mStreamType; + unsigned mStreamID; + unsigned mContinuityCounter; + + AString mMIME; + Vector > mCSD; + + Vector > mDescriptors; + + bool mAudioLacksATDSHeaders; + bool mFinalized; + bool mExtractedCSD; + + DISALLOW_EVIL_CONSTRUCTORS(Track); +}; + +TSPacketizer::Track::Track( + const sp &format, + unsigned PID, unsigned streamType, unsigned streamID) + : mFormat(format), + mPID(PID), + mStreamType(streamType), + mStreamID(streamID), + mContinuityCounter(0), + mAudioLacksATDSHeaders(false), + mFinalized(false), + mExtractedCSD(false) { + CHECK(format->findString("mime", &mMIME)); +} + +void TSPacketizer::Track::extractCSDIfNecessary() { + if (mExtractedCSD) { + return; + } + + if (!strcasecmp(mMIME.c_str(), MEDIA_MIMETYPE_VIDEO_AVC) + || !strcasecmp(mMIME.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) { + for (size_t i = 0;; ++i) { + sp csd; + if (!mFormat->findBuffer(AStringPrintf("csd-%d", i).c_str(), &csd)) { + break; + } + + mCSD.push(csd); + } + + if (!strcasecmp(mMIME.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) { + int32_t isADTS; + if (!mFormat->findInt32("is-adts", &isADTS) || isADTS == 0) { + mAudioLacksATDSHeaders = true; + } + } + } + + mExtractedCSD = true; +} + +TSPacketizer::Track::~Track() { +} + +unsigned TSPacketizer::Track::PID() const { + return mPID; +} + +unsigned TSPacketizer::Track::streamType() const { + return mStreamType; +} + +unsigned TSPacketizer::Track::streamID() const { + return mStreamID; +} + +unsigned TSPacketizer::Track::incrementContinuityCounter() { + unsigned prevCounter = mContinuityCounter; + + if (++mContinuityCounter == 16) { + mContinuityCounter = 0; + } + + return prevCounter; +} + +bool TSPacketizer::Track::isAudio() const { + return !strncasecmp("audio/", mMIME.c_str(), 6); +} + +bool TSPacketizer::Track::isVideo() const { + return !strncasecmp("video/", mMIME.c_str(), 6); +} + +bool TSPacketizer::Track::isH264() const { + return !strcasecmp(mMIME.c_str(), MEDIA_MIMETYPE_VIDEO_AVC); +} + +bool TSPacketizer::Track::isAAC() const { + return !strcasecmp(mMIME.c_str(), MEDIA_MIMETYPE_AUDIO_AAC); +} + +bool TSPacketizer::Track::isPCMAudio() const { + return !strcasecmp(mMIME.c_str(), MEDIA_MIMETYPE_AUDIO_RAW); +} + +bool TSPacketizer::Track::lacksADTSHeader() const { + return mAudioLacksATDSHeaders; +} + +sp TSPacketizer::Track::prependCSD( + const sp &accessUnit) const { + size_t size = 0; + for (size_t i = 0; i < mCSD.size(); ++i) { + size += mCSD.itemAt(i)->size(); + } + + sp dup = new ABuffer(accessUnit->size() + size); + size_t offset = 0; + for (size_t i = 0; i < mCSD.size(); ++i) { + const sp &csd = mCSD.itemAt(i); + + memcpy(dup->data() + offset, csd->data(), csd->size()); + offset += csd->size(); + } + + memcpy(dup->data() + offset, accessUnit->data(), accessUnit->size()); + + return dup; +} + +sp TSPacketizer::Track::prependADTSHeader( + const sp &accessUnit) const { + CHECK_EQ(mCSD.size(), 1u); + + const uint8_t *codec_specific_data = mCSD.itemAt(0)->data(); + + const uint32_t aac_frame_length = accessUnit->size() + 7; + + sp dup = new ABuffer(aac_frame_length); + + unsigned profile = (codec_specific_data[0] >> 3) - 1; + + unsigned sampling_freq_index = + ((codec_specific_data[0] & 7) << 1) + | (codec_specific_data[1] >> 7); + + unsigned channel_configuration = + (codec_specific_data[1] >> 3) & 0x0f; + + uint8_t *ptr = dup->data(); + + *ptr++ = 0xff; + *ptr++ = 0xf9; // b11111001, ID=1(MPEG-2), layer=0, protection_absent=1 + + *ptr++ = + profile << 6 + | sampling_freq_index << 2 + | ((channel_configuration >> 2) & 1); // private_bit=0 + + // original_copy=0, home=0, copyright_id_bit=0, copyright_id_start=0 + *ptr++ = + (channel_configuration & 3) << 6 + | aac_frame_length >> 11; + *ptr++ = (aac_frame_length >> 3) & 0xff; + *ptr++ = (aac_frame_length & 7) << 5; + + // adts_buffer_fullness=0, number_of_raw_data_blocks_in_frame=0 + *ptr++ = 0; + + memcpy(ptr, accessUnit->data(), accessUnit->size()); + + return dup; +} + +size_t TSPacketizer::Track::countDescriptors() const { + return mDescriptors.size(); +} + +sp TSPacketizer::Track::descriptorAt(size_t index) const { + CHECK_LT(index, mDescriptors.size()); + return mDescriptors.itemAt(index); +} + +void TSPacketizer::Track::finalize() { + if (mFinalized) { + return; + } + + if (isH264()) { + { + // AVC video descriptor (40) + + sp descriptor = new ABuffer(6); + uint8_t *data = descriptor->data(); + data[0] = 40; // descriptor_tag + data[1] = 4; // descriptor_length + + if (mCSD.size() > 0) { + CHECK_GE(mCSD.size(), 1u); + const sp &sps = mCSD.itemAt(0); + CHECK(!memcmp("\x00\x00\x00\x01", sps->data(), 4)); + CHECK_GE(sps->size(), 7u); + // profile_idc, constraint_set*, level_idc + memcpy(&data[2], sps->data() + 4, 3); + } else { + int32_t profileIdc, levelIdc, constraintSet; + CHECK(mFormat->findInt32("profile-idc", &profileIdc)); + CHECK(mFormat->findInt32("level-idc", &levelIdc)); + CHECK(mFormat->findInt32("constraint-set", &constraintSet)); + CHECK_GE(profileIdc, 0); + CHECK_GE(levelIdc, 0); + data[2] = profileIdc; // profile_idc + data[3] = constraintSet; // constraint_set* + data[4] = levelIdc; // level_idc + } + + // AVC_still_present=0, AVC_24_hour_picture_flag=0, reserved + data[5] = 0x3f; + + mDescriptors.push_back(descriptor); + } + + { + // AVC timing and HRD descriptor (42) + + sp descriptor = new ABuffer(4); + uint8_t *data = descriptor->data(); + data[0] = 42; // descriptor_tag + data[1] = 2; // descriptor_length + + // hrd_management_valid_flag = 0 + // reserved = 111111b + // picture_and_timing_info_present = 0 + + data[2] = 0x7e; + + // fixed_frame_rate_flag = 0 + // temporal_poc_flag = 0 + // picture_to_display_conversion_flag = 0 + // reserved = 11111b + data[3] = 0x1f; + + mDescriptors.push_back(descriptor); + } + } else if (isPCMAudio()) { + // LPCM audio stream descriptor (0x83) + + int32_t channelCount; + CHECK(mFormat->findInt32("channel-count", &channelCount)); + CHECK_EQ(channelCount, 2); + + int32_t sampleRate; + CHECK(mFormat->findInt32("sample-rate", &sampleRate)); + CHECK(sampleRate == 44100 || sampleRate == 48000); + + sp descriptor = new ABuffer(4); + uint8_t *data = descriptor->data(); + data[0] = 0x83; // descriptor_tag + data[1] = 2; // descriptor_length + + unsigned sampling_frequency = (sampleRate == 44100) ? 1 : 2; + + data[2] = (sampling_frequency << 5) + | (3 /* reserved */ << 1) + | 0 /* emphasis_flag */; + + data[3] = + (1 /* number_of_channels = stereo */ << 5) + | 0xf /* reserved */; + + mDescriptors.push_back(descriptor); + } + + mFinalized = true; +} + +//////////////////////////////////////////////////////////////////////////////// + +TSPacketizer::TSPacketizer(uint32_t flags) + : mFlags(flags), + mPATContinuityCounter(0), + mPMTContinuityCounter(0) { + initCrcTable(); + + if (flags & (EMIT_HDCP20_DESCRIPTOR | EMIT_HDCP21_DESCRIPTOR)) { + int32_t hdcpVersion; + if (flags & EMIT_HDCP20_DESCRIPTOR) { + CHECK(!(flags & EMIT_HDCP21_DESCRIPTOR)); + hdcpVersion = 0x20; + } else { + CHECK(!(flags & EMIT_HDCP20_DESCRIPTOR)); + + // HDCP2.0 _and_ HDCP 2.1 specs say to set the version + // inside the HDCP descriptor to 0x20!!! + hdcpVersion = 0x20; + } + + // HDCP descriptor + sp descriptor = new ABuffer(7); + uint8_t *data = descriptor->data(); + data[0] = 0x05; // descriptor_tag + data[1] = 5; // descriptor_length + data[2] = 'H'; + data[3] = 'D'; + data[4] = 'C'; + data[5] = 'P'; + data[6] = hdcpVersion; + + mProgramInfoDescriptors.push_back(descriptor); + } +} + +TSPacketizer::~TSPacketizer() { +} + +ssize_t TSPacketizer::addTrack(const sp &format) { + AString mime; + CHECK(format->findString("mime", &mime)); + + unsigned PIDStart; + bool isVideo = !strncasecmp("video/", mime.c_str(), 6); + bool isAudio = !strncasecmp("audio/", mime.c_str(), 6); + + if (isVideo) { + PIDStart = 0x1011; + } else if (isAudio) { + PIDStart = 0x1100; + } else { + return ERROR_UNSUPPORTED; + } + + unsigned streamType; + unsigned streamIDStart; + unsigned streamIDStop; + + if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) { + streamType = 0x1b; + streamIDStart = 0xe0; + streamIDStop = 0xef; + } else if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_AAC)) { + streamType = 0x0f; + streamIDStart = 0xc0; + streamIDStop = 0xdf; + } else if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_AUDIO_RAW)) { + streamType = 0x83; + streamIDStart = 0xbd; + streamIDStop = 0xbd; + } else { + return ERROR_UNSUPPORTED; + } + + size_t numTracksOfThisType = 0; + unsigned PID = PIDStart; + + for (size_t i = 0; i < mTracks.size(); ++i) { + const sp &track = mTracks.itemAt(i); + + if (track->streamType() == streamType) { + ++numTracksOfThisType; + } + + if ((isAudio && track->isAudio()) || (isVideo && track->isVideo())) { + ++PID; + } + } + + unsigned streamID = streamIDStart + numTracksOfThisType; + if (streamID > streamIDStop) { + return -ERANGE; + } + + sp track = new Track(format, PID, streamType, streamID); + return mTracks.add(track); +} + +status_t TSPacketizer::extractCSDIfNecessary(size_t trackIndex) { + if (trackIndex >= mTracks.size()) { + return -ERANGE; + } + + const sp &track = mTracks.itemAt(trackIndex); + track->extractCSDIfNecessary(); + + return OK; +} + +status_t TSPacketizer::packetize( + size_t trackIndex, + const sp &_accessUnit, + sp *packets, + uint32_t flags, + const uint8_t *PES_private_data, size_t PES_private_data_len, + size_t numStuffingBytes) { + sp accessUnit = _accessUnit; + + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + packets->clear(); + + if (trackIndex >= mTracks.size()) { + return -ERANGE; + } + + const sp &track = mTracks.itemAt(trackIndex); + + if (track->isH264() && (flags & PREPEND_SPS_PPS_TO_IDR_FRAMES) + && IsIDR(accessUnit)) { + // prepend codec specific data, i.e. SPS and PPS. + accessUnit = track->prependCSD(accessUnit); + } else if (track->isAAC() && track->lacksADTSHeader()) { + CHECK(!(flags & IS_ENCRYPTED)); + accessUnit = track->prependADTSHeader(accessUnit); + } + + // 0x47 + // transport_error_indicator = b0 + // payload_unit_start_indicator = b1 + // transport_priority = b0 + // PID + // transport_scrambling_control = b00 + // adaptation_field_control = b?? + // continuity_counter = b???? + // -- payload follows + // packet_startcode_prefix = 0x000001 + // stream_id + // PES_packet_length = 0x???? + // reserved = b10 + // PES_scrambling_control = b00 + // PES_priority = b0 + // data_alignment_indicator = b1 + // copyright = b0 + // original_or_copy = b0 + // PTS_DTS_flags = b10 (PTS only) + // ESCR_flag = b0 + // ES_rate_flag = b0 + // DSM_trick_mode_flag = b0 + // additional_copy_info_flag = b0 + // PES_CRC_flag = b0 + // PES_extension_flag = b0 + // PES_header_data_length = 0x05 + // reserved = b0010 (PTS) + // PTS[32..30] = b??? + // reserved = b1 + // PTS[29..15] = b??? ???? ???? ???? (15 bits) + // reserved = b1 + // PTS[14..0] = b??? ???? ???? ???? (15 bits) + // reserved = b1 + // the first fragment of "buffer" follows + + // Each transport packet (except for the last one contributing to the PES + // payload) must contain a multiple of 16 bytes of payload per HDCP spec. + bool alignPayload = + (mFlags & (EMIT_HDCP20_DESCRIPTOR | EMIT_HDCP21_DESCRIPTOR)); + + /* + a) The very first PES transport stream packet contains + + 4 bytes of TS header + ... padding + 14 bytes of static PES header + PES_private_data_len + 1 bytes (only if PES_private_data_len > 0) + numStuffingBytes bytes + + followed by the payload + + b) Subsequent PES transport stream packets contain + + 4 bytes of TS header + ... padding + + followed by the payload + */ + + size_t PES_packet_length = accessUnit->size() + 8 + numStuffingBytes; + if (PES_private_data_len > 0) { + PES_packet_length += PES_private_data_len + 1; + } + + size_t numTSPackets = 1; + + { + // Make sure the PES header fits into a single TS packet: + size_t PES_header_size = 14 + numStuffingBytes; + if (PES_private_data_len > 0) { + PES_header_size += PES_private_data_len + 1; + } + + CHECK_LE(PES_header_size, 188u - 4u); + + size_t sizeAvailableForPayload = 188 - 4 - PES_header_size; + size_t numBytesOfPayload = accessUnit->size(); + + if (numBytesOfPayload > sizeAvailableForPayload) { + numBytesOfPayload = sizeAvailableForPayload; + + if (alignPayload && numBytesOfPayload > 16) { + numBytesOfPayload -= (numBytesOfPayload % 16); + } + } + + size_t numPaddingBytes = sizeAvailableForPayload - numBytesOfPayload; + ALOGV("packet 1 contains %zd padding bytes and %zd bytes of payload", + numPaddingBytes, numBytesOfPayload); + + size_t numBytesOfPayloadRemaining = accessUnit->size() - numBytesOfPayload; + +#if 0 + // The following hopefully illustrates the logic that led to the + // more efficient computation in the #else block... + + while (numBytesOfPayloadRemaining > 0) { + size_t sizeAvailableForPayload = 188 - 4; + + size_t numBytesOfPayload = numBytesOfPayloadRemaining; + + if (numBytesOfPayload > sizeAvailableForPayload) { + numBytesOfPayload = sizeAvailableForPayload; + + if (alignPayload && numBytesOfPayload > 16) { + numBytesOfPayload -= (numBytesOfPayload % 16); + } + } + + size_t numPaddingBytes = sizeAvailableForPayload - numBytesOfPayload; + ALOGI("packet %zd contains %zd padding bytes and %zd bytes of payload", + numTSPackets + 1, numPaddingBytes, numBytesOfPayload); + + numBytesOfPayloadRemaining -= numBytesOfPayload; + ++numTSPackets; + } +#else + // This is how many bytes of payload each subsequent TS packet + // can contain at most. + sizeAvailableForPayload = 188 - 4; + size_t sizeAvailableForAlignedPayload = sizeAvailableForPayload; + if (alignPayload) { + // We're only going to use a subset of the available space + // since we need to make each fragment a multiple of 16 in size. + sizeAvailableForAlignedPayload -= + (sizeAvailableForAlignedPayload % 16); + } + + size_t numFullTSPackets = + numBytesOfPayloadRemaining / sizeAvailableForAlignedPayload; + + numTSPackets += numFullTSPackets; + + numBytesOfPayloadRemaining -= + numFullTSPackets * sizeAvailableForAlignedPayload; + + // numBytesOfPayloadRemaining < sizeAvailableForAlignedPayload + if (numFullTSPackets == 0 && numBytesOfPayloadRemaining > 0) { + // There wasn't enough payload left to form a full aligned payload, + // the last packet doesn't have to be aligned. + ++numTSPackets; + } else if (numFullTSPackets > 0 + && numBytesOfPayloadRemaining + + sizeAvailableForAlignedPayload > sizeAvailableForPayload) { + // The last packet emitted had a full aligned payload and together + // with the bytes remaining does exceed the unaligned payload + // size, so we need another packet. + ++numTSPackets; + } +#endif + } + + if (flags & EMIT_PAT_AND_PMT) { + numTSPackets += 2; + } + + if (flags & EMIT_PCR) { + ++numTSPackets; + } + + sp buffer = new ABuffer(numTSPackets * 188); + uint8_t *packetDataStart = buffer->data(); + + if (flags & EMIT_PAT_AND_PMT) { + // Program Association Table (PAT): + // 0x47 + // transport_error_indicator = b0 + // payload_unit_start_indicator = b1 + // transport_priority = b0 + // PID = b0000000000000 (13 bits) + // transport_scrambling_control = b00 + // adaptation_field_control = b01 (no adaptation field, payload only) + // continuity_counter = b???? + // skip = 0x00 + // --- payload follows + // table_id = 0x00 + // section_syntax_indicator = b1 + // must_be_zero = b0 + // reserved = b11 + // section_length = 0x00d + // transport_stream_id = 0x0000 + // reserved = b11 + // version_number = b00001 + // current_next_indicator = b1 + // section_number = 0x00 + // last_section_number = 0x00 + // one program follows: + // program_number = 0x0001 + // reserved = b111 + // program_map_PID = kPID_PMT (13 bits!) + // CRC = 0x???????? + + if (++mPATContinuityCounter == 16) { + mPATContinuityCounter = 0; + } + + uint8_t *ptr = packetDataStart; + *ptr++ = 0x47; + *ptr++ = 0x40; + *ptr++ = 0x00; + *ptr++ = 0x10 | mPATContinuityCounter; + *ptr++ = 0x00; + + uint8_t *crcDataStart = ptr; + *ptr++ = 0x00; + *ptr++ = 0xb0; + *ptr++ = 0x0d; + *ptr++ = 0x00; + *ptr++ = 0x00; + *ptr++ = 0xc3; + *ptr++ = 0x00; + *ptr++ = 0x00; + *ptr++ = 0x00; + *ptr++ = 0x01; + *ptr++ = 0xe0 | (kPID_PMT >> 8); + *ptr++ = kPID_PMT & 0xff; + + CHECK_EQ(ptr - crcDataStart, 12); + uint32_t crc = htonl(crc32(crcDataStart, ptr - crcDataStart)); + memcpy(ptr, &crc, 4); + ptr += 4; + + size_t sizeLeft = packetDataStart + 188 - ptr; + memset(ptr, 0xff, sizeLeft); + + packetDataStart += 188; + + // Program Map (PMT): + // 0x47 + // transport_error_indicator = b0 + // payload_unit_start_indicator = b1 + // transport_priority = b0 + // PID = kPID_PMT (13 bits) + // transport_scrambling_control = b00 + // adaptation_field_control = b01 (no adaptation field, payload only) + // continuity_counter = b???? + // skip = 0x00 + // -- payload follows + // table_id = 0x02 + // section_syntax_indicator = b1 + // must_be_zero = b0 + // reserved = b11 + // section_length = 0x??? + // program_number = 0x0001 + // reserved = b11 + // version_number = b00001 + // current_next_indicator = b1 + // section_number = 0x00 + // last_section_number = 0x00 + // reserved = b111 + // PCR_PID = kPCR_PID (13 bits) + // reserved = b1111 + // program_info_length = 0x??? + // program_info_descriptors follow + // one or more elementary stream descriptions follow: + // stream_type = 0x?? + // reserved = b111 + // elementary_PID = b? ???? ???? ???? (13 bits) + // reserved = b1111 + // ES_info_length = 0x000 + // CRC = 0x???????? + + if (++mPMTContinuityCounter == 16) { + mPMTContinuityCounter = 0; + } + + ptr = packetDataStart; + *ptr++ = 0x47; + *ptr++ = 0x40 | (kPID_PMT >> 8); + *ptr++ = kPID_PMT & 0xff; + *ptr++ = 0x10 | mPMTContinuityCounter; + *ptr++ = 0x00; + + crcDataStart = ptr; + *ptr++ = 0x02; + + *ptr++ = 0x00; // section_length to be filled in below. + *ptr++ = 0x00; + + *ptr++ = 0x00; + *ptr++ = 0x01; + *ptr++ = 0xc3; + *ptr++ = 0x00; + *ptr++ = 0x00; + *ptr++ = 0xe0 | (kPID_PCR >> 8); + *ptr++ = kPID_PCR & 0xff; + + size_t program_info_length = 0; + for (size_t i = 0; i < mProgramInfoDescriptors.size(); ++i) { + program_info_length += mProgramInfoDescriptors.itemAt(i)->size(); + } + + CHECK_LT(program_info_length, 0x400u); + *ptr++ = 0xf0 | (program_info_length >> 8); + *ptr++ = (program_info_length & 0xff); + + for (size_t i = 0; i < mProgramInfoDescriptors.size(); ++i) { + const sp &desc = mProgramInfoDescriptors.itemAt(i); + memcpy(ptr, desc->data(), desc->size()); + ptr += desc->size(); + } + + for (size_t i = 0; i < mTracks.size(); ++i) { + const sp &track = mTracks.itemAt(i); + + // Make sure all the decriptors have been added. + track->finalize(); + + *ptr++ = track->streamType(); + *ptr++ = 0xe0 | (track->PID() >> 8); + *ptr++ = track->PID() & 0xff; + + size_t ES_info_length = 0; + for (size_t i = 0; i < track->countDescriptors(); ++i) { + ES_info_length += track->descriptorAt(i)->size(); + } + CHECK_LE(ES_info_length, 0xfffu); + + *ptr++ = 0xf0 | (ES_info_length >> 8); + *ptr++ = (ES_info_length & 0xff); + + for (size_t i = 0; i < track->countDescriptors(); ++i) { + const sp &descriptor = track->descriptorAt(i); + memcpy(ptr, descriptor->data(), descriptor->size()); + ptr += descriptor->size(); + } + } + + size_t section_length = ptr - (crcDataStart + 3) + 4 /* CRC */; + + crcDataStart[1] = 0xb0 | (section_length >> 8); + crcDataStart[2] = section_length & 0xff; + + crc = htonl(crc32(crcDataStart, ptr - crcDataStart)); + memcpy(ptr, &crc, 4); + ptr += 4; + + sizeLeft = packetDataStart + 188 - ptr; + memset(ptr, 0xff, sizeLeft); + + packetDataStart += 188; + } + + if (flags & EMIT_PCR) { + // PCR stream + // 0x47 + // transport_error_indicator = b0 + // payload_unit_start_indicator = b1 + // transport_priority = b0 + // PID = kPCR_PID (13 bits) + // transport_scrambling_control = b00 + // adaptation_field_control = b10 (adaptation field only, no payload) + // continuity_counter = b0000 (does not increment) + // adaptation_field_length = 183 + // discontinuity_indicator = b0 + // random_access_indicator = b0 + // elementary_stream_priority_indicator = b0 + // PCR_flag = b1 + // OPCR_flag = b0 + // splicing_point_flag = b0 + // transport_private_data_flag = b0 + // adaptation_field_extension_flag = b0 + // program_clock_reference_base = b????????????????????????????????? + // reserved = b111111 + // program_clock_reference_extension = b????????? + + int64_t nowUs = ALooper::GetNowUs(); + + uint64_t PCR = nowUs * 27; // PCR based on a 27MHz clock + uint64_t PCR_base = PCR / 300; + uint32_t PCR_ext = PCR % 300; + + uint8_t *ptr = packetDataStart; + *ptr++ = 0x47; + *ptr++ = 0x40 | (kPID_PCR >> 8); + *ptr++ = kPID_PCR & 0xff; + *ptr++ = 0x20; + *ptr++ = 0xb7; // adaptation_field_length + *ptr++ = 0x10; + *ptr++ = (PCR_base >> 25) & 0xff; + *ptr++ = (PCR_base >> 17) & 0xff; + *ptr++ = (PCR_base >> 9) & 0xff; + *ptr++ = ((PCR_base & 1) << 7) | 0x7e | ((PCR_ext >> 8) & 1); + *ptr++ = (PCR_ext & 0xff); + + size_t sizeLeft = packetDataStart + 188 - ptr; + memset(ptr, 0xff, sizeLeft); + + packetDataStart += 188; + } + + uint64_t PTS = (timeUs * 9ll) / 100ll; + + if (PES_packet_length >= 65536) { + // This really should only happen for video. + CHECK(track->isVideo()); + + // It's valid to set this to 0 for video according to the specs. + PES_packet_length = 0; + } + + size_t sizeAvailableForPayload = 188 - 4 - 14 - numStuffingBytes; + if (PES_private_data_len > 0) { + sizeAvailableForPayload -= PES_private_data_len + 1; + } + + size_t copy = accessUnit->size(); + + if (copy > sizeAvailableForPayload) { + copy = sizeAvailableForPayload; + + if (alignPayload && copy > 16) { + copy -= (copy % 16); + } + } + + size_t numPaddingBytes = sizeAvailableForPayload - copy; + + uint8_t *ptr = packetDataStart; + *ptr++ = 0x47; + *ptr++ = 0x40 | (track->PID() >> 8); + *ptr++ = track->PID() & 0xff; + + *ptr++ = (numPaddingBytes > 0 ? 0x30 : 0x10) + | track->incrementContinuityCounter(); + + if (numPaddingBytes > 0) { + *ptr++ = numPaddingBytes - 1; + if (numPaddingBytes >= 2) { + *ptr++ = 0x00; + memset(ptr, 0xff, numPaddingBytes - 2); + ptr += numPaddingBytes - 2; + } + } + + *ptr++ = 0x00; + *ptr++ = 0x00; + *ptr++ = 0x01; + *ptr++ = track->streamID(); + *ptr++ = PES_packet_length >> 8; + *ptr++ = PES_packet_length & 0xff; + *ptr++ = 0x84; + *ptr++ = (PES_private_data_len > 0) ? 0x81 : 0x80; + + size_t headerLength = 0x05 + numStuffingBytes; + if (PES_private_data_len > 0) { + headerLength += 1 + PES_private_data_len; + } + + *ptr++ = headerLength; + + *ptr++ = 0x20 | (((PTS >> 30) & 7) << 1) | 1; + *ptr++ = (PTS >> 22) & 0xff; + *ptr++ = (((PTS >> 15) & 0x7f) << 1) | 1; + *ptr++ = (PTS >> 7) & 0xff; + *ptr++ = ((PTS & 0x7f) << 1) | 1; + + if (PES_private_data_len > 0) { + *ptr++ = 0x8e; // PES_private_data_flag, reserved. + memcpy(ptr, PES_private_data, PES_private_data_len); + ptr += PES_private_data_len; + } + + for (size_t i = 0; i < numStuffingBytes; ++i) { + *ptr++ = 0xff; + } + + memcpy(ptr, accessUnit->data(), copy); + ptr += copy; + + CHECK_EQ(ptr, packetDataStart + 188); + packetDataStart += 188; + + size_t offset = copy; + while (offset < accessUnit->size()) { + // for subsequent fragments of "buffer": + // 0x47 + // transport_error_indicator = b0 + // payload_unit_start_indicator = b0 + // transport_priority = b0 + // PID = b0 0001 1110 ???? (13 bits) [0x1e0 + 1 + sourceIndex] + // transport_scrambling_control = b00 + // adaptation_field_control = b?? + // continuity_counter = b???? + // the fragment of "buffer" follows. + + size_t sizeAvailableForPayload = 188 - 4; + + size_t copy = accessUnit->size() - offset; + + if (copy > sizeAvailableForPayload) { + copy = sizeAvailableForPayload; + + if (alignPayload && copy > 16) { + copy -= (copy % 16); + } + } + + size_t numPaddingBytes = sizeAvailableForPayload - copy; + + uint8_t *ptr = packetDataStart; + *ptr++ = 0x47; + *ptr++ = 0x00 | (track->PID() >> 8); + *ptr++ = track->PID() & 0xff; + + *ptr++ = (numPaddingBytes > 0 ? 0x30 : 0x10) + | track->incrementContinuityCounter(); + + if (numPaddingBytes > 0) { + *ptr++ = numPaddingBytes - 1; + if (numPaddingBytes >= 2) { + *ptr++ = 0x00; + memset(ptr, 0xff, numPaddingBytes - 2); + ptr += numPaddingBytes - 2; + } + } + + memcpy(ptr, accessUnit->data() + offset, copy); + ptr += copy; + CHECK_EQ(ptr, packetDataStart + 188); + + offset += copy; + packetDataStart += 188; + } + + CHECK(packetDataStart == buffer->data() + buffer->capacity()); + + *packets = buffer; + + return OK; +} + +void TSPacketizer::initCrcTable() { + uint32_t poly = 0x04C11DB7; + + for (int i = 0; i < 256; i++) { + uint32_t crc = i << 24; + for (int j = 0; j < 8; j++) { + crc = (crc << 1) ^ ((crc & 0x80000000) ? (poly) : 0); + } + mCrcTable[i] = crc; + } +} + +uint32_t TSPacketizer::crc32(const uint8_t *start, size_t size) const { + uint32_t crc = 0xFFFFFFFF; + const uint8_t *p; + + for (p = start; p < start + size; ++p) { + crc = (crc << 8) ^ mCrcTable[((crc >> 24) ^ *p) & 0xFF]; + } + + return crc; +} + +sp TSPacketizer::prependCSD( + size_t trackIndex, const sp &accessUnit) const { + CHECK_LT(trackIndex, mTracks.size()); + + const sp &track = mTracks.itemAt(trackIndex); + CHECK(track->isH264() && IsIDR(accessUnit)); + + int64_t timeUs; + CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); + + sp accessUnit2 = track->prependCSD(accessUnit); + + accessUnit2->meta()->setInt64("timeUs", timeUs); + + return accessUnit2; +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/source/TSPacketizer.h b/media/libstagefright/wifi-display/source/TSPacketizer.h new file mode 100644 index 0000000000..0dcb179551 --- /dev/null +++ b/media/libstagefright/wifi-display/source/TSPacketizer.h @@ -0,0 +1,94 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TS_PACKETIZER_H_ + +#define TS_PACKETIZER_H_ + +#include +#include +#include +#include + +namespace android { + +struct ABuffer; +struct AMessage; + +// Forms the packets of a transport stream given access units. +// Emits metadata tables (PAT and PMT) and timestamp stream (PCR) based +// on flags. +struct TSPacketizer : public RefBase { + enum { + EMIT_HDCP20_DESCRIPTOR = 1, + EMIT_HDCP21_DESCRIPTOR = 2, + }; + explicit TSPacketizer(uint32_t flags); + + // Returns trackIndex or error. + ssize_t addTrack(const sp &format); + + enum { + EMIT_PAT_AND_PMT = 1, + EMIT_PCR = 2, + IS_ENCRYPTED = 4, + PREPEND_SPS_PPS_TO_IDR_FRAMES = 8, + }; + status_t packetize( + size_t trackIndex, const sp &accessUnit, + sp *packets, + uint32_t flags, + const uint8_t *PES_private_data, size_t PES_private_data_len, + size_t numStuffingBytes = 0); + + status_t extractCSDIfNecessary(size_t trackIndex); + + // XXX to be removed once encoder config option takes care of this for + // encrypted mode. + sp prependCSD( + size_t trackIndex, const sp &accessUnit) const; + +protected: + virtual ~TSPacketizer(); + +private: + enum { + kPID_PMT = 0x100, + kPID_PCR = 0x1000, + }; + + struct Track; + + uint32_t mFlags; + Vector > mTracks; + + Vector > mProgramInfoDescriptors; + + unsigned mPATContinuityCounter; + unsigned mPMTContinuityCounter; + + uint32_t mCrcTable[256]; + + void initCrcTable(); + uint32_t crc32(const uint8_t *start, size_t size) const; + + DISALLOW_EVIL_CONSTRUCTORS(TSPacketizer); +}; + +} // namespace android + +#endif // TS_PACKETIZER_H_ + diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp new file mode 100644 index 0000000000..4695e5d289 --- /dev/null +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp @@ -0,0 +1,1737 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "WifiDisplaySource" +#include + +#include "WifiDisplaySource.h" +#include "PlaybackSession.h" +#include "Parameters.h" +#include "rtp/RTPSender.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace android { + +// static +const int64_t WifiDisplaySource::kReaperIntervalUs; +const int64_t WifiDisplaySource::kTeardownTriggerTimeouSecs; +const int64_t WifiDisplaySource::kPlaybackSessionTimeoutSecs; +const int64_t WifiDisplaySource::kPlaybackSessionTimeoutUs; +const AString WifiDisplaySource::sUserAgent = MakeUserAgent(); + +WifiDisplaySource::WifiDisplaySource( + const String16 &opPackageName, + const sp &netSession, + const sp &client, + const char *path) + : mOpPackageName(opPackageName), + mState(INITIALIZED), + mNetSession(netSession), + mClient(client), + mSessionID(0), + mStopReplyID(NULL), + mChosenRTPPort(-1), + mUsingPCMAudio(false), + mClientSessionID(0), + mReaperPending(false), + mNextCSeq(1), + mUsingHDCP(false), + mIsHDCP2_0(false), + mHDCPPort(0), + mHDCPInitializationComplete(false), + mSetupTriggerDeferred(false), + mPlaybackSessionEstablished(false) { + if (path != NULL) { + mMediaPath.setTo(path); + } + + mSupportedSourceVideoFormats.disableAll(); + + mSupportedSourceVideoFormats.setNativeResolution( + VideoFormats::RESOLUTION_CEA, 5); // 1280x720 p30 + + // Enable all resolutions up to 1280x720p30 + mSupportedSourceVideoFormats.enableResolutionUpto( + VideoFormats::RESOLUTION_CEA, 5, + VideoFormats::PROFILE_CHP, // Constrained High Profile + VideoFormats::LEVEL_32); // Level 3.2 +} + +WifiDisplaySource::~WifiDisplaySource() { +} + +static status_t PostAndAwaitResponse( + const sp &msg, sp *response) { + status_t err = msg->postAndAwaitResponse(response); + + if (err != OK) { + return err; + } + + if (response == NULL || !(*response)->findInt32("err", &err)) { + err = OK; + } + + return err; +} + +status_t WifiDisplaySource::start(const char *iface) { + CHECK_EQ(mState, INITIALIZED); + + sp msg = new AMessage(kWhatStart, this); + msg->setString("iface", iface); + + sp response; + return PostAndAwaitResponse(msg, &response); +} + +status_t WifiDisplaySource::stop() { + sp msg = new AMessage(kWhatStop, this); + + sp response; + return PostAndAwaitResponse(msg, &response); +} + +status_t WifiDisplaySource::pause() { + sp msg = new AMessage(kWhatPause, this); + + sp response; + return PostAndAwaitResponse(msg, &response); +} + +status_t WifiDisplaySource::resume() { + sp msg = new AMessage(kWhatResume, this); + + sp response; + return PostAndAwaitResponse(msg, &response); +} + +void WifiDisplaySource::onMessageReceived(const sp &msg) { + switch (msg->what()) { + case kWhatStart: + { + sp replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + + AString iface; + CHECK(msg->findString("iface", &iface)); + + status_t err = OK; + + ssize_t colonPos = iface.find(":"); + + unsigned long port; + + if (colonPos >= 0) { + const char *s = iface.c_str() + colonPos + 1; + + char *end; + port = strtoul(s, &end, 10); + + if (end == s || *end != '\0' || port > 65535) { + err = -EINVAL; + } else { + iface.erase(colonPos, iface.size() - colonPos); + } + } else { + port = kWifiDisplayDefaultPort; + } + + if (err == OK) { + if (inet_aton(iface.c_str(), &mInterfaceAddr) != 0) { + sp notify = new AMessage(kWhatRTSPNotify, this); + + err = mNetSession->createRTSPServer( + mInterfaceAddr, port, notify, &mSessionID); + } else { + err = -EINVAL; + } + } + + mState = AWAITING_CLIENT_CONNECTION; + + sp response = new AMessage; + response->setInt32("err", err); + response->postReply(replyID); + break; + } + + case kWhatRTSPNotify: + { + int32_t reason; + CHECK(msg->findInt32("reason", &reason)); + + switch (reason) { + case ANetworkSession::kWhatError: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + int32_t err; + CHECK(msg->findInt32("err", &err)); + + AString detail; + CHECK(msg->findString("detail", &detail)); + + ALOGE("An error occurred in session %d (%d, '%s/%s').", + sessionID, + err, + detail.c_str(), + strerror(-err)); + + mNetSession->destroySession(sessionID); + + if (sessionID == mClientSessionID) { + mClientSessionID = 0; + + mClient->onDisplayError( + IRemoteDisplayClient::kDisplayErrorUnknown); + } + break; + } + + case ANetworkSession::kWhatClientConnected: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + if (mClientSessionID > 0) { + ALOGW("A client tried to connect, but we already " + "have one."); + + mNetSession->destroySession(sessionID); + break; + } + + CHECK_EQ(mState, AWAITING_CLIENT_CONNECTION); + + CHECK(msg->findString("client-ip", &mClientInfo.mRemoteIP)); + CHECK(msg->findString("server-ip", &mClientInfo.mLocalIP)); + + if (mClientInfo.mRemoteIP == mClientInfo.mLocalIP) { + // Disallow connections from the local interface + // for security reasons. + mNetSession->destroySession(sessionID); + break; + } + + CHECK(msg->findInt32( + "server-port", &mClientInfo.mLocalPort)); + mClientInfo.mPlaybackSessionID = -1; + + mClientSessionID = sessionID; + + ALOGI("We now have a client (%d) connected.", sessionID); + + mState = AWAITING_CLIENT_SETUP; + + status_t err = sendM1(sessionID); + CHECK_EQ(err, (status_t)OK); + break; + } + + case ANetworkSession::kWhatData: + { + status_t err = onReceiveClientData(msg); + + if (err != OK) { + mClient->onDisplayError( + IRemoteDisplayClient::kDisplayErrorUnknown); + } + +#if 0 + // testing only. + char val[PROPERTY_VALUE_MAX]; + if (property_get("media.wfd.trigger", val, NULL)) { + if (!strcasecmp(val, "pause") && mState == PLAYING) { + mState = PLAYING_TO_PAUSED; + sendTrigger(mClientSessionID, TRIGGER_PAUSE); + } else if (!strcasecmp(val, "play") + && mState == PAUSED) { + mState = PAUSED_TO_PLAYING; + sendTrigger(mClientSessionID, TRIGGER_PLAY); + } + } +#endif + break; + } + + case ANetworkSession::kWhatNetworkStall: + { + break; + } + + default: + TRESPASS(); + } + break; + } + + case kWhatStop: + { + CHECK(msg->senderAwaitsResponse(&mStopReplyID)); + + CHECK_LT(mState, AWAITING_CLIENT_TEARDOWN); + + if (mState >= AWAITING_CLIENT_PLAY) { + // We have a session, i.e. a previous SETUP succeeded. + + status_t err = sendTrigger( + mClientSessionID, TRIGGER_TEARDOWN); + + if (err == OK) { + mState = AWAITING_CLIENT_TEARDOWN; + + (new AMessage(kWhatTeardownTriggerTimedOut, this))->post( + kTeardownTriggerTimeouSecs * 1000000ll); + + break; + } + + // fall through. + } + + finishStop(); + break; + } + + case kWhatPause: + { + sp replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + + status_t err = OK; + + if (mState != PLAYING) { + err = INVALID_OPERATION; + } else { + mState = PLAYING_TO_PAUSED; + sendTrigger(mClientSessionID, TRIGGER_PAUSE); + } + + sp response = new AMessage; + response->setInt32("err", err); + response->postReply(replyID); + break; + } + + case kWhatResume: + { + sp replyID; + CHECK(msg->senderAwaitsResponse(&replyID)); + + status_t err = OK; + + if (mState != PAUSED) { + err = INVALID_OPERATION; + } else { + mState = PAUSED_TO_PLAYING; + sendTrigger(mClientSessionID, TRIGGER_PLAY); + } + + sp response = new AMessage; + response->setInt32("err", err); + response->postReply(replyID); + break; + } + + case kWhatReapDeadClients: + { + mReaperPending = false; + + if (mClientSessionID == 0 + || mClientInfo.mPlaybackSession == NULL) { + break; + } + + if (mClientInfo.mPlaybackSession->getLastLifesignUs() + + kPlaybackSessionTimeoutUs < ALooper::GetNowUs()) { + ALOGI("playback session timed out, reaping."); + + mNetSession->destroySession(mClientSessionID); + mClientSessionID = 0; + + mClient->onDisplayError( + IRemoteDisplayClient::kDisplayErrorUnknown); + } else { + scheduleReaper(); + } + break; + } + + case kWhatPlaybackSessionNotify: + { + int32_t playbackSessionID; + CHECK(msg->findInt32("playbackSessionID", &playbackSessionID)); + + int32_t what; + CHECK(msg->findInt32("what", &what)); + + if (what == PlaybackSession::kWhatSessionDead) { + ALOGI("playback session wants to quit."); + + mClient->onDisplayError( + IRemoteDisplayClient::kDisplayErrorUnknown); + } else if (what == PlaybackSession::kWhatSessionEstablished) { + mPlaybackSessionEstablished = true; + + if (mClient != NULL) { + if (!mSinkSupportsVideo) { + mClient->onDisplayConnected( + NULL, // SurfaceTexture + 0, // width, + 0, // height, + mUsingHDCP + ? IRemoteDisplayClient::kDisplayFlagSecure + : 0, + 0); + } else { + size_t width, height; + + CHECK(VideoFormats::GetConfiguration( + mChosenVideoResolutionType, + mChosenVideoResolutionIndex, + &width, + &height, + NULL /* framesPerSecond */, + NULL /* interlaced */)); + + mClient->onDisplayConnected( + mClientInfo.mPlaybackSession + ->getSurfaceTexture(), + width, + height, + mUsingHDCP + ? IRemoteDisplayClient::kDisplayFlagSecure + : 0, + playbackSessionID); + } + } + + finishPlay(); + + if (mState == ABOUT_TO_PLAY) { + mState = PLAYING; + } + } else if (what == PlaybackSession::kWhatSessionDestroyed) { + disconnectClient2(); + } else { + CHECK_EQ(what, PlaybackSession::kWhatBinaryData); + + int32_t channel; + CHECK(msg->findInt32("channel", &channel)); + + sp data; + CHECK(msg->findBuffer("data", &data)); + + CHECK_LE(channel, 0xff); + CHECK_LE(data->size(), 0xffffu); + + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + char header[4]; + header[0] = '$'; + header[1] = channel; + header[2] = data->size() >> 8; + header[3] = data->size() & 0xff; + + mNetSession->sendRequest( + sessionID, header, sizeof(header)); + + mNetSession->sendRequest( + sessionID, data->data(), data->size()); + } + break; + } + + case kWhatKeepAlive: + { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + if (mClientSessionID != sessionID) { + // Obsolete event, client is already gone. + break; + } + + sendM16(sessionID); + break; + } + + case kWhatTeardownTriggerTimedOut: + { + if (mState == AWAITING_CLIENT_TEARDOWN) { + ALOGI("TEARDOWN trigger timed out, forcing disconnection."); + + CHECK(mStopReplyID != NULL); + finishStop(); + break; + } + break; + } + + case kWhatHDCPNotify: + { + int32_t msgCode, ext1, ext2; + CHECK(msg->findInt32("msg", &msgCode)); + CHECK(msg->findInt32("ext1", &ext1)); + CHECK(msg->findInt32("ext2", &ext2)); + + ALOGI("Saw HDCP notification code %d, ext1 %d, ext2 %d", + msgCode, ext1, ext2); + + switch (msgCode) { + case HDCPModule::HDCP_INITIALIZATION_COMPLETE: + { + mHDCPInitializationComplete = true; + + if (mSetupTriggerDeferred) { + mSetupTriggerDeferred = false; + + sendTrigger(mClientSessionID, TRIGGER_SETUP); + } + break; + } + + case HDCPModule::HDCP_SHUTDOWN_COMPLETE: + case HDCPModule::HDCP_SHUTDOWN_FAILED: + { + // Ugly hack to make sure that the call to + // HDCPObserver::notify is completely handled before + // we clear the HDCP instance and unload the shared + // library :( + (new AMessage(kWhatFinishStop2, this))->post(300000ll); + break; + } + + default: + { + ALOGE("HDCP failure, shutting down."); + + mClient->onDisplayError( + IRemoteDisplayClient::kDisplayErrorUnknown); + break; + } + } + break; + } + + case kWhatFinishStop2: + { + finishStop2(); + break; + } + + default: + TRESPASS(); + } +} + +void WifiDisplaySource::registerResponseHandler( + int32_t sessionID, int32_t cseq, HandleRTSPResponseFunc func) { + ResponseID id; + id.mSessionID = sessionID; + id.mCSeq = cseq; + mResponseHandlers.add(id, func); +} + +status_t WifiDisplaySource::sendM1(int32_t sessionID) { + AString request = "OPTIONS * RTSP/1.0\r\n"; + AppendCommonResponse(&request, mNextCSeq); + + request.append( + "Require: org.wfa.wfd1.0\r\n" + "\r\n"); + + status_t err = + mNetSession->sendRequest(sessionID, request.c_str(), request.size()); + + if (err != OK) { + return err; + } + + registerResponseHandler( + sessionID, mNextCSeq, &WifiDisplaySource::onReceiveM1Response); + + ++mNextCSeq; + + return OK; +} + +status_t WifiDisplaySource::sendM3(int32_t sessionID) { + AString body = + "wfd_content_protection\r\n" + "wfd_video_formats\r\n" + "wfd_audio_codecs\r\n" + "wfd_client_rtp_ports\r\n"; + + AString request = "GET_PARAMETER rtsp://localhost/wfd1.0 RTSP/1.0\r\n"; + AppendCommonResponse(&request, mNextCSeq); + + request.append("Content-Type: text/parameters\r\n"); + request.append(AStringPrintf("Content-Length: %d\r\n", body.size())); + request.append("\r\n"); + request.append(body); + + status_t err = + mNetSession->sendRequest(sessionID, request.c_str(), request.size()); + + if (err != OK) { + return err; + } + + registerResponseHandler( + sessionID, mNextCSeq, &WifiDisplaySource::onReceiveM3Response); + + ++mNextCSeq; + + return OK; +} + +status_t WifiDisplaySource::sendM4(int32_t sessionID) { + CHECK_EQ(sessionID, mClientSessionID); + + AString body; + + if (mSinkSupportsVideo) { + body.append("wfd_video_formats: "); + + VideoFormats chosenVideoFormat; + chosenVideoFormat.disableAll(); + chosenVideoFormat.setNativeResolution( + mChosenVideoResolutionType, mChosenVideoResolutionIndex); + chosenVideoFormat.setProfileLevel( + mChosenVideoResolutionType, mChosenVideoResolutionIndex, + mChosenVideoProfile, mChosenVideoLevel); + + body.append(chosenVideoFormat.getFormatSpec(true /* forM4Message */)); + body.append("\r\n"); + } + + if (mSinkSupportsAudio) { + body.append( + AStringPrintf("wfd_audio_codecs: %s\r\n", + (mUsingPCMAudio + ? "LPCM 00000002 00" // 2 ch PCM 48kHz + : "AAC 00000001 00"))); // 2 ch AAC 48kHz + } + + body.append( + AStringPrintf( + "wfd_presentation_URL: rtsp://%s/wfd1.0/streamid=0 none\r\n", + mClientInfo.mLocalIP.c_str())); + + body.append( + AStringPrintf( + "wfd_client_rtp_ports: %s\r\n", mWfdClientRtpPorts.c_str())); + + AString request = "SET_PARAMETER rtsp://localhost/wfd1.0 RTSP/1.0\r\n"; + AppendCommonResponse(&request, mNextCSeq); + + request.append("Content-Type: text/parameters\r\n"); + request.append(AStringPrintf("Content-Length: %d\r\n", body.size())); + request.append("\r\n"); + request.append(body); + + status_t err = + mNetSession->sendRequest(sessionID, request.c_str(), request.size()); + + if (err != OK) { + return err; + } + + registerResponseHandler( + sessionID, mNextCSeq, &WifiDisplaySource::onReceiveM4Response); + + ++mNextCSeq; + + return OK; +} + +status_t WifiDisplaySource::sendTrigger( + int32_t sessionID, TriggerType triggerType) { + AString body = "wfd_trigger_method: "; + switch (triggerType) { + case TRIGGER_SETUP: + body.append("SETUP"); + break; + case TRIGGER_TEARDOWN: + ALOGI("Sending TEARDOWN trigger."); + body.append("TEARDOWN"); + break; + case TRIGGER_PAUSE: + body.append("PAUSE"); + break; + case TRIGGER_PLAY: + body.append("PLAY"); + break; + default: + TRESPASS(); + } + + body.append("\r\n"); + + AString request = "SET_PARAMETER rtsp://localhost/wfd1.0 RTSP/1.0\r\n"; + AppendCommonResponse(&request, mNextCSeq); + + request.append("Content-Type: text/parameters\r\n"); + request.append(AStringPrintf("Content-Length: %d\r\n", body.size())); + request.append("\r\n"); + request.append(body); + + status_t err = + mNetSession->sendRequest(sessionID, request.c_str(), request.size()); + + if (err != OK) { + return err; + } + + registerResponseHandler( + sessionID, mNextCSeq, &WifiDisplaySource::onReceiveM5Response); + + ++mNextCSeq; + + return OK; +} + +status_t WifiDisplaySource::sendM16(int32_t sessionID) { + AString request = "GET_PARAMETER rtsp://localhost/wfd1.0 RTSP/1.0\r\n"; + AppendCommonResponse(&request, mNextCSeq); + + CHECK_EQ(sessionID, mClientSessionID); + request.append( + AStringPrintf("Session: %d\r\n", mClientInfo.mPlaybackSessionID)); + request.append("\r\n"); // Empty body + + status_t err = + mNetSession->sendRequest(sessionID, request.c_str(), request.size()); + + if (err != OK) { + return err; + } + + registerResponseHandler( + sessionID, mNextCSeq, &WifiDisplaySource::onReceiveM16Response); + + ++mNextCSeq; + + scheduleKeepAlive(sessionID); + + return OK; +} + +status_t WifiDisplaySource::onReceiveM1Response( + int32_t /* sessionID */, const sp &msg) { + int32_t statusCode; + if (!msg->getStatusCode(&statusCode)) { + return ERROR_MALFORMED; + } + + if (statusCode != 200) { + return ERROR_UNSUPPORTED; + } + + return OK; +} + +// sink_audio_list := ("LPCM"|"AAC"|"AC3" HEXDIGIT*8 HEXDIGIT*2) +// (", " sink_audio_list)* +static void GetAudioModes(const char *s, const char *prefix, uint32_t *modes) { + *modes = 0; + + size_t prefixLen = strlen(prefix); + + while (*s != '0') { + if (!strncmp(s, prefix, prefixLen) && s[prefixLen] == ' ') { + unsigned latency; + if (sscanf(&s[prefixLen + 1], "%08x %02x", modes, &latency) != 2) { + *modes = 0; + } + + return; + } + + const char *commaPos = strchr(s, ','); + if (commaPos != NULL) { + s = commaPos + 1; + + while (isspace(*s)) { + ++s; + } + } else { + break; + } + } +} + +status_t WifiDisplaySource::onReceiveM3Response( + int32_t sessionID, const sp &msg) { + int32_t statusCode; + if (!msg->getStatusCode(&statusCode)) { + return ERROR_MALFORMED; + } + + if (statusCode != 200) { + return ERROR_UNSUPPORTED; + } + + sp params = + Parameters::Parse(msg->getContent(), strlen(msg->getContent())); + + if (params == NULL) { + return ERROR_MALFORMED; + } + + AString value; + if (!params->findParameter("wfd_client_rtp_ports", &value)) { + ALOGE("Sink doesn't report its choice of wfd_client_rtp_ports."); + return ERROR_MALFORMED; + } + + unsigned port0 = 0, port1 = 0; + if (sscanf(value.c_str(), + "RTP/AVP/UDP;unicast %u %u mode=play", + &port0, + &port1) == 2 + || sscanf(value.c_str(), + "RTP/AVP/TCP;unicast %u %u mode=play", + &port0, + &port1) == 2) { + if (port0 == 0 || port0 > 65535 || port1 != 0) { + ALOGE("Sink chose its wfd_client_rtp_ports poorly (%s)", + value.c_str()); + + return ERROR_MALFORMED; + } + } else if (strcmp(value.c_str(), "RTP/AVP/TCP;interleaved mode=play")) { + ALOGE("Unsupported value for wfd_client_rtp_ports (%s)", + value.c_str()); + + return ERROR_UNSUPPORTED; + } + + mWfdClientRtpPorts = value; + mChosenRTPPort = port0; + + if (!params->findParameter("wfd_video_formats", &value)) { + ALOGE("Sink doesn't report its choice of wfd_video_formats."); + return ERROR_MALFORMED; + } + + mSinkSupportsVideo = false; + + if (!(value == "none")) { + mSinkSupportsVideo = true; + if (!mSupportedSinkVideoFormats.parseFormatSpec(value.c_str())) { + ALOGE("Failed to parse sink provided wfd_video_formats (%s)", + value.c_str()); + + return ERROR_MALFORMED; + } + + if (!VideoFormats::PickBestFormat( + mSupportedSinkVideoFormats, + mSupportedSourceVideoFormats, + &mChosenVideoResolutionType, + &mChosenVideoResolutionIndex, + &mChosenVideoProfile, + &mChosenVideoLevel)) { + ALOGE("Sink and source share no commonly supported video " + "formats."); + + return ERROR_UNSUPPORTED; + } + + size_t width, height, framesPerSecond; + bool interlaced; + CHECK(VideoFormats::GetConfiguration( + mChosenVideoResolutionType, + mChosenVideoResolutionIndex, + &width, + &height, + &framesPerSecond, + &interlaced)); + + ALOGI("Picked video resolution %zu x %zu %c%zu", + width, height, interlaced ? 'i' : 'p', framesPerSecond); + + ALOGI("Picked AVC profile %d, level %d", + mChosenVideoProfile, mChosenVideoLevel); + } else { + ALOGI("Sink doesn't support video at all."); + } + + if (!params->findParameter("wfd_audio_codecs", &value)) { + ALOGE("Sink doesn't report its choice of wfd_audio_codecs."); + return ERROR_MALFORMED; + } + + mSinkSupportsAudio = false; + + if (!(value == "none")) { + mSinkSupportsAudio = true; + + uint32_t modes; + GetAudioModes(value.c_str(), "AAC", &modes); + + bool supportsAAC = (modes & 1) != 0; // AAC 2ch 48kHz + + GetAudioModes(value.c_str(), "LPCM", &modes); + + bool supportsPCM = (modes & 2) != 0; // LPCM 2ch 48kHz + + if (supportsPCM + && property_get_bool("media.wfd.use-pcm-audio", false)) { + ALOGI("Using PCM audio."); + mUsingPCMAudio = true; + } else if (supportsAAC) { + ALOGI("Using AAC audio."); + mUsingPCMAudio = false; + } else if (supportsPCM) { + ALOGI("Using PCM audio."); + mUsingPCMAudio = true; + } else { + ALOGI("Sink doesn't support an audio format we do."); + return ERROR_UNSUPPORTED; + } + } else { + ALOGI("Sink doesn't support audio at all."); + } + + if (!mSinkSupportsVideo && !mSinkSupportsAudio) { + ALOGE("Sink supports neither video nor audio..."); + return ERROR_UNSUPPORTED; + } + + mUsingHDCP = false; + if (!params->findParameter("wfd_content_protection", &value)) { + ALOGI("Sink doesn't appear to support content protection."); + } else if (value == "none") { + ALOGI("Sink does not support content protection."); + } else { + mUsingHDCP = true; + + bool isHDCP2_0 = false; + if (value.startsWith("HDCP2.0 ")) { + isHDCP2_0 = true; + } else if (!value.startsWith("HDCP2.1 ")) { + ALOGE("malformed wfd_content_protection: '%s'", value.c_str()); + + return ERROR_MALFORMED; + } + + int32_t hdcpPort; + if (!ParsedMessage::GetInt32Attribute( + value.c_str() + 8, "port", &hdcpPort) + || hdcpPort < 1 || hdcpPort > 65535) { + return ERROR_MALFORMED; + } + + mIsHDCP2_0 = isHDCP2_0; + mHDCPPort = hdcpPort; + + status_t err = makeHDCP(); + if (err != OK) { + ALOGE("Unable to instantiate HDCP component. " + "Not using HDCP after all."); + + mUsingHDCP = false; + } + } + + return sendM4(sessionID); +} + +status_t WifiDisplaySource::onReceiveM4Response( + int32_t sessionID, const sp &msg) { + int32_t statusCode; + if (!msg->getStatusCode(&statusCode)) { + return ERROR_MALFORMED; + } + + if (statusCode != 200) { + return ERROR_UNSUPPORTED; + } + + if (mUsingHDCP && !mHDCPInitializationComplete) { + ALOGI("Deferring SETUP trigger until HDCP initialization completes."); + + mSetupTriggerDeferred = true; + return OK; + } + + return sendTrigger(sessionID, TRIGGER_SETUP); +} + +status_t WifiDisplaySource::onReceiveM5Response( + int32_t /* sessionID */, const sp &msg) { + int32_t statusCode; + if (!msg->getStatusCode(&statusCode)) { + return ERROR_MALFORMED; + } + + if (statusCode != 200) { + return ERROR_UNSUPPORTED; + } + + return OK; +} + +status_t WifiDisplaySource::onReceiveM16Response( + int32_t sessionID, const sp & /* msg */) { + // If only the response was required to include a "Session:" header... + + CHECK_EQ(sessionID, mClientSessionID); + + if (mClientInfo.mPlaybackSession != NULL) { + mClientInfo.mPlaybackSession->updateLiveness(); + } + + return OK; +} + +void WifiDisplaySource::scheduleReaper() { + if (mReaperPending) { + return; + } + + mReaperPending = true; + (new AMessage(kWhatReapDeadClients, this))->post(kReaperIntervalUs); +} + +void WifiDisplaySource::scheduleKeepAlive(int32_t sessionID) { + // We need to send updates at least 5 secs before the timeout is set to + // expire, make sure the timeout is greater than 5 secs to begin with. + CHECK_GT(kPlaybackSessionTimeoutUs, 5000000ll); + + sp msg = new AMessage(kWhatKeepAlive, this); + msg->setInt32("sessionID", sessionID); + msg->post(kPlaybackSessionTimeoutUs - 5000000ll); +} + +status_t WifiDisplaySource::onReceiveClientData(const sp &msg) { + int32_t sessionID; + CHECK(msg->findInt32("sessionID", &sessionID)); + + sp obj; + CHECK(msg->findObject("data", &obj)); + + sp data = + static_cast(obj.get()); + + ALOGV("session %d received '%s'", + sessionID, data->debugString().c_str()); + + AString method; + AString uri; + data->getRequestField(0, &method); + + int32_t cseq; + if (!data->findInt32("cseq", &cseq)) { + sendErrorResponse(sessionID, "400 Bad Request", -1 /* cseq */); + return ERROR_MALFORMED; + } + + if (method.startsWith("RTSP/")) { + // This is a response. + + ResponseID id; + id.mSessionID = sessionID; + id.mCSeq = cseq; + + ssize_t index = mResponseHandlers.indexOfKey(id); + + if (index < 0) { + ALOGW("Received unsolicited server response, cseq %d", cseq); + return ERROR_MALFORMED; + } + + HandleRTSPResponseFunc func = mResponseHandlers.valueAt(index); + mResponseHandlers.removeItemsAt(index); + + status_t err = (this->*func)(sessionID, data); + + if (err != OK) { + ALOGW("Response handler for session %d, cseq %d returned " + "err %d (%s)", + sessionID, cseq, err, strerror(-err)); + + return err; + } + + return OK; + } + + AString version; + data->getRequestField(2, &version); + if (!(version == AString("RTSP/1.0"))) { + sendErrorResponse(sessionID, "505 RTSP Version not supported", cseq); + return ERROR_UNSUPPORTED; + } + + status_t err; + if (method == "OPTIONS") { + err = onOptionsRequest(sessionID, cseq, data); + } else if (method == "SETUP") { + err = onSetupRequest(sessionID, cseq, data); + } else if (method == "PLAY") { + err = onPlayRequest(sessionID, cseq, data); + } else if (method == "PAUSE") { + err = onPauseRequest(sessionID, cseq, data); + } else if (method == "TEARDOWN") { + err = onTeardownRequest(sessionID, cseq, data); + } else if (method == "GET_PARAMETER") { + err = onGetParameterRequest(sessionID, cseq, data); + } else if (method == "SET_PARAMETER") { + err = onSetParameterRequest(sessionID, cseq, data); + } else { + sendErrorResponse(sessionID, "405 Method Not Allowed", cseq); + + err = ERROR_UNSUPPORTED; + } + + return err; +} + +status_t WifiDisplaySource::onOptionsRequest( + int32_t sessionID, + int32_t cseq, + const sp &data) { + int32_t playbackSessionID; + sp playbackSession = + findPlaybackSession(data, &playbackSessionID); + + if (playbackSession != NULL) { + playbackSession->updateLiveness(); + } + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq); + + response.append( + "Public: org.wfa.wfd1.0, SETUP, TEARDOWN, PLAY, PAUSE, " + "GET_PARAMETER, SET_PARAMETER\r\n"); + + response.append("\r\n"); + + status_t err = mNetSession->sendRequest(sessionID, response.c_str()); + + if (err == OK) { + err = sendM3(sessionID); + } + + return err; +} + +status_t WifiDisplaySource::onSetupRequest( + int32_t sessionID, + int32_t cseq, + const sp &data) { + CHECK_EQ(sessionID, mClientSessionID); + if (mClientInfo.mPlaybackSessionID != -1) { + // We only support a single playback session per client. + // This is due to the reversed keep-alive design in the wfd specs... + sendErrorResponse(sessionID, "400 Bad Request", cseq); + return ERROR_MALFORMED; + } + + AString transport; + if (!data->findString("transport", &transport)) { + sendErrorResponse(sessionID, "400 Bad Request", cseq); + return ERROR_MALFORMED; + } + + RTPSender::TransportMode rtpMode = RTPSender::TRANSPORT_UDP; + + int clientRtp, clientRtcp; + if (transport.startsWith("RTP/AVP/TCP;")) { + AString interleaved; + if (ParsedMessage::GetAttribute( + transport.c_str(), "interleaved", &interleaved) + && sscanf(interleaved.c_str(), "%d-%d", + &clientRtp, &clientRtcp) == 2) { + rtpMode = RTPSender::TRANSPORT_TCP_INTERLEAVED; + } else { + bool badRequest = false; + + AString clientPort; + if (!ParsedMessage::GetAttribute( + transport.c_str(), "client_port", &clientPort)) { + badRequest = true; + } else if (sscanf(clientPort.c_str(), "%d-%d", + &clientRtp, &clientRtcp) == 2) { + } else if (sscanf(clientPort.c_str(), "%d", &clientRtp) == 1) { + // No RTCP. + clientRtcp = -1; + } else { + badRequest = true; + } + + if (badRequest) { + sendErrorResponse(sessionID, "400 Bad Request", cseq); + return ERROR_MALFORMED; + } + + rtpMode = RTPSender::TRANSPORT_TCP; + } + } else if (transport.startsWith("RTP/AVP;unicast;") + || transport.startsWith("RTP/AVP/UDP;unicast;")) { + bool badRequest = false; + + AString clientPort; + if (!ParsedMessage::GetAttribute( + transport.c_str(), "client_port", &clientPort)) { + badRequest = true; + } else if (sscanf(clientPort.c_str(), "%d-%d", + &clientRtp, &clientRtcp) == 2) { + } else if (sscanf(clientPort.c_str(), "%d", &clientRtp) == 1) { + // No RTCP. + clientRtcp = -1; + } else { + badRequest = true; + } + + if (badRequest) { + sendErrorResponse(sessionID, "400 Bad Request", cseq); + return ERROR_MALFORMED; + } +#if 1 + // The older LG dongles doesn't specify client_port=xxx apparently. + } else if (transport == "RTP/AVP/UDP;unicast") { + clientRtp = 19000; + clientRtcp = -1; +#endif + } else { + sendErrorResponse(sessionID, "461 Unsupported Transport", cseq); + return ERROR_UNSUPPORTED; + } + + int32_t playbackSessionID = makeUniquePlaybackSessionID(); + + sp notify = new AMessage(kWhatPlaybackSessionNotify, this); + notify->setInt32("playbackSessionID", playbackSessionID); + notify->setInt32("sessionID", sessionID); + + sp playbackSession = + new PlaybackSession( + mOpPackageName, mNetSession, notify, mInterfaceAddr, mHDCP, mMediaPath.c_str()); + + looper()->registerHandler(playbackSession); + + AString uri; + data->getRequestField(1, &uri); + + if (strncasecmp("rtsp://", uri.c_str(), 7)) { + sendErrorResponse(sessionID, "400 Bad Request", cseq); + return ERROR_MALFORMED; + } + + if (!(uri.startsWith("rtsp://") && uri.endsWith("/wfd1.0/streamid=0"))) { + sendErrorResponse(sessionID, "404 Not found", cseq); + return ERROR_MALFORMED; + } + + RTPSender::TransportMode rtcpMode = RTPSender::TRANSPORT_UDP; + if (clientRtcp < 0) { + rtcpMode = RTPSender::TRANSPORT_NONE; + } + + status_t err = playbackSession->init( + mClientInfo.mRemoteIP.c_str(), + clientRtp, + rtpMode, + clientRtcp, + rtcpMode, + mSinkSupportsAudio, + mUsingPCMAudio, + mSinkSupportsVideo, + mChosenVideoResolutionType, + mChosenVideoResolutionIndex, + mChosenVideoProfile, + mChosenVideoLevel); + + if (err != OK) { + looper()->unregisterHandler(playbackSession->id()); + playbackSession.clear(); + } + + switch (err) { + case OK: + break; + case -ENOENT: + sendErrorResponse(sessionID, "404 Not Found", cseq); + return err; + default: + sendErrorResponse(sessionID, "403 Forbidden", cseq); + return err; + } + + mClientInfo.mPlaybackSessionID = playbackSessionID; + mClientInfo.mPlaybackSession = playbackSession; + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq, playbackSessionID); + + if (rtpMode == RTPSender::TRANSPORT_TCP_INTERLEAVED) { + response.append( + AStringPrintf( + "Transport: RTP/AVP/TCP;interleaved=%d-%d;", + clientRtp, clientRtcp)); + } else { + int32_t serverRtp = playbackSession->getRTPPort(); + + AString transportString = "UDP"; + if (rtpMode == RTPSender::TRANSPORT_TCP) { + transportString = "TCP"; + } + + if (clientRtcp >= 0) { + response.append( + AStringPrintf( + "Transport: RTP/AVP/%s;unicast;client_port=%d-%d;" + "server_port=%d-%d\r\n", + transportString.c_str(), + clientRtp, clientRtcp, serverRtp, serverRtp + 1)); + } else { + response.append( + AStringPrintf( + "Transport: RTP/AVP/%s;unicast;client_port=%d;" + "server_port=%d\r\n", + transportString.c_str(), + clientRtp, serverRtp)); + } + } + + response.append("\r\n"); + + err = mNetSession->sendRequest(sessionID, response.c_str()); + + if (err != OK) { + return err; + } + + mState = AWAITING_CLIENT_PLAY; + + scheduleReaper(); + scheduleKeepAlive(sessionID); + + return OK; +} + +status_t WifiDisplaySource::onPlayRequest( + int32_t sessionID, + int32_t cseq, + const sp &data) { + int32_t playbackSessionID; + sp playbackSession = + findPlaybackSession(data, &playbackSessionID); + + if (playbackSession == NULL) { + sendErrorResponse(sessionID, "454 Session Not Found", cseq); + return ERROR_MALFORMED; + } + + if (mState != AWAITING_CLIENT_PLAY + && mState != PAUSED_TO_PLAYING + && mState != PAUSED) { + ALOGW("Received PLAY request but we're in state %d", mState); + + sendErrorResponse( + sessionID, "455 Method Not Valid in This State", cseq); + + return INVALID_OPERATION; + } + + ALOGI("Received PLAY request."); + if (mPlaybackSessionEstablished) { + finishPlay(); + } else { + ALOGI("deferring PLAY request until session established."); + } + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq, playbackSessionID); + response.append("Range: npt=now-\r\n"); + response.append("\r\n"); + + status_t err = mNetSession->sendRequest(sessionID, response.c_str()); + + if (err != OK) { + return err; + } + + if (mState == PAUSED_TO_PLAYING || mPlaybackSessionEstablished) { + mState = PLAYING; + return OK; + } + + CHECK_EQ(mState, AWAITING_CLIENT_PLAY); + mState = ABOUT_TO_PLAY; + + return OK; +} + +void WifiDisplaySource::finishPlay() { + const sp &playbackSession = + mClientInfo.mPlaybackSession; + + status_t err = playbackSession->play(); + CHECK_EQ(err, (status_t)OK); +} + +status_t WifiDisplaySource::onPauseRequest( + int32_t sessionID, + int32_t cseq, + const sp &data) { + int32_t playbackSessionID; + sp playbackSession = + findPlaybackSession(data, &playbackSessionID); + + if (playbackSession == NULL) { + sendErrorResponse(sessionID, "454 Session Not Found", cseq); + return ERROR_MALFORMED; + } + + ALOGI("Received PAUSE request."); + + if (mState != PLAYING_TO_PAUSED && mState != PLAYING) { + return INVALID_OPERATION; + } + + status_t err = playbackSession->pause(); + CHECK_EQ(err, (status_t)OK); + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq, playbackSessionID); + response.append("\r\n"); + + err = mNetSession->sendRequest(sessionID, response.c_str()); + + if (err != OK) { + return err; + } + + mState = PAUSED; + + return err; +} + +status_t WifiDisplaySource::onTeardownRequest( + int32_t sessionID, + int32_t cseq, + const sp &data) { + ALOGI("Received TEARDOWN request."); + + int32_t playbackSessionID; + sp playbackSession = + findPlaybackSession(data, &playbackSessionID); + + if (playbackSession == NULL) { + sendErrorResponse(sessionID, "454 Session Not Found", cseq); + return ERROR_MALFORMED; + } + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq, playbackSessionID); + response.append("Connection: close\r\n"); + response.append("\r\n"); + + mNetSession->sendRequest(sessionID, response.c_str()); + + if (mState == AWAITING_CLIENT_TEARDOWN) { + CHECK(mStopReplyID != NULL); + finishStop(); + } else { + mClient->onDisplayError(IRemoteDisplayClient::kDisplayErrorUnknown); + } + + return OK; +} + +void WifiDisplaySource::finishStop() { + ALOGV("finishStop"); + + mState = STOPPING; + + disconnectClientAsync(); +} + +void WifiDisplaySource::finishStopAfterDisconnectingClient() { + ALOGV("finishStopAfterDisconnectingClient"); + + if (mHDCP != NULL) { + ALOGI("Initiating HDCP shutdown."); + mHDCP->shutdownAsync(); + return; + } + + finishStop2(); +} + +void WifiDisplaySource::finishStop2() { + ALOGV("finishStop2"); + + if (mHDCP != NULL) { + mHDCP->setObserver(NULL); + mHDCPObserver.clear(); + mHDCP.clear(); + } + + if (mSessionID != 0) { + mNetSession->destroySession(mSessionID); + mSessionID = 0; + } + + ALOGI("We're stopped."); + mState = STOPPED; + + status_t err = OK; + + sp response = new AMessage; + response->setInt32("err", err); + response->postReply(mStopReplyID); +} + +status_t WifiDisplaySource::onGetParameterRequest( + int32_t sessionID, + int32_t cseq, + const sp &data) { + int32_t playbackSessionID; + sp playbackSession = + findPlaybackSession(data, &playbackSessionID); + + if (playbackSession == NULL) { + sendErrorResponse(sessionID, "454 Session Not Found", cseq); + return ERROR_MALFORMED; + } + + playbackSession->updateLiveness(); + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq, playbackSessionID); + response.append("\r\n"); + + status_t err = mNetSession->sendRequest(sessionID, response.c_str()); + return err; +} + +status_t WifiDisplaySource::onSetParameterRequest( + int32_t sessionID, + int32_t cseq, + const sp &data) { + int32_t playbackSessionID; + sp playbackSession = + findPlaybackSession(data, &playbackSessionID); + + if (playbackSession == NULL) { + sendErrorResponse(sessionID, "454 Session Not Found", cseq); + return ERROR_MALFORMED; + } + + if (strstr(data->getContent(), "wfd_idr_request\r\n")) { + playbackSession->requestIDRFrame(); + } + + playbackSession->updateLiveness(); + + AString response = "RTSP/1.0 200 OK\r\n"; + AppendCommonResponse(&response, cseq, playbackSessionID); + response.append("\r\n"); + + status_t err = mNetSession->sendRequest(sessionID, response.c_str()); + return err; +} + +// static +void WifiDisplaySource::AppendCommonResponse( + AString *response, int32_t cseq, int32_t playbackSessionID) { + time_t now = time(NULL); + struct tm *now2 = gmtime(&now); + char buf[128]; + strftime(buf, sizeof(buf), "%a, %d %b %Y %H:%M:%S %z", now2); + + response->append("Date: "); + response->append(buf); + response->append("\r\n"); + + response->append(AStringPrintf("Server: %s\r\n", sUserAgent.c_str())); + + if (cseq >= 0) { + response->append(AStringPrintf("CSeq: %d\r\n", cseq)); + } + + if (playbackSessionID >= 0ll) { + response->append( + AStringPrintf( + "Session: %d;timeout=%lld\r\n", + playbackSessionID, kPlaybackSessionTimeoutSecs)); + } +} + +void WifiDisplaySource::sendErrorResponse( + int32_t sessionID, + const char *errorDetail, + int32_t cseq) { + AString response; + response.append("RTSP/1.0 "); + response.append(errorDetail); + response.append("\r\n"); + + AppendCommonResponse(&response, cseq); + + response.append("\r\n"); + + mNetSession->sendRequest(sessionID, response.c_str()); +} + +int32_t WifiDisplaySource::makeUniquePlaybackSessionID() const { + return rand(); +} + +sp WifiDisplaySource::findPlaybackSession( + const sp &data, int32_t *playbackSessionID) const { + if (!data->findInt32("session", playbackSessionID)) { + // XXX the older dongles do not always include a "Session:" header. + *playbackSessionID = mClientInfo.mPlaybackSessionID; + return mClientInfo.mPlaybackSession; + } + + if (*playbackSessionID != mClientInfo.mPlaybackSessionID) { + return NULL; + } + + return mClientInfo.mPlaybackSession; +} + +void WifiDisplaySource::disconnectClientAsync() { + ALOGV("disconnectClient"); + + if (mClientInfo.mPlaybackSession == NULL) { + disconnectClient2(); + return; + } + + if (mClientInfo.mPlaybackSession != NULL) { + ALOGV("Destroying PlaybackSession"); + mClientInfo.mPlaybackSession->destroyAsync(); + } +} + +void WifiDisplaySource::disconnectClient2() { + ALOGV("disconnectClient2"); + + if (mClientInfo.mPlaybackSession != NULL) { + looper()->unregisterHandler(mClientInfo.mPlaybackSession->id()); + mClientInfo.mPlaybackSession.clear(); + } + + if (mClientSessionID != 0) { + mNetSession->destroySession(mClientSessionID); + mClientSessionID = 0; + } + + mClient->onDisplayDisconnected(); + + finishStopAfterDisconnectingClient(); +} + +struct WifiDisplaySource::HDCPObserver : public BnHDCPObserver { + explicit HDCPObserver(const sp ¬ify); + + virtual void notify( + int msg, int ext1, int ext2, const Parcel *obj); + +private: + sp mNotify; + + DISALLOW_EVIL_CONSTRUCTORS(HDCPObserver); +}; + +WifiDisplaySource::HDCPObserver::HDCPObserver( + const sp ¬ify) + : mNotify(notify) { +} + +void WifiDisplaySource::HDCPObserver::notify( + int msg, int ext1, int ext2, const Parcel * /* obj */) { + sp notify = mNotify->dup(); + notify->setInt32("msg", msg); + notify->setInt32("ext1", ext1); + notify->setInt32("ext2", ext2); + notify->post(); +} + +status_t WifiDisplaySource::makeHDCP() { + sp sm = defaultServiceManager(); + sp binder = sm->getService(String16("media.player")); + + sp service = + interface_cast(binder); + + CHECK(service != NULL); + + mHDCP = service->makeHDCP(true /* createEncryptionModule */); + + if (mHDCP == NULL) { + return ERROR_UNSUPPORTED; + } + + sp notify = new AMessage(kWhatHDCPNotify, this); + mHDCPObserver = new HDCPObserver(notify); + + status_t err = mHDCP->setObserver(mHDCPObserver); + + if (err != OK) { + ALOGE("Failed to set HDCP observer."); + + mHDCPObserver.clear(); + mHDCP.clear(); + + return err; + } + + ALOGI("Initiating HDCP negotiation w/ host %s:%d", + mClientInfo.mRemoteIP.c_str(), mHDCPPort); + + err = mHDCP->initAsync(mClientInfo.mRemoteIP.c_str(), mHDCPPort); + + if (err != OK) { + return err; + } + + return OK; +} + +} // namespace android + diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.h b/media/libstagefright/wifi-display/source/WifiDisplaySource.h new file mode 100644 index 0000000000..c25a675ee2 --- /dev/null +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.h @@ -0,0 +1,278 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WIFI_DISPLAY_SOURCE_H_ + +#define WIFI_DISPLAY_SOURCE_H_ + +#include "VideoFormats.h" + +#include +#include + +#include + +#include + +namespace android { + +struct AReplyToken; +struct IHDCP; +class IRemoteDisplayClient; +struct ParsedMessage; + +// Represents the RTSP server acting as a wifi display source. +// Manages incoming connections, sets up Playback sessions as necessary. +struct WifiDisplaySource : public AHandler { + static const unsigned kWifiDisplayDefaultPort = 7236; + + WifiDisplaySource( + const String16 &opPackageName, + const sp &netSession, + const sp &client, + const char *path = NULL); + + status_t start(const char *iface); + status_t stop(); + + status_t pause(); + status_t resume(); + +protected: + virtual ~WifiDisplaySource(); + virtual void onMessageReceived(const sp &msg); + +private: + struct PlaybackSession; + struct HDCPObserver; + + enum State { + INITIALIZED, + AWAITING_CLIENT_CONNECTION, + AWAITING_CLIENT_SETUP, + AWAITING_CLIENT_PLAY, + ABOUT_TO_PLAY, + PLAYING, + PLAYING_TO_PAUSED, + PAUSED, + PAUSED_TO_PLAYING, + AWAITING_CLIENT_TEARDOWN, + STOPPING, + STOPPED, + }; + + enum { + kWhatStart, + kWhatRTSPNotify, + kWhatStop, + kWhatPause, + kWhatResume, + kWhatReapDeadClients, + kWhatPlaybackSessionNotify, + kWhatKeepAlive, + kWhatHDCPNotify, + kWhatFinishStop2, + kWhatTeardownTriggerTimedOut, + }; + + struct ResponseID { + int32_t mSessionID; + int32_t mCSeq; + + bool operator<(const ResponseID &other) const { + return mSessionID < other.mSessionID + || (mSessionID == other.mSessionID + && mCSeq < other.mCSeq); + } + }; + + typedef status_t (WifiDisplaySource::*HandleRTSPResponseFunc)( + int32_t sessionID, const sp &msg); + + static const int64_t kReaperIntervalUs = 1000000ll; + + // We request that the dongle send us a "TEARDOWN" in order to + // perform an orderly shutdown. We're willing to wait up to 2 secs + // for this message to arrive, after that we'll force a disconnect + // instead. + static const int64_t kTeardownTriggerTimeouSecs = 2; + + static const int64_t kPlaybackSessionTimeoutSecs = 30; + + static const int64_t kPlaybackSessionTimeoutUs = + kPlaybackSessionTimeoutSecs * 1000000ll; + + static const AString sUserAgent; + + String16 mOpPackageName; + + State mState; + VideoFormats mSupportedSourceVideoFormats; + sp mNetSession; + sp mClient; + AString mMediaPath; + struct in_addr mInterfaceAddr; + int32_t mSessionID; + + sp mStopReplyID; + + AString mWfdClientRtpPorts; + int32_t mChosenRTPPort; // extracted from "wfd_client_rtp_ports" + + bool mSinkSupportsVideo; + VideoFormats mSupportedSinkVideoFormats; + + VideoFormats::ResolutionType mChosenVideoResolutionType; + size_t mChosenVideoResolutionIndex; + VideoFormats::ProfileType mChosenVideoProfile; + VideoFormats::LevelType mChosenVideoLevel; + + bool mSinkSupportsAudio; + + bool mUsingPCMAudio; + int32_t mClientSessionID; + + struct ClientInfo { + AString mRemoteIP; + AString mLocalIP; + int32_t mLocalPort; + int32_t mPlaybackSessionID; + sp mPlaybackSession; + }; + ClientInfo mClientInfo; + + bool mReaperPending; + + int32_t mNextCSeq; + + KeyedVector mResponseHandlers; + + // HDCP specific section >>>> + bool mUsingHDCP; + bool mIsHDCP2_0; + int32_t mHDCPPort; + sp mHDCP; + sp mHDCPObserver; + + bool mHDCPInitializationComplete; + bool mSetupTriggerDeferred; + + bool mPlaybackSessionEstablished; + + status_t makeHDCP(); + // <<<< HDCP specific section + + status_t sendM1(int32_t sessionID); + status_t sendM3(int32_t sessionID); + status_t sendM4(int32_t sessionID); + + enum TriggerType { + TRIGGER_SETUP, + TRIGGER_TEARDOWN, + TRIGGER_PAUSE, + TRIGGER_PLAY, + }; + + // M5 + status_t sendTrigger(int32_t sessionID, TriggerType triggerType); + + status_t sendM16(int32_t sessionID); + + status_t onReceiveM1Response( + int32_t sessionID, const sp &msg); + + status_t onReceiveM3Response( + int32_t sessionID, const sp &msg); + + status_t onReceiveM4Response( + int32_t sessionID, const sp &msg); + + status_t onReceiveM5Response( + int32_t sessionID, const sp &msg); + + status_t onReceiveM16Response( + int32_t sessionID, const sp &msg); + + void registerResponseHandler( + int32_t sessionID, int32_t cseq, HandleRTSPResponseFunc func); + + status_t onReceiveClientData(const sp &msg); + + status_t onOptionsRequest( + int32_t sessionID, + int32_t cseq, + const sp &data); + + status_t onSetupRequest( + int32_t sessionID, + int32_t cseq, + const sp &data); + + status_t onPlayRequest( + int32_t sessionID, + int32_t cseq, + const sp &data); + + status_t onPauseRequest( + int32_t sessionID, + int32_t cseq, + const sp &data); + + status_t onTeardownRequest( + int32_t sessionID, + int32_t cseq, + const sp &data); + + status_t onGetParameterRequest( + int32_t sessionID, + int32_t cseq, + const sp &data); + + status_t onSetParameterRequest( + int32_t sessionID, + int32_t cseq, + const sp &data); + + void sendErrorResponse( + int32_t sessionID, + const char *errorDetail, + int32_t cseq); + + static void AppendCommonResponse( + AString *response, int32_t cseq, int32_t playbackSessionID = -1ll); + + void scheduleReaper(); + void scheduleKeepAlive(int32_t sessionID); + + int32_t makeUniquePlaybackSessionID() const; + + sp findPlaybackSession( + const sp &data, int32_t *playbackSessionID) const; + + void finishStop(); + void disconnectClientAsync(); + void disconnectClient2(); + void finishStopAfterDisconnectingClient(); + void finishStop2(); + + void finishPlay(); + + DISALLOW_EVIL_CONSTRUCTORS(WifiDisplaySource); +}; + +} // namespace android + +#endif // WIFI_DISPLAY_SOURCE_H_ From dc368f84c7697e15a40beaa20756064266159b50 Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Wed, 9 Jan 2019 02:50:08 +0200 Subject: [PATCH 05/20] Revert "Removed unused class and its test" This adds back the SurfaceMediaSource class, needed for WFD. This reverts commit e885915204f252c93a072ba1a8802f5811e40b3d. Change-Id: I3f67d01f18441e49205e2e263d20f0fb6fc91fe6 Signed-off-by: Vladimir Oltean Signed-off-by: DennySPb Signed-off-by: Edwiin Kusuma Jaya --- media/libstagefright/Android.bp | 1 + media/libstagefright/SurfaceMediaSource.cpp | 477 +++++++++ .../media/stagefright/SurfaceMediaSource.h | 248 +++++ media/libstagefright/tests/Android.bp | 40 + media/libstagefright/tests/DummyRecorder.cpp | 91 ++ media/libstagefright/tests/DummyRecorder.h | 58 ++ .../tests/SurfaceMediaSource_test.cpp | 944 ++++++++++++++++++ 7 files changed, 1859 insertions(+) create mode 100644 media/libstagefright/SurfaceMediaSource.cpp create mode 100644 media/libstagefright/include/media/stagefright/SurfaceMediaSource.h create mode 100644 media/libstagefright/tests/DummyRecorder.cpp create mode 100644 media/libstagefright/tests/DummyRecorder.h create mode 100644 media/libstagefright/tests/SurfaceMediaSource_test.cpp diff --git a/media/libstagefright/Android.bp b/media/libstagefright/Android.bp index 10baec47f9..9af38fcb04 100644 --- a/media/libstagefright/Android.bp +++ b/media/libstagefright/Android.bp @@ -329,6 +329,7 @@ cc_library { "RemoteMediaSource.cpp", "SimpleDecodingSource.cpp", "StagefrightMediaScanner.cpp", + "SurfaceMediaSource.cpp", "SurfaceUtils.cpp", "ThrottledSource.cpp", "Utils.cpp", diff --git a/media/libstagefright/SurfaceMediaSource.cpp b/media/libstagefright/SurfaceMediaSource.cpp new file mode 100644 index 0000000000..4b3076a30d --- /dev/null +++ b/media/libstagefright/SurfaceMediaSource.cpp @@ -0,0 +1,477 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +//#define LOG_NDEBUG 0 +#define LOG_TAG "SurfaceMediaSource" + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include + +namespace android { + +SurfaceMediaSource::SurfaceMediaSource(uint32_t bufferWidth, uint32_t bufferHeight) : + mWidth(bufferWidth), + mHeight(bufferHeight), + mCurrentSlot(BufferQueue::INVALID_BUFFER_SLOT), + mNumPendingBuffers(0), + mCurrentTimestamp(0), + mFrameRate(30), + mStarted(false), + mNumFramesReceived(0), + mNumFramesEncoded(0), + mFirstFrameTimestamp(0), + mMaxAcquiredBufferCount(4), // XXX double-check the default + mUseAbsoluteTimestamps(false) { + ALOGV("SurfaceMediaSource"); + + if (bufferWidth == 0 || bufferHeight == 0) { + ALOGE("Invalid dimensions %dx%d", bufferWidth, bufferHeight); + } + + BufferQueue::createBufferQueue(&mProducer, &mConsumer); + mConsumer->setDefaultBufferSize(bufferWidth, bufferHeight); + mConsumer->setConsumerUsageBits(GRALLOC_USAGE_HW_VIDEO_ENCODER | + GRALLOC_USAGE_HW_TEXTURE); + + sp composer(ComposerService::getComposerService()); + + // Note that we can't create an sp<...>(this) in a ctor that will not keep a + // reference once the ctor ends, as that would cause the refcount of 'this' + // dropping to 0 at the end of the ctor. Since all we need is a wp<...> + // that's what we create. + wp listener = static_cast(this); + sp proxy = new BufferQueue::ProxyConsumerListener(listener); + + status_t err = mConsumer->consumerConnect(proxy, false); + if (err != NO_ERROR) { + ALOGE("SurfaceMediaSource: error connecting to BufferQueue: %s (%d)", + strerror(-err), err); + } +} + +SurfaceMediaSource::~SurfaceMediaSource() { + ALOGV("~SurfaceMediaSource"); + CHECK(!mStarted); +} + +nsecs_t SurfaceMediaSource::getTimestamp() { + ALOGV("getTimestamp"); + Mutex::Autolock lock(mMutex); + return mCurrentTimestamp; +} + +void SurfaceMediaSource::setFrameAvailableListener( + const sp& listener) { + ALOGV("setFrameAvailableListener"); + Mutex::Autolock lock(mMutex); + mFrameAvailableListener = listener; +} + +void SurfaceMediaSource::dumpState(String8& result) const +{ + char buffer[1024]; + dumpState(result, "", buffer, 1024); +} + +void SurfaceMediaSource::dumpState( + String8& result, + const char* /* prefix */, + char* buffer, + size_t /* SIZE */) const +{ + Mutex::Autolock lock(mMutex); + + result.append(buffer); + mConsumer->dumpState(result, ""); +} + +status_t SurfaceMediaSource::setFrameRate(int32_t fps) +{ + ALOGV("setFrameRate"); + Mutex::Autolock lock(mMutex); + const int MAX_FRAME_RATE = 60; + if (fps < 0 || fps > MAX_FRAME_RATE) { + return BAD_VALUE; + } + mFrameRate = fps; + return OK; +} + +MetadataBufferType SurfaceMediaSource::metaDataStoredInVideoBuffers() const { + ALOGV("isMetaDataStoredInVideoBuffers"); + return kMetadataBufferTypeANWBuffer; +} + +int32_t SurfaceMediaSource::getFrameRate( ) const { + ALOGV("getFrameRate"); + Mutex::Autolock lock(mMutex); + return mFrameRate; +} + +status_t SurfaceMediaSource::start(MetaData *params) +{ + ALOGV("start"); + + Mutex::Autolock lock(mMutex); + + CHECK(!mStarted); + + mStartTimeNs = 0; + int64_t startTimeUs; + int32_t bufferCount = 0; + if (params) { + if (params->findInt64(kKeyTime, &startTimeUs)) { + mStartTimeNs = startTimeUs * 1000; + } + + if (!params->findInt32(kKeyNumBuffers, &bufferCount)) { + ALOGE("Failed to find the advertised buffer count"); + return UNKNOWN_ERROR; + } + + if (bufferCount <= 1) { + ALOGE("bufferCount %d is too small", bufferCount); + return BAD_VALUE; + } + + mMaxAcquiredBufferCount = bufferCount; + } + + CHECK_GT(mMaxAcquiredBufferCount, 1u); + + status_t err = + mConsumer->setMaxAcquiredBufferCount(mMaxAcquiredBufferCount); + + if (err != OK) { + return err; + } + + mNumPendingBuffers = 0; + mStarted = true; + + return OK; +} + +status_t SurfaceMediaSource::setMaxAcquiredBufferCount(size_t count) { + ALOGV("setMaxAcquiredBufferCount(%zu)", count); + Mutex::Autolock lock(mMutex); + + CHECK_GT(count, 1u); + mMaxAcquiredBufferCount = count; + + return OK; +} + +status_t SurfaceMediaSource::setUseAbsoluteTimestamps() { + ALOGV("setUseAbsoluteTimestamps"); + Mutex::Autolock lock(mMutex); + mUseAbsoluteTimestamps = true; + + return OK; +} + +status_t SurfaceMediaSource::stop() +{ + ALOGV("stop"); + Mutex::Autolock lock(mMutex); + + if (!mStarted) { + return OK; + } + + mStarted = false; + mFrameAvailableCondition.signal(); + + while (mNumPendingBuffers > 0) { + ALOGI("Still waiting for %zu buffers to be returned.", + mNumPendingBuffers); + +#if DEBUG_PENDING_BUFFERS + for (size_t i = 0; i < mPendingBuffers.size(); ++i) { + ALOGI("%d: %p", i, mPendingBuffers.itemAt(i)); + } +#endif + + mMediaBuffersAvailableCondition.wait(mMutex); + } + + mMediaBuffersAvailableCondition.signal(); + + return mConsumer->consumerDisconnect(); +} + +sp SurfaceMediaSource::getFormat() +{ + ALOGV("getFormat"); + + Mutex::Autolock lock(mMutex); + sp meta = new MetaData; + + meta->setInt32(kKeyWidth, mWidth); + meta->setInt32(kKeyHeight, mHeight); + // The encoder format is set as an opaque colorformat + // The encoder will later find out the actual colorformat + // from the GL Frames itself. + meta->setInt32(kKeyColorFormat, OMX_COLOR_FormatAndroidOpaque); + meta->setInt32(kKeyStride, mWidth); + meta->setInt32(kKeySliceHeight, mHeight); + meta->setInt32(kKeyFrameRate, mFrameRate); + meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_RAW); + return meta; +} + +// Pass the data to the MediaBuffer. Pass in only the metadata +// Note: Call only when you have the lock +void SurfaceMediaSource::passMetadataBuffer_l(MediaBufferBase **buffer, + ANativeWindowBuffer *bufferHandle) const { + *buffer = new MediaBuffer(sizeof(VideoNativeMetadata)); + VideoNativeMetadata *data = (VideoNativeMetadata *)(*buffer)->data(); + if (data == NULL) { + ALOGE("Cannot allocate memory for metadata buffer!"); + return; + } + data->eType = metaDataStoredInVideoBuffers(); + data->pBuffer = bufferHandle; + data->nFenceFd = -1; + ALOGV("handle = %p, offset = %zu, length = %zu", + bufferHandle, (*buffer)->range_length(), (*buffer)->range_offset()); +} + +status_t SurfaceMediaSource::read( + MediaBufferBase **buffer, const ReadOptions * /* options */) { + ALOGV("read"); + Mutex::Autolock lock(mMutex); + + *buffer = NULL; + + while (mStarted && mNumPendingBuffers == mMaxAcquiredBufferCount) { + mMediaBuffersAvailableCondition.wait(mMutex); + } + + // Update the current buffer info + // TODO: mCurrentSlot can be made a bufferstate since there + // can be more than one "current" slots. + + BufferItem item; + // If the recording has started and the queue is empty, then just + // wait here till the frames come in from the client side + while (mStarted) { + + status_t err = mConsumer->acquireBuffer(&item, 0); + if (err == BufferQueue::NO_BUFFER_AVAILABLE) { + // wait for a buffer to be queued + mFrameAvailableCondition.wait(mMutex); + } else if (err == OK) { + err = item.mFence->waitForever("SurfaceMediaSource::read"); + if (err) { + ALOGW("read: failed to wait for buffer fence: %d", err); + } + + // First time seeing the buffer? Added it to the SMS slot + if (item.mGraphicBuffer != NULL) { + mSlots[item.mSlot].mGraphicBuffer = item.mGraphicBuffer; + } + mSlots[item.mSlot].mFrameNumber = item.mFrameNumber; + + // check for the timing of this buffer + if (mNumFramesReceived == 0 && !mUseAbsoluteTimestamps) { + mFirstFrameTimestamp = item.mTimestamp; + // Initial delay + if (mStartTimeNs > 0) { + if (item.mTimestamp < mStartTimeNs) { + // This frame predates start of record, discard + mConsumer->releaseBuffer( + item.mSlot, item.mFrameNumber, EGL_NO_DISPLAY, + EGL_NO_SYNC_KHR, Fence::NO_FENCE); + continue; + } + mStartTimeNs = item.mTimestamp - mStartTimeNs; + } + } + item.mTimestamp = mStartTimeNs + (item.mTimestamp - mFirstFrameTimestamp); + + mNumFramesReceived++; + + break; + } else { + ALOGE("read: acquire failed with error code %d", err); + return ERROR_END_OF_STREAM; + } + + } + + // If the loop was exited as a result of stopping the recording, + // it is OK + if (!mStarted) { + ALOGV("Read: SurfaceMediaSource is stopped. Returning ERROR_END_OF_STREAM."); + return ERROR_END_OF_STREAM; + } + + mCurrentSlot = item.mSlot; + + // First time seeing the buffer? Added it to the SMS slot + if (item.mGraphicBuffer != NULL) { + mSlots[item.mSlot].mGraphicBuffer = item.mGraphicBuffer; + } + mSlots[item.mSlot].mFrameNumber = item.mFrameNumber; + + mCurrentBuffers.push_back(mSlots[mCurrentSlot].mGraphicBuffer); + int64_t prevTimeStamp = mCurrentTimestamp; + mCurrentTimestamp = item.mTimestamp; + + mNumFramesEncoded++; + // Pass the data to the MediaBuffer. Pass in only the metadata + + passMetadataBuffer_l(buffer, mSlots[mCurrentSlot].mGraphicBuffer->getNativeBuffer()); + + (*buffer)->setObserver(this); + (*buffer)->add_ref(); + (*buffer)->meta_data()->setInt64(kKeyTime, mCurrentTimestamp / 1000); + ALOGV("Frames encoded = %d, timestamp = %" PRId64 ", time diff = %" PRId64, + mNumFramesEncoded, mCurrentTimestamp / 1000, + mCurrentTimestamp / 1000 - prevTimeStamp / 1000); + + ++mNumPendingBuffers; + +#if DEBUG_PENDING_BUFFERS + mPendingBuffers.push_back(*buffer); +#endif + + ALOGV("returning mbuf %p", *buffer); + + return OK; +} + +static buffer_handle_t getMediaBufferHandle(MediaBufferBase *buffer) { + // need to convert to char* for pointer arithmetic and then + // copy the byte stream into our handle + buffer_handle_t bufferHandle; + memcpy(&bufferHandle, (char*)(buffer->data()) + 4, sizeof(buffer_handle_t)); + return bufferHandle; +} + +void SurfaceMediaSource::signalBufferReturned(MediaBufferBase *buffer) { + ALOGV("signalBufferReturned"); + + bool foundBuffer = false; + + Mutex::Autolock lock(mMutex); + + buffer_handle_t bufferHandle = getMediaBufferHandle(buffer); + + for (size_t i = 0; i < mCurrentBuffers.size(); i++) { + if (mCurrentBuffers[i]->handle == bufferHandle) { + mCurrentBuffers.removeAt(i); + foundBuffer = true; + break; + } + } + + if (!foundBuffer) { + ALOGW("returned buffer was not found in the current buffer list"); + } + + for (int id = 0; id < BufferQueue::NUM_BUFFER_SLOTS; id++) { + if (mSlots[id].mGraphicBuffer == NULL) { + continue; + } + + if (bufferHandle == mSlots[id].mGraphicBuffer->handle) { + ALOGV("Slot %d returned, matches handle = %p", id, + mSlots[id].mGraphicBuffer->handle); + + mConsumer->releaseBuffer(id, mSlots[id].mFrameNumber, + EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, + Fence::NO_FENCE); + + buffer->setObserver(0); + buffer->release(); + + foundBuffer = true; + break; + } + } + + if (!foundBuffer) { + CHECK(!"signalBufferReturned: bogus buffer"); + } + +#if DEBUG_PENDING_BUFFERS + for (size_t i = 0; i < mPendingBuffers.size(); ++i) { + if (mPendingBuffers.itemAt(i) == buffer) { + mPendingBuffers.removeAt(i); + break; + } + } +#endif + + --mNumPendingBuffers; + mMediaBuffersAvailableCondition.broadcast(); +} + +// Part of the BufferQueue::ConsumerListener +void SurfaceMediaSource::onFrameAvailable(const BufferItem& /* item */) { + ALOGV("onFrameAvailable"); + + sp listener; + { // scope for the lock + Mutex::Autolock lock(mMutex); + mFrameAvailableCondition.broadcast(); + listener = mFrameAvailableListener; + } + + if (listener != NULL) { + ALOGV("actually calling onFrameAvailable"); + listener->onFrameAvailable(); + } +} + +// SurfaceMediaSource hijacks this event to assume +// the prodcuer is disconnecting from the BufferQueue +// and that it should stop the recording +void SurfaceMediaSource::onBuffersReleased() { + ALOGV("onBuffersReleased"); + + Mutex::Autolock lock(mMutex); + + mFrameAvailableCondition.signal(); + + for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) { + mSlots[i].mGraphicBuffer = 0; + } +} + +void SurfaceMediaSource::onSidebandStreamChanged() { + ALOG_ASSERT(false, "SurfaceMediaSource can't consume sideband streams"); +} + +} // end of namespace android diff --git a/media/libstagefright/include/media/stagefright/SurfaceMediaSource.h b/media/libstagefright/include/media/stagefright/SurfaceMediaSource.h new file mode 100644 index 0000000000..d49e44cd8a --- /dev/null +++ b/media/libstagefright/include/media/stagefright/SurfaceMediaSource.h @@ -0,0 +1,248 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ANDROID_GUI_SURFACEMEDIASOURCE_H +#define ANDROID_GUI_SURFACEMEDIASOURCE_H + +#include +#include + +#include +#include +#include +#include + +#include + +#include "foundation/ABase.h" + +namespace android { +// ---------------------------------------------------------------------------- + +class String8; +class GraphicBuffer; + +// ASSUMPTIONS +// 1. SurfaceMediaSource is initialized with width*height which +// can never change. However, deqeueue buffer does not currently +// enforce this as in BufferQueue, dequeue can be used by Surface +// which can modify the default width and heght. Also neither the width +// nor height can be 0. +// 2. setSynchronousMode is never used (basically no one should call +// setSynchronousMode(false) +// 3. setCrop, setTransform, setScalingMode should never be used +// 4. queueBuffer returns a filled buffer to the SurfaceMediaSource. In addition, a +// timestamp must be provided for the buffer. The timestamp is in +// nanoseconds, and must be monotonically increasing. Its other semantics +// (zero point, etc) are client-dependent and should be documented by the +// client. +// 5. Once disconnected, SurfaceMediaSource can be reused (can not +// connect again) +// 6. Stop is a hard stop, the last few frames held by the encoder +// may be dropped. It is possible to wait for the buffers to be +// returned (but not implemented) + +#define DEBUG_PENDING_BUFFERS 0 + +class SurfaceMediaSource : public MediaSource, + public MediaBufferObserver, + protected ConsumerListener { +public: + enum { MIN_UNDEQUEUED_BUFFERS = 4}; + + struct FrameAvailableListener : public virtual RefBase { + // onFrameAvailable() is called from queueBuffer() is the FIFO is + // empty. You can use SurfaceMediaSource::getQueuedCount() to + // figure out if there are more frames waiting. + // This is called without any lock held can be called concurrently by + // multiple threads. + virtual void onFrameAvailable() = 0; + }; + + SurfaceMediaSource(uint32_t bufferWidth, uint32_t bufferHeight); + + virtual ~SurfaceMediaSource(); + + // For the MediaSource interface for use by StageFrightRecorder: + virtual status_t start(MetaData *params = NULL); + virtual status_t stop(); + virtual status_t read(MediaBufferBase **buffer, + const ReadOptions *options = NULL); + virtual sp getFormat(); + + // Get / Set the frame rate used for encoding. Default fps = 30 + status_t setFrameRate(int32_t fps) ; + int32_t getFrameRate( ) const; + + // The call for the StageFrightRecorder to tell us that + // it is done using the MediaBuffer data so that its state + // can be set to FREE for dequeuing + virtual void signalBufferReturned(MediaBufferBase* buffer); + // end of MediaSource interface + + // getTimestamp retrieves the timestamp associated with the image + // set by the most recent call to read() + // + // The timestamp is in nanoseconds, and is monotonically increasing. Its + // other semantics (zero point, etc) are source-dependent and should be + // documented by the source. + int64_t getTimestamp(); + + // setFrameAvailableListener sets the listener object that will be notified + // when a new frame becomes available. + void setFrameAvailableListener(const sp& listener); + + // dump our state in a String + void dumpState(String8& result) const; + void dumpState(String8& result, const char* prefix, char* buffer, + size_t SIZE) const; + + // metaDataStoredInVideoBuffers tells the encoder what kind of metadata + // is passed through the buffers. Currently, it is set to ANWBuffer + MetadataBufferType metaDataStoredInVideoBuffers() const; + + sp getProducer() const { return mProducer; } + + // To be called before start() + status_t setMaxAcquiredBufferCount(size_t count); + + // To be called before start() + status_t setUseAbsoluteTimestamps(); + +protected: + + // Implementation of the BufferQueue::ConsumerListener interface. These + // calls are used to notify the Surface of asynchronous events in the + // BufferQueue. + virtual void onFrameAvailable(const BufferItem& item); + + // Used as a hook to BufferQueue::disconnect() + // This is called by the client side when it is done + // TODO: Currently, this also sets mStopped to true which + // is needed for unblocking the encoder which might be + // waiting to read more frames. So if on the client side, + // the same thread supplies the frames and also calls stop + // on the encoder, the client has to call disconnect before + // it calls stop. + // In the case of the camera, + // that need not be required since the thread supplying the + // frames is separate than the one calling stop. + virtual void onBuffersReleased(); + + // SurfaceMediaSource can't handle sideband streams, so this is not expected + // to ever be called. Does nothing. + virtual void onSidebandStreamChanged(); + + static bool isExternalFormat(uint32_t format); + +private: + // A BufferQueue, represented by these interfaces, is the exchange point + // between the producer and this consumer + sp mProducer; + sp mConsumer; + + struct SlotData { + sp mGraphicBuffer; + uint64_t mFrameNumber; + }; + + // mSlots caches GraphicBuffers and frameNumbers from the buffer queue + SlotData mSlots[BufferQueue::NUM_BUFFER_SLOTS]; + + // The permenent width and height of SMS buffers + int mWidth; + int mHeight; + + // mCurrentSlot is the buffer slot index of the buffer that is currently + // being used by buffer consumer + // (e.g. StageFrightRecorder in the case of SurfaceMediaSource or GLTexture + // in the case of Surface). + // It is initialized to INVALID_BUFFER_SLOT, + // indicating that no buffer slot is currently bound to the texture. Note, + // however, that a value of INVALID_BUFFER_SLOT does not necessarily mean + // that no buffer is bound to the texture. A call to setBufferCount will + // reset mCurrentTexture to INVALID_BUFFER_SLOT. + int mCurrentSlot; + + // mCurrentBuffers is a list of the graphic buffers that are being used by + // buffer consumer (i.e. the video encoder). It's possible that these + // buffers are not associated with any buffer slots, so we must track them + // separately. Buffers are added to this list in read, and removed from + // this list in signalBufferReturned + Vector > mCurrentBuffers; + + size_t mNumPendingBuffers; + +#if DEBUG_PENDING_BUFFERS + Vector mPendingBuffers; +#endif + + // mCurrentTimestamp is the timestamp for the current texture. It + // gets set to mLastQueuedTimestamp each time updateTexImage is called. + int64_t mCurrentTimestamp; + + // mFrameAvailableListener is the listener object that will be called when a + // new frame becomes available. If it is not NULL it will be called from + // queueBuffer. + sp mFrameAvailableListener; + + // mMutex is the mutex used to prevent concurrent access to the member + // variables of SurfaceMediaSource objects. It must be locked whenever the + // member variables are accessed. + mutable Mutex mMutex; + + ////////////////////////// For MediaSource + // Set to a default of 30 fps if not specified by the client side + int32_t mFrameRate; + + // mStarted is a flag to check if the recording is going on + bool mStarted; + + // mNumFramesReceived indicates the number of frames recieved from + // the client side + int mNumFramesReceived; + // mNumFramesEncoded indicates the number of frames passed on to the + // encoder + int mNumFramesEncoded; + + // mFirstFrameTimestamp is the timestamp of the first received frame. + // It is used to offset the output timestamps so recording starts at time 0. + int64_t mFirstFrameTimestamp; + // mStartTimeNs is the start time passed into the source at start, used to + // offset timestamps. + int64_t mStartTimeNs; + + size_t mMaxAcquiredBufferCount; + + bool mUseAbsoluteTimestamps; + + // mFrameAvailableCondition condition used to indicate whether there + // is a frame available for dequeuing + Condition mFrameAvailableCondition; + + Condition mMediaBuffersAvailableCondition; + + // Allocate and return a new MediaBuffer and pass the ANW buffer as metadata into it. + void passMetadataBuffer_l(MediaBufferBase **buffer, ANativeWindowBuffer *bufferHandle) const; + + // Avoid copying and equating and default constructor + DISALLOW_EVIL_CONSTRUCTORS(SurfaceMediaSource); +}; + +// ---------------------------------------------------------------------------- +}; // namespace android + +#endif // ANDROID_GUI_SURFACEMEDIASOURCE_H diff --git a/media/libstagefright/tests/Android.bp b/media/libstagefright/tests/Android.bp index e6b67ce283..0e9440b9f4 100644 --- a/media/libstagefright/tests/Android.bp +++ b/media/libstagefright/tests/Android.bp @@ -19,6 +19,46 @@ license { ], } +cc_test { + name: "SurfaceMediaSource_test", + + srcs: [ + "SurfaceMediaSource_test.cpp", + "DummyRecorder.cpp", + ], + + shared_libs: [ + "libEGL", + "libGLESv2", + "libbinder", + "libcutils", + "libgui", + "libmedia", + "libmediaextractor", + "libstagefright", + "libstagefright_foundation", + "libstagefright_omx", + "libsync", + "libui", + "libutils", + "liblog", + ], + + include_dirs: [ + "frameworks/av/media/libstagefright", + "frameworks/av/media/libstagefright/include", + "frameworks/native/include/media/openmax", + "frameworks/native/include/media/hardware", + ], + + cflags: [ + "-Werror", + "-Wall", + ], + + compile_multilib: "32", +} + cc_test { name: "MediaCodecListOverrides_test", diff --git a/media/libstagefright/tests/DummyRecorder.cpp b/media/libstagefright/tests/DummyRecorder.cpp new file mode 100644 index 0000000000..c79e6b1a9c --- /dev/null +++ b/media/libstagefright/tests/DummyRecorder.cpp @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define LOG_TAG "DummyRecorder" +// #define LOG_NDEBUG 0 + +#include +#include +#include "DummyRecorder.h" + +#include + +namespace android { + +// static +void *DummyRecorder::threadWrapper(void *pthis) { + ALOGV("ThreadWrapper: %p", pthis); + DummyRecorder *writer = static_cast(pthis); + writer->readFromSource(); + return NULL; +} + + +status_t DummyRecorder::start() { + ALOGV("Start"); + mStarted = true; + + mSource->start(); + + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + int err = pthread_create(&mThread, &attr, threadWrapper, this); + pthread_attr_destroy(&attr); + + if (err) { + ALOGE("Error creating thread!"); + return -ENODEV; + } + return OK; +} + + +status_t DummyRecorder::stop() { + ALOGV("Stop"); + mStarted = false; + + mSource->stop(); + void *dummy; + pthread_join(mThread, &dummy); + status_t err = static_cast(reinterpret_cast(dummy)); + + ALOGV("Ending the reading thread"); + return err; +} + +// pretend to read the source buffers +void DummyRecorder::readFromSource() { + ALOGV("ReadFromSource"); + if (!mStarted) { + return; + } + + status_t err = OK; + MediaBufferBase *buffer; + ALOGV("A fake writer accessing the frames"); + while (mStarted && (err = mSource->read(&buffer)) == OK){ + // if not getting a valid buffer from source, then exit + if (buffer == NULL) { + return; + } + buffer->release(); + buffer = NULL; + } +} + + +} // end of namespace android diff --git a/media/libstagefright/tests/DummyRecorder.h b/media/libstagefright/tests/DummyRecorder.h new file mode 100644 index 0000000000..075977784f --- /dev/null +++ b/media/libstagefright/tests/DummyRecorder.h @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DUMMY_RECORDER_H_ +#define DUMMY_RECORDER_H_ + +#include +#include +#include + + +namespace android { + +struct MediaSource; +class MediaBuffer; + +class DummyRecorder { + public: + // The media source from which this will receive frames + sp mSource; + bool mStarted; + pthread_t mThread; + + status_t start(); + status_t stop(); + + // actual entry point for the thread + void readFromSource(); + + // static function to wrap the actual thread entry point + static void *threadWrapper(void *pthis); + + explicit DummyRecorder(const sp &source) : mSource(source) + , mStarted(false) {} + ~DummyRecorder( ) {} + + private: + + DISALLOW_EVIL_CONSTRUCTORS(DummyRecorder); +}; + +} // end of namespace android +#endif + + diff --git a/media/libstagefright/tests/SurfaceMediaSource_test.cpp b/media/libstagefright/tests/SurfaceMediaSource_test.cpp new file mode 100644 index 0000000000..1b1c3b8cdb --- /dev/null +++ b/media/libstagefright/tests/SurfaceMediaSource_test.cpp @@ -0,0 +1,944 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "SurfaceMediaSource_test" + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include "DummyRecorder.h" + + +namespace android { + +class GLTest : public ::testing::Test { +protected: + + GLTest(): + mEglDisplay(EGL_NO_DISPLAY), + mEglSurface(EGL_NO_SURFACE), + mEglContext(EGL_NO_CONTEXT) { + } + + virtual void SetUp() { + ALOGV("GLTest::SetUp()"); + mEglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + ASSERT_NE(EGL_NO_DISPLAY, mEglDisplay); + + EGLint majorVersion; + EGLint minorVersion; + EXPECT_TRUE(eglInitialize(mEglDisplay, &majorVersion, &minorVersion)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + RecordProperty("EglVersionMajor", majorVersion); + RecordProperty("EglVersionMajor", minorVersion); + + EGLint numConfigs = 0; + EXPECT_TRUE(eglChooseConfig(mEglDisplay, getConfigAttribs(), &mGlConfig, + 1, &numConfigs)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + + char* displaySecsEnv = getenv("GLTEST_DISPLAY_SECS"); + if (displaySecsEnv != NULL) { + mDisplaySecs = atoi(displaySecsEnv); + if (mDisplaySecs < 0) { + mDisplaySecs = 0; + } + } else { + mDisplaySecs = 0; + } + + if (mDisplaySecs > 0) { + mComposerClient = new SurfaceComposerClient; + ASSERT_EQ(NO_ERROR, mComposerClient->initCheck()); + + mSurfaceControl = mComposerClient->createSurface( + String8("Test Surface"), + getSurfaceWidth(), getSurfaceHeight(), + PIXEL_FORMAT_RGB_888, 0); + + ASSERT_TRUE(mSurfaceControl != NULL); + ASSERT_TRUE(mSurfaceControl->isValid()); + + SurfaceComposerClient::Transaction{} + .setLayer(mSurfaceControl, 0x7FFFFFFF) + .show(mSurfaceControl) + .apply(); + + sp window = mSurfaceControl->getSurface(); + mEglSurface = eglCreateWindowSurface(mEglDisplay, mGlConfig, + window.get(), NULL); + } else { + ALOGV("No actual display. Choosing EGLSurface based on SurfaceMediaSource"); + sp sms = (new SurfaceMediaSource( + getSurfaceWidth(), getSurfaceHeight()))->getProducer(); + sp stc = new Surface(sms); + sp window = stc; + + mEglSurface = eglCreateWindowSurface(mEglDisplay, mGlConfig, + window.get(), NULL); + } + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + ASSERT_NE(EGL_NO_SURFACE, mEglSurface); + + mEglContext = eglCreateContext(mEglDisplay, mGlConfig, EGL_NO_CONTEXT, + getContextAttribs()); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + ASSERT_NE(EGL_NO_CONTEXT, mEglContext); + + EXPECT_TRUE(eglMakeCurrent(mEglDisplay, mEglSurface, mEglSurface, + mEglContext)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + + EGLint w, h; + EXPECT_TRUE(eglQuerySurface(mEglDisplay, mEglSurface, EGL_WIDTH, &w)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + EXPECT_TRUE(eglQuerySurface(mEglDisplay, mEglSurface, EGL_HEIGHT, &h)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + RecordProperty("EglSurfaceWidth", w); + RecordProperty("EglSurfaceHeight", h); + + glViewport(0, 0, w, h); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + } + + virtual void TearDown() { + // Display the result + if (mDisplaySecs > 0 && mEglSurface != EGL_NO_SURFACE) { + eglSwapBuffers(mEglDisplay, mEglSurface); + sleep(mDisplaySecs); + } + + if (mComposerClient != NULL) { + mComposerClient->dispose(); + } + if (mEglContext != EGL_NO_CONTEXT) { + eglDestroyContext(mEglDisplay, mEglContext); + } + if (mEglSurface != EGL_NO_SURFACE) { + eglDestroySurface(mEglDisplay, mEglSurface); + } + if (mEglDisplay != EGL_NO_DISPLAY) { + eglTerminate(mEglDisplay); + } + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + } + + virtual EGLint const* getConfigAttribs() { + ALOGV("GLTest getConfigAttribs"); + static EGLint sDefaultConfigAttribs[] = { + EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, + EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_RED_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_BLUE_SIZE, 8, + EGL_ALPHA_SIZE, 8, + EGL_DEPTH_SIZE, 16, + EGL_STENCIL_SIZE, 8, + EGL_NONE }; + + return sDefaultConfigAttribs; + } + + virtual EGLint const* getContextAttribs() { + static EGLint sDefaultContextAttribs[] = { + EGL_CONTEXT_CLIENT_VERSION, 2, + EGL_NONE }; + + return sDefaultContextAttribs; + } + + virtual EGLint getSurfaceWidth() { + return 512; + } + + virtual EGLint getSurfaceHeight() { + return 512; + } + + void loadShader(GLenum shaderType, const char* pSource, GLuint* outShader) { + GLuint shader = glCreateShader(shaderType); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + if (shader) { + glShaderSource(shader, 1, &pSource, NULL); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + glCompileShader(shader); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + GLint compiled = 0; + glGetShaderiv(shader, GL_COMPILE_STATUS, &compiled); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + if (!compiled) { + GLint infoLen = 0; + glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &infoLen); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + if (infoLen) { + char* buf = (char*) malloc(infoLen); + if (buf) { + glGetShaderInfoLog(shader, infoLen, NULL, buf); + printf("Shader compile log:\n%s\n", buf); + free(buf); + FAIL(); + } + } else { + char* buf = (char*) malloc(0x1000); + if (buf) { + glGetShaderInfoLog(shader, 0x1000, NULL, buf); + printf("Shader compile log:\n%s\n", buf); + free(buf); + FAIL(); + } + } + glDeleteShader(shader); + shader = 0; + } + } + ASSERT_TRUE(shader != 0); + *outShader = shader; + } + + void createProgram(const char* pVertexSource, const char* pFragmentSource, + GLuint* outPgm) { + GLuint vertexShader, fragmentShader; + { + SCOPED_TRACE("compiling vertex shader"); + loadShader(GL_VERTEX_SHADER, pVertexSource, &vertexShader); + if (HasFatalFailure()) { + return; + } + } + { + SCOPED_TRACE("compiling fragment shader"); + loadShader(GL_FRAGMENT_SHADER, pFragmentSource, &fragmentShader); + if (HasFatalFailure()) { + return; + } + } + + GLuint program = glCreateProgram(); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + if (program) { + glAttachShader(program, vertexShader); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + glAttachShader(program, fragmentShader); + ASSERT_EQ(GLenum(GL_NO_ERROR), glGetError()); + glLinkProgram(program); + GLint linkStatus = GL_FALSE; + glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); + if (linkStatus != GL_TRUE) { + GLint bufLength = 0; + glGetProgramiv(program, GL_INFO_LOG_LENGTH, &bufLength); + if (bufLength) { + char* buf = (char*) malloc(bufLength); + if (buf) { + glGetProgramInfoLog(program, bufLength, NULL, buf); + printf("Program link log:\n%s\n", buf); + free(buf); + FAIL(); + } + } + glDeleteProgram(program); + program = 0; + } + } + glDeleteShader(vertexShader); + glDeleteShader(fragmentShader); + ASSERT_TRUE(program != 0); + *outPgm = program; + } + + static int abs(int value) { + return value > 0 ? value : -value; + } + + ::testing::AssertionResult checkPixel(int x, int y, int r, + int g, int b, int a, int tolerance=2) { + GLubyte pixel[4]; + String8 msg; + glReadPixels(x, y, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, pixel); + GLenum err = glGetError(); + if (err != GL_NO_ERROR) { + msg += String8::format("error reading pixel: %#x", err); + while ((err = glGetError()) != GL_NO_ERROR) { + msg += String8::format(", %#x", err); + } + fprintf(stderr, "pixel check failure: %s\n", msg.string()); + return ::testing::AssertionFailure( + ::testing::Message(msg.string())); + } + if (r >= 0 && abs(r - int(pixel[0])) > tolerance) { + msg += String8::format("r(%d isn't %d)", pixel[0], r); + } + if (g >= 0 && abs(g - int(pixel[1])) > tolerance) { + if (!msg.isEmpty()) { + msg += " "; + } + msg += String8::format("g(%d isn't %d)", pixel[1], g); + } + if (b >= 0 && abs(b - int(pixel[2])) > tolerance) { + if (!msg.isEmpty()) { + msg += " "; + } + msg += String8::format("b(%d isn't %d)", pixel[2], b); + } + if (a >= 0 && abs(a - int(pixel[3])) > tolerance) { + if (!msg.isEmpty()) { + msg += " "; + } + msg += String8::format("a(%d isn't %d)", pixel[3], a); + } + if (!msg.isEmpty()) { + fprintf(stderr, "pixel check failure: %s\n", msg.string()); + return ::testing::AssertionFailure( + ::testing::Message(msg.string())); + } else { + return ::testing::AssertionSuccess(); + } + } + + int mDisplaySecs; + sp mComposerClient; + sp mSurfaceControl; + + EGLDisplay mEglDisplay; + EGLSurface mEglSurface; + EGLContext mEglContext; + EGLConfig mGlConfig; +}; + +/////////////////////////////////////////////////////////////////////// +// Class for the NON-GL tests +/////////////////////////////////////////////////////////////////////// +class SurfaceMediaSourceTest : public ::testing::Test { +public: + + SurfaceMediaSourceTest( ): mYuvTexWidth(176), mYuvTexHeight(144) { } + void oneBufferPass(int width, int height ); + void oneBufferPassNoFill(int width, int height ); + static void fillYV12Buffer(uint8_t* buf, int w, int h, int stride) ; + static void fillYV12BufferRect(uint8_t* buf, int w, int h, + int stride, const android_native_rect_t& rect) ; +protected: + + virtual void SetUp() { + android::ProcessState::self()->startThreadPool(); + mSMS = new SurfaceMediaSource(mYuvTexWidth, mYuvTexHeight); + mSTC = new Surface(mSMS->getProducer()); + mANW = mSTC; + } + + virtual void TearDown() { + mSMS.clear(); + mSTC.clear(); + mANW.clear(); + } + + const int mYuvTexWidth; + const int mYuvTexHeight; + + sp mSMS; + sp mSTC; + sp mANW; +}; + +/////////////////////////////////////////////////////////////////////// +// Class for the GL tests +/////////////////////////////////////////////////////////////////////// +class SurfaceMediaSourceGLTest : public GLTest { +public: + + SurfaceMediaSourceGLTest( ): mYuvTexWidth(176), mYuvTexHeight(144) { } + virtual EGLint const* getConfigAttribs(); + void oneBufferPassGL(int num = 0); + static sp setUpMediaRecorder(int fileDescriptor, int videoSource, + int outputFormat, int videoEncoder, int width, int height, int fps); +protected: + + virtual void SetUp() { + ALOGV("SMS-GLTest::SetUp()"); + android::ProcessState::self()->startThreadPool(); + mSMS = new SurfaceMediaSource(mYuvTexWidth, mYuvTexHeight); + mSTC = new Surface(mSMS->getProducer()); + mANW = mSTC; + + // Doing the setup related to the GL Side + GLTest::SetUp(); + } + + virtual void TearDown() { + mSMS.clear(); + mSTC.clear(); + mANW.clear(); + GLTest::TearDown(); + } + + void setUpEGLSurfaceFromMediaRecorder(sp& mr); + + const int mYuvTexWidth; + const int mYuvTexHeight; + + sp mSMS; + sp mSTC; + sp mANW; +}; + +///////////////////////////////////////////////////////////////////// +// Methods in SurfaceMediaSourceGLTest +///////////////////////////////////////////////////////////////////// +EGLint const* SurfaceMediaSourceGLTest::getConfigAttribs() { + ALOGV("SurfaceMediaSourceGLTest getConfigAttribs"); + static EGLint sDefaultConfigAttribs[] = { + EGL_SURFACE_TYPE, EGL_WINDOW_BIT, + EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_RED_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_BLUE_SIZE, 8, + EGL_RECORDABLE_ANDROID, EGL_TRUE, + EGL_NONE }; + + return sDefaultConfigAttribs; +} + +// One pass of dequeuing and queuing a GLBuffer +void SurfaceMediaSourceGLTest::oneBufferPassGL(int num) { + int d = num % 50; + float f = 0.2f; // 0.1f * d; + + glClearColor(0, 0.3, 0, 0.6); + glClear(GL_COLOR_BUFFER_BIT); + + glEnable(GL_SCISSOR_TEST); + glScissor(4 + d, 4 + d, 4, 4); + glClearColor(1.0 - f, f, f, 1.0); + glClear(GL_COLOR_BUFFER_BIT); + + glScissor(24 + d, 48 + d, 4, 4); + glClearColor(f, 1.0 - f, f, 1.0); + glClear(GL_COLOR_BUFFER_BIT); + + glScissor(37 + d, 17 + d, 4, 4); + glClearColor(f, f, 1.0 - f, 1.0); + glClear(GL_COLOR_BUFFER_BIT); + + // The following call dequeues and queues the buffer + eglSwapBuffers(mEglDisplay, mEglSurface); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + glDisable(GL_SCISSOR_TEST); +} + +// Set up the MediaRecorder which runs in the same process as mediaserver +sp SurfaceMediaSourceGLTest::setUpMediaRecorder(int fd, int videoSource, + int outputFormat, int videoEncoder, int width, int height, int fps) { + sp mr = new MediaRecorder(String16()); + mr->setVideoSource(videoSource); + mr->setOutputFormat(outputFormat); + mr->setVideoEncoder(videoEncoder); + mr->setOutputFile(fd); + mr->setVideoSize(width, height); + mr->setVideoFrameRate(fps); + mr->prepare(); + ALOGV("Starting MediaRecorder..."); + CHECK_EQ((status_t)OK, mr->start()); + return mr; +} + +// query the mediarecorder for a surfacemeidasource and create an egl surface with that +void SurfaceMediaSourceGLTest::setUpEGLSurfaceFromMediaRecorder(sp& mr) { + sp iST = mr->querySurfaceMediaSourceFromMediaServer(); + mSTC = new Surface(iST); + mANW = mSTC; + + if (mEglSurface != EGL_NO_SURFACE) { + EXPECT_TRUE(eglDestroySurface(mEglDisplay, mEglSurface)); + mEglSurface = EGL_NO_SURFACE; + } + mEglSurface = eglCreateWindowSurface(mEglDisplay, mGlConfig, + mANW.get(), NULL); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + ASSERT_NE(EGL_NO_SURFACE, mEglSurface) ; + + EXPECT_TRUE(eglMakeCurrent(mEglDisplay, mEglSurface, mEglSurface, + mEglContext)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); +} + + +///////////////////////////////////////////////////////////////////// +// Methods in SurfaceMediaSourceTest +///////////////////////////////////////////////////////////////////// + +// One pass of dequeuing and queuing the buffer. Fill it in with +// cpu YV12 buffer +void SurfaceMediaSourceTest::oneBufferPass(int width, int height ) { + ANativeWindowBuffer* anb; + ASSERT_EQ(NO_ERROR, native_window_dequeue_buffer_and_wait(mANW.get(), &anb)); + ASSERT_TRUE(anb != NULL); + + + // Fill the buffer with the a checkerboard pattern + uint8_t* img = NULL; + sp buf(GraphicBuffer::from(anb)); + buf->lock(GRALLOC_USAGE_SW_WRITE_OFTEN, (void**)(&img)); + SurfaceMediaSourceTest::fillYV12Buffer(img, width, height, buf->getStride()); + buf->unlock(); + + ASSERT_EQ(NO_ERROR, mANW->queueBuffer(mANW.get(), buf->getNativeBuffer(), + -1)); +} + +// Dequeuing and queuing the buffer without really filling it in. +void SurfaceMediaSourceTest::oneBufferPassNoFill( + int /* width */, int /* height */) { + ANativeWindowBuffer* anb; + ASSERT_EQ(NO_ERROR, native_window_dequeue_buffer_and_wait(mANW.get(), &anb)); + ASSERT_TRUE(anb != NULL); + + // We do not fill the buffer in. Just queue it back. + sp buf(GraphicBuffer::from(anb)); + ASSERT_EQ(NO_ERROR, mANW->queueBuffer(mANW.get(), buf->getNativeBuffer(), + -1)); +} + +// Fill a YV12 buffer with a multi-colored checkerboard pattern +void SurfaceMediaSourceTest::fillYV12Buffer(uint8_t* buf, int w, int h, int stride) { + const int blockWidth = w > 16 ? w / 16 : 1; + const int blockHeight = h > 16 ? h / 16 : 1; + const int yuvTexOffsetY = 0; + int yuvTexStrideY = stride; + int yuvTexOffsetV = yuvTexStrideY * h; + int yuvTexStrideV = (yuvTexStrideY/2 + 0xf) & ~0xf; + int yuvTexOffsetU = yuvTexOffsetV + yuvTexStrideV * h/2; + int yuvTexStrideU = yuvTexStrideV; + for (int x = 0; x < w; x++) { + for (int y = 0; y < h; y++) { + int parityX = (x / blockWidth) & 1; + int parityY = (y / blockHeight) & 1; + unsigned char intensity = (parityX ^ parityY) ? 63 : 191; + buf[yuvTexOffsetY + (y * yuvTexStrideY) + x] = intensity; + if (x < w / 2 && y < h / 2) { + buf[yuvTexOffsetU + (y * yuvTexStrideU) + x] = intensity; + if (x * 2 < w / 2 && y * 2 < h / 2) { + buf[yuvTexOffsetV + (y*2 * yuvTexStrideV) + x*2 + 0] = + buf[yuvTexOffsetV + (y*2 * yuvTexStrideV) + x*2 + 1] = + buf[yuvTexOffsetV + ((y*2+1) * yuvTexStrideV) + x*2 + 0] = + buf[yuvTexOffsetV + ((y*2+1) * yuvTexStrideV) + x*2 + 1] = + intensity; + } + } + } + } +} + +// Fill a YV12 buffer with red outside a given rectangle and green inside it. +void SurfaceMediaSourceTest::fillYV12BufferRect(uint8_t* buf, int w, + int h, int stride, const android_native_rect_t& rect) { + const int yuvTexOffsetY = 0; + int yuvTexStrideY = stride; + int yuvTexOffsetV = yuvTexStrideY * h; + int yuvTexStrideV = (yuvTexStrideY/2 + 0xf) & ~0xf; + int yuvTexOffsetU = yuvTexOffsetV + yuvTexStrideV * h/2; + int yuvTexStrideU = yuvTexStrideV; + for (int x = 0; x < w; x++) { + for (int y = 0; y < h; y++) { + bool inside = rect.left <= x && x < rect.right && + rect.top <= y && y < rect.bottom; + buf[yuvTexOffsetY + (y * yuvTexStrideY) + x] = inside ? 240 : 64; + if (x < w / 2 && y < h / 2) { + bool inside = rect.left <= 2*x && 2*x < rect.right && + rect.top <= 2*y && 2*y < rect.bottom; + buf[yuvTexOffsetU + (y * yuvTexStrideU) + x] = 16; + buf[yuvTexOffsetV + (y * yuvTexStrideV) + x] = + inside ? 16 : 255; + } + } + } +} ///////// End of class SurfaceMediaSourceTest + +/////////////////////////////////////////////////////////////////// +// Class to imitate the recording ///////////////////////////// +// //////////////////////////////////////////////////////////////// +struct SimpleDummyRecorder { + sp mSource; + + explicit SimpleDummyRecorder + (const sp &source): mSource(source) {} + + status_t start() { return mSource->start();} + status_t stop() { return mSource->stop();} + + // fakes reading from a media source + status_t readFromSource() { + MediaBufferBase *buffer; + status_t err = mSource->read(&buffer); + if (err != OK) { + return err; + } + buffer->release(); + buffer = NULL; + return OK; + } +}; +/////////////////////////////////////////////////////////////////// +// TESTS +// SurfaceMediaSourceTest class contains tests that fill the buffers +// using the cpu calls +// SurfaceMediaSourceGLTest class contains tests that fill the buffers +// using the GL calls. +// TODO: None of the tests actually verify the encoded images.. so at this point, +// these are mostly functionality tests + visual inspection +////////////////////////////////////////////////////////////////////// + +// Just pass one buffer from the native_window to the SurfaceMediaSource +// Dummy Encoder +static int testId = 1; +TEST_F(SurfaceMediaSourceTest, DISABLED_DummyEncodingFromCpuFilledYV12BufferNpotOneBufferPass) { + ALOGV("Test # %d", testId++); + ALOGV("Testing OneBufferPass ******************************"); + + ASSERT_EQ(NO_ERROR, native_window_set_buffers_format(mANW.get(), + HAL_PIXEL_FORMAT_YV12)); + oneBufferPass(mYuvTexWidth, mYuvTexHeight); +} + +// Pass the buffer with the wrong height and weight and should not be accepted +// Dummy Encoder +TEST_F(SurfaceMediaSourceTest, DISABLED_DummyEncodingFromCpuFilledYV12BufferNpotWrongSizeBufferPass) { + ALOGV("Test # %d", testId++); + ALOGV("Testing Wrong size BufferPass ******************************"); + + // setting the client side buffer size different than the server size + ASSERT_EQ(NO_ERROR, native_window_set_buffers_dimensions(mANW.get(), + 10, 10)); + ASSERT_EQ(NO_ERROR, native_window_set_buffers_format(mANW.get(), + HAL_PIXEL_FORMAT_YV12)); + + ANativeWindowBuffer* anb; + + // Note: make sure we get an ERROR back when dequeuing! + ASSERT_NE(NO_ERROR, native_window_dequeue_buffer_and_wait(mANW.get(), &anb)); +} + +// pass multiple buffers from the native_window the SurfaceMediaSource +// Dummy Encoder +TEST_F(SurfaceMediaSourceTest, DISABLED_DummyEncodingFromCpuFilledYV12BufferNpotMultiBufferPass) { + ALOGV("Test # %d", testId++); + ALOGV("Testing MultiBufferPass, Dummy Recorder *********************"); + ASSERT_EQ(NO_ERROR, native_window_set_buffers_format(mANW.get(), + HAL_PIXEL_FORMAT_YV12)); + + SimpleDummyRecorder writer(mSMS); + writer.start(); + + int32_t nFramesCount = 0; + while (nFramesCount < 300) { + oneBufferPass(mYuvTexWidth, mYuvTexHeight); + + ASSERT_EQ(NO_ERROR, writer.readFromSource()); + + nFramesCount++; + } + writer.stop(); +} + +// Delayed pass of multiple buffers from the native_window the SurfaceMediaSource +// Dummy Encoder +TEST_F(SurfaceMediaSourceTest, DummyLagEncodingFromCpuFilledYV12BufferNpotMultiBufferPass) { + ALOGV("Test # %d", testId++); + ALOGV("Testing MultiBufferPass, Dummy Recorder Lagging **************"); + + ASSERT_EQ(NO_ERROR, native_window_set_buffers_format(mANW.get(), + HAL_PIXEL_FORMAT_YV12)); + + SimpleDummyRecorder writer(mSMS); + writer.start(); + + int32_t nFramesCount = 1; + const int FRAMES_LAG = SurfaceMediaSource::MIN_UNDEQUEUED_BUFFERS; + + while (nFramesCount <= 300) { + ALOGV("Frame: %d", nFramesCount); + oneBufferPass(mYuvTexWidth, mYuvTexHeight); + // Forcing the writer to lag behind a few frames + if (nFramesCount > FRAMES_LAG) { + ASSERT_EQ(NO_ERROR, writer.readFromSource()); + } + nFramesCount++; + } + writer.stop(); +} + +// pass multiple buffers from the native_window the SurfaceMediaSource +// A dummy writer (MULTITHREADED) is used to simulate actual MPEG4Writer +TEST_F(SurfaceMediaSourceTest, DummyThreadedEncodingFromCpuFilledYV12BufferNpotMultiBufferPass) { + ALOGV("Test # %d", testId++); + ALOGV("Testing MultiBufferPass, Dummy Recorder Multi-Threaded **********"); + ASSERT_EQ(NO_ERROR, native_window_set_buffers_format(mANW.get(), + HAL_PIXEL_FORMAT_YV12)); + + DummyRecorder writer(mSMS); + writer.start(); + + int32_t nFramesCount = 0; + while (nFramesCount <= 300) { + ALOGV("Frame: %d", nFramesCount); + oneBufferPass(mYuvTexWidth, mYuvTexHeight); + + nFramesCount++; + } + writer.stop(); +} + +// Test to examine actual encoding using mediarecorder +// We use the mediaserver to create a mediarecorder and send +// it back to us. So SurfaceMediaSource lives in the same process +// as the mediaserver. +// Very close to the actual camera, except that the +// buffers are filled and queueud by the CPU instead of GL. +TEST_F(SurfaceMediaSourceTest, DISABLED_EncodingFromCpuYV12BufferNpotWriteMediaServer) { + ALOGV("Test # %d", testId++); + ALOGV("************** Testing the whole pipeline with actual MediaRecorder ***********"); + ALOGV("************** SurfaceMediaSource is same process as mediaserver ***********"); + + const char *fileName = "/sdcard/outputSurfEncMSource.mp4"; + int fd = open(fileName, O_RDWR | O_CREAT, 0744); + if (fd < 0) { + ALOGE("ERROR: Could not open the the file %s, fd = %d !!", fileName, fd); + } + CHECK(fd >= 0); + + sp mr = SurfaceMediaSourceGLTest::setUpMediaRecorder(fd, + VIDEO_SOURCE_SURFACE, OUTPUT_FORMAT_MPEG_4, VIDEO_ENCODER_H264, + mYuvTexWidth, mYuvTexHeight, 30); + // get the reference to the surfacemediasource living in + // mediaserver that is created by stagefrightrecorder + sp iST = mr->querySurfaceMediaSourceFromMediaServer(); + mSTC = new Surface(iST); + mANW = mSTC; + ASSERT_EQ(NO_ERROR, native_window_api_connect(mANW.get(), NATIVE_WINDOW_API_CPU)); + ASSERT_EQ(NO_ERROR, native_window_set_buffers_format(mANW.get(), + HAL_PIXEL_FORMAT_YV12)); + + int32_t nFramesCount = 0; + while (nFramesCount <= 300) { + oneBufferPassNoFill(mYuvTexWidth, mYuvTexHeight); + nFramesCount++; + ALOGV("framesCount = %d", nFramesCount); + } + + ASSERT_EQ(NO_ERROR, native_window_api_disconnect(mANW.get(), NATIVE_WINDOW_API_CPU)); + ALOGV("Stopping MediaRecorder..."); + CHECK_EQ((status_t)OK, mr->stop()); + mr.clear(); + close(fd); +} + +////////////////////////////////////////////////////////////////////// +// GL tests +///////////////////////////////////////////////////////////////////// + +// Test to examine whether we can choose the Recordable Android GLConfig +// DummyRecorder used- no real encoding here +TEST_F(SurfaceMediaSourceGLTest, ChooseAndroidRecordableEGLConfigDummyWriter) { + ALOGV("Test # %d", testId++); + ALOGV("Verify creating a surface w/ right config + dummy writer*********"); + + mSMS = new SurfaceMediaSource(mYuvTexWidth, mYuvTexHeight); + mSTC = new Surface(mSMS->getProducer()); + mANW = mSTC; + + DummyRecorder writer(mSMS); + writer.start(); + + if (mEglSurface != EGL_NO_SURFACE) { + EXPECT_TRUE(eglDestroySurface(mEglDisplay, mEglSurface)); + mEglSurface = EGL_NO_SURFACE; + } + + mEglSurface = eglCreateWindowSurface(mEglDisplay, mGlConfig, + mANW.get(), NULL); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + ASSERT_NE(EGL_NO_SURFACE, mEglSurface) ; + + EXPECT_TRUE(eglMakeCurrent(mEglDisplay, mEglSurface, mEglSurface, + mEglContext)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + + int32_t nFramesCount = 0; + while (nFramesCount <= 300) { + oneBufferPassGL(); + nFramesCount++; + ALOGV("framesCount = %d", nFramesCount); + } + + EXPECT_TRUE(eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, + EGL_NO_CONTEXT)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + eglDestroySurface(mEglDisplay, mEglSurface); + mEglSurface = EGL_NO_SURFACE; + + writer.stop(); +} +// Test to examine whether we can render GL buffers in to the surface +// created with the native window handle +TEST_F(SurfaceMediaSourceGLTest, RenderingToRecordableEGLSurfaceWorks) { + ALOGV("Test # %d", testId++); + ALOGV("RenderingToRecordableEGLSurfaceWorks *********************"); + // Do the producer side of things + glClearColor(0.6, 0.6, 0.6, 0.6); + glClear(GL_COLOR_BUFFER_BIT); + + glEnable(GL_SCISSOR_TEST); + glScissor(4, 4, 4, 4); + glClearColor(1.0, 0.0, 0.0, 1.0); + glClear(GL_COLOR_BUFFER_BIT); + + glScissor(24, 48, 4, 4); + glClearColor(0.0, 1.0, 0.0, 1.0); + glClear(GL_COLOR_BUFFER_BIT); + + glScissor(37, 17, 4, 4); + glClearColor(0.0, 0.0, 1.0, 1.0); + glClear(GL_COLOR_BUFFER_BIT); + + EXPECT_TRUE(checkPixel( 0, 0, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(63, 0, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(63, 63, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel( 0, 63, 153, 153, 153, 153)); + + EXPECT_TRUE(checkPixel( 4, 7, 255, 0, 0, 255)); + EXPECT_TRUE(checkPixel(25, 51, 0, 255, 0, 255)); + EXPECT_TRUE(checkPixel(40, 19, 0, 0, 255, 255)); + EXPECT_TRUE(checkPixel(29, 51, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel( 5, 32, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(13, 8, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(46, 3, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(30, 33, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel( 6, 52, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(55, 33, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(16, 29, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel( 1, 30, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(41, 37, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(46, 29, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel(15, 25, 153, 153, 153, 153)); + EXPECT_TRUE(checkPixel( 3, 52, 153, 153, 153, 153)); +} + +// Test to examine the actual encoding with GL buffers +// Actual encoder, Actual GL Buffers Filled SurfaceMediaSource +// The same pattern is rendered every frame +TEST_F(SurfaceMediaSourceGLTest, EncodingFromGLRgbaSameImageEachBufNpotWrite) { + ALOGV("Test # %d", testId++); + ALOGV("************** Testing the whole pipeline with actual Recorder ***********"); + ALOGV("************** GL Filling the buffers ***********"); + // Note: No need to set the colorformat for the buffers. The colorformat is + // in the GRAlloc buffers itself. + + const char *fileName = "/sdcard/outputSurfEncMSourceGL.mp4"; + int fd = open(fileName, O_RDWR | O_CREAT, 0744); + if (fd < 0) { + ALOGE("ERROR: Could not open the the file %s, fd = %d !!", fileName, fd); + } + CHECK(fd >= 0); + + sp mr = setUpMediaRecorder(fd, VIDEO_SOURCE_SURFACE, + OUTPUT_FORMAT_MPEG_4, VIDEO_ENCODER_H264, mYuvTexWidth, mYuvTexHeight, 30); + + // get the reference to the surfacemediasource living in + // mediaserver that is created by stagefrightrecorder + setUpEGLSurfaceFromMediaRecorder(mr); + + int32_t nFramesCount = 0; + while (nFramesCount <= 300) { + oneBufferPassGL(); + nFramesCount++; + ALOGV("framesCount = %d", nFramesCount); + } + + EXPECT_TRUE(eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, + EGL_NO_CONTEXT)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + eglDestroySurface(mEglDisplay, mEglSurface); + mEglSurface = EGL_NO_SURFACE; + + ALOGV("Stopping MediaRecorder..."); + CHECK_EQ((status_t)OK, mr->stop()); + mr.clear(); + close(fd); +} + +// Test to examine the actual encoding from the GL Buffers +// Actual encoder, Actual GL Buffers Filled SurfaceMediaSource +// A different pattern is rendered every frame +TEST_F(SurfaceMediaSourceGLTest, EncodingFromGLRgbaDiffImageEachBufNpotWrite) { + ALOGV("Test # %d", testId++); + ALOGV("************** Testing the whole pipeline with actual Recorder ***********"); + ALOGV("************** Diff GL Filling the buffers ***********"); + // Note: No need to set the colorformat for the buffers. The colorformat is + // in the GRAlloc buffers itself. + + const char *fileName = "/sdcard/outputSurfEncMSourceGLDiff.mp4"; + int fd = open(fileName, O_RDWR | O_CREAT, 0744); + if (fd < 0) { + ALOGE("ERROR: Could not open the the file %s, fd = %d !!", fileName, fd); + } + CHECK(fd >= 0); + + sp mr = setUpMediaRecorder(fd, VIDEO_SOURCE_SURFACE, + OUTPUT_FORMAT_MPEG_4, VIDEO_ENCODER_H264, mYuvTexWidth, mYuvTexHeight, 30); + + // get the reference to the surfacemediasource living in + // mediaserver that is created by stagefrightrecorder + setUpEGLSurfaceFromMediaRecorder(mr); + + int32_t nFramesCount = 0; + while (nFramesCount <= 300) { + oneBufferPassGL(nFramesCount); + nFramesCount++; + ALOGV("framesCount = %d", nFramesCount); + } + + EXPECT_TRUE(eglMakeCurrent(mEglDisplay, EGL_NO_SURFACE, EGL_NO_SURFACE, + EGL_NO_CONTEXT)); + ASSERT_EQ(EGL_SUCCESS, eglGetError()); + eglDestroySurface(mEglDisplay, mEglSurface); + mEglSurface = EGL_NO_SURFACE; + + ALOGV("Stopping MediaRecorder..."); + CHECK_EQ((status_t)OK, mr->stop()); + mr.clear(); + close(fd); +} +} // namespace android From 02c20a1e4c74e6716151f3247c5a5595d9f971d0 Mon Sep 17 00:00:00 2001 From: Adithya R Date: Sat, 14 Dec 2019 02:22:56 +0530 Subject: [PATCH 06/20] libstagefright: Remove libmediaextractor dependency Change-Id: Ifb564aafcf6f9bd45010500a589050b6577c0f7a Signed-off-by: DennySPb Signed-off-by: Edwiin Kusuma Jaya --- media/libstagefright/tests/Android.bp | 1 - 1 file changed, 1 deletion(-) diff --git a/media/libstagefright/tests/Android.bp b/media/libstagefright/tests/Android.bp index 0e9440b9f4..08cf42b88c 100644 --- a/media/libstagefright/tests/Android.bp +++ b/media/libstagefright/tests/Android.bp @@ -34,7 +34,6 @@ cc_test { "libcutils", "libgui", "libmedia", - "libmediaextractor", "libstagefright", "libstagefright_foundation", "libstagefright_omx", From cba0c42d81d65286155459ee61ecf90a1de777ad Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Wed, 9 Jan 2019 01:43:08 +0200 Subject: [PATCH 07/20] libstagefright_wfd: compilation fixes * Among others, adapt to the ABuffer API changes in "f03606d9 Move MediaBufferXXX from foundation to libmediaextractor" Change-Id: Ie92fc035c6430f1458d45995a5b2627d0bc75122 Signed-off-by: Vladimir Oltean Signed-off-by: DennySPb libstagefright_wfd: Adapt to A11 changes Change-Id: I69f13c93a68b1ad60e085000361331a3fdc98049 Signed-off-by: DennySPb libstagefright: Remove libmediaextractor dependency Change-Id: Ifb564aafcf6f9bd45010500a589050b6577c0f7a Signed-off-by: DennySPb Remove libmediaextractor dependency from libstagefright_wfd Change-Id: I3d417d2f2ce468eb9d45f55f7818bb9a46348667 Signed-off-by: DennySPb libstagefright_wfd: Log MediaExtractor entry point to MediaMetrics Change-Id: Ic8fac2cd5ee4d3bb59804e69efc1d5c482aa2417 Change-Id: Ie92fc035c6430f1458d45995a5b2627d0bc75122 Signed-off-by: Edwiin Kusuma Jaya --- media/libmedia/Android.bp | 1 + media/libmedia/IHDCP.cpp | 4 +-- media/libstagefright/SurfaceMediaSource.cpp | 4 +-- .../media/stagefright/SurfaceMediaSource.h | 6 ++--- media/libstagefright/tests/DummyRecorder.cpp | 2 +- media/libstagefright/wifi-display/Android.bp | 6 +++++ .../wifi-display/MediaSender.cpp | 5 ++-- .../wifi-display/rtp/RTPSender.cpp | 4 +-- .../wifi-display/source/Converter.cpp | 27 +++++++++++-------- .../wifi-display/source/MediaPuller.cpp | 11 ++++---- .../wifi-display/source/PlaybackSession.cpp | 10 ++++--- .../wifi-display/source/RepeaterSource.cpp | 8 +++--- .../wifi-display/source/RepeaterSource.h | 4 +-- .../wifi-display/source/TSPacketizer.cpp | 6 ++--- .../wifi-display/source/WifiDisplaySource.cpp | 1 + 15 files changed, 58 insertions(+), 41 deletions(-) diff --git a/media/libmedia/Android.bp b/media/libmedia/Android.bp index 11b6d56aa4..5c90e0216e 100644 --- a/media/libmedia/Android.bp +++ b/media/libmedia/Android.bp @@ -372,6 +372,7 @@ cc_library { "libcamera_client", "libstagefright_foundation", "libgui", + "libui", "libdl", "libaudioclient", "libmedia_codeclist", diff --git a/media/libmedia/IHDCP.cpp b/media/libmedia/IHDCP.cpp index a46017ff9d..31e37535c2 100644 --- a/media/libmedia/IHDCP.cpp +++ b/media/libmedia/IHDCP.cpp @@ -55,7 +55,7 @@ struct BpHDCPObserver : public BpInterface { } }; -IMPLEMENT_META_INTERFACE(HDCPObserver, "android.hardware.IHDCPObserver"); +DO_NOT_DIRECTLY_USE_ME_IMPLEMENT_META_INTERFACE(HDCPObserver, "android.hardware.IHDCPObserver"); struct BpHDCP : public BpInterface { explicit BpHDCP(const sp &impl) @@ -166,7 +166,7 @@ struct BpHDCP : public BpInterface { } }; -IMPLEMENT_META_INTERFACE(HDCP, "android.hardware.IHDCP"); +DO_NOT_DIRECTLY_USE_ME_IMPLEMENT_META_INTERFACE(HDCP, "android.hardware.IHDCP"); status_t BnHDCPObserver::onTransact( uint32_t code, const Parcel &data, Parcel *reply, uint32_t flags) { diff --git a/media/libstagefright/SurfaceMediaSource.cpp b/media/libstagefright/SurfaceMediaSource.cpp index 4b3076a30d..333c618fbd 100644 --- a/media/libstagefright/SurfaceMediaSource.cpp +++ b/media/libstagefright/SurfaceMediaSource.cpp @@ -217,7 +217,7 @@ status_t SurfaceMediaSource::stop() #if DEBUG_PENDING_BUFFERS for (size_t i = 0; i < mPendingBuffers.size(); ++i) { - ALOGI("%d: %p", i, mPendingBuffers.itemAt(i)); + ALOGI("%zu: %p", i, mPendingBuffers.itemAt(i)); } #endif @@ -355,7 +355,7 @@ status_t SurfaceMediaSource::read( (*buffer)->setObserver(this); (*buffer)->add_ref(); - (*buffer)->meta_data()->setInt64(kKeyTime, mCurrentTimestamp / 1000); + (*buffer)->meta_data().setInt64(kKeyTime, mCurrentTimestamp / 1000); ALOGV("Frames encoded = %d, timestamp = %" PRId64 ", time diff = %" PRId64, mNumFramesEncoded, mCurrentTimestamp / 1000, mCurrentTimestamp / 1000 - prevTimeStamp / 1000); diff --git a/media/libstagefright/include/media/stagefright/SurfaceMediaSource.h b/media/libstagefright/include/media/stagefright/SurfaceMediaSource.h index d49e44cd8a..c67defe30b 100644 --- a/media/libstagefright/include/media/stagefright/SurfaceMediaSource.h +++ b/media/libstagefright/include/media/stagefright/SurfaceMediaSource.h @@ -22,12 +22,12 @@ #include #include -#include +#include #include #include -#include "foundation/ABase.h" +#include namespace android { // ---------------------------------------------------------------------------- @@ -187,7 +187,7 @@ class SurfaceMediaSource : public MediaSource, size_t mNumPendingBuffers; #if DEBUG_PENDING_BUFFERS - Vector mPendingBuffers; + Vector mPendingBuffers; #endif // mCurrentTimestamp is the timestamp for the current texture. It diff --git a/media/libstagefright/tests/DummyRecorder.cpp b/media/libstagefright/tests/DummyRecorder.cpp index c79e6b1a9c..596b7e6fd7 100644 --- a/media/libstagefright/tests/DummyRecorder.cpp +++ b/media/libstagefright/tests/DummyRecorder.cpp @@ -17,7 +17,7 @@ #define LOG_TAG "DummyRecorder" // #define LOG_NDEBUG 0 -#include +#include #include #include "DummyRecorder.h" diff --git a/media/libstagefright/wifi-display/Android.bp b/media/libstagefright/wifi-display/Android.bp index fb08c5b072..2f1e74943f 100644 --- a/media/libstagefright/wifi-display/Android.bp +++ b/media/libstagefright/wifi-display/Android.bp @@ -26,6 +26,7 @@ cc_library_shared { "libcutils", "liblog", "libmedia", + "libmedia_omx", "libstagefright", "libstagefright_foundation", "libui", @@ -33,6 +34,11 @@ cc_library_shared { "libutils", ], + header_libs: [ + "libmediadrm_headers", + "libmediametrics_headers", + ], + cflags: [ "-Wno-multichar", "-Werror", diff --git a/media/libstagefright/wifi-display/MediaSender.cpp b/media/libstagefright/wifi-display/MediaSender.cpp index cc412f5f46..6f1dbbfc87 100644 --- a/media/libstagefright/wifi-display/MediaSender.cpp +++ b/media/libstagefright/wifi-display/MediaSender.cpp @@ -23,10 +23,9 @@ #include "rtp/RTPSender.h" #include "source/TSPacketizer.h" -#include "include/avc_utils.h" - #include #include +#include #include #include #include @@ -400,7 +399,7 @@ status_t MediaSender::packetizeAccessUnit( bool manuallyPrependSPSPPS = !info.mIsAudio && (info.mFlags & FLAG_MANUALLY_PREPEND_SPS_PPS) - && IsIDR(accessUnit); + && IsIDR(accessUnit->data(), accessUnit->size()); if (mHDCP != NULL && !info.mIsAudio) { isHDCPEncrypted = true; diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.cpp b/media/libstagefright/wifi-display/rtp/RTPSender.cpp index ca9fdd2bd7..bfdd5fc491 100644 --- a/media/libstagefright/wifi-display/rtp/RTPSender.cpp +++ b/media/libstagefright/wifi-display/rtp/RTPSender.cpp @@ -20,16 +20,16 @@ #include "RTPSender.h" +#include #include #include #include #include +#include #include #include #include -#include "include/avc_utils.h" - namespace android { RTPSender::RTPSender( diff --git a/media/libstagefright/wifi-display/source/Converter.cpp b/media/libstagefright/wifi-display/source/Converter.cpp index 273af18d62..5502b063ab 100644 --- a/media/libstagefright/wifi-display/source/Converter.cpp +++ b/media/libstagefright/wifi-display/source/Converter.cpp @@ -21,19 +21,20 @@ #include "Converter.h" #include "MediaPuller.h" -#include "include/avc_utils.h" #include #include -#include +#include #include +#include #include #include #include -#include +#include #include #include #include +#include #include @@ -305,7 +306,7 @@ void Converter::onMessageReceived(const sp &msg) { sp accessUnit; CHECK(msg->findBuffer("accessUnit", &accessUnit)); - accessUnit->setMediaBufferBase(NULL); + accessUnit->meta()->setObject("mediaBufferHolder", sp(nullptr)); } break; } @@ -328,7 +329,7 @@ void Converter::onMessageReceived(const sp &msg) { ALOGI("dropping frame."); } - accessUnit->setMediaBufferBase(NULL); + accessUnit->meta()->setObject("mediaBufferHolder", sp(nullptr)); break; } @@ -625,13 +626,17 @@ status_t Converter::feedEncoderInputBuffers() { buffer->data(), buffer->size()); - MediaBuffer *mediaBuffer = - (MediaBuffer *)(buffer->getMediaBufferBase()); + MediaBufferBase *mediaBuffer = NULL; + sp holder; + + if (buffer->meta()->findObject("mediaBufferHolder", &holder)) { + mediaBuffer = (holder != nullptr) ? + static_cast(holder.get())->mediaBuffer() : nullptr; + } if (mediaBuffer != NULL) { - mEncoderInputBuffers.itemAt(bufferIndex)->setMediaBufferBase( - mediaBuffer); + mEncoderInputBuffers.itemAt(bufferIndex)->meta()->setObject("mediaBufferHolder", new MediaBufferHolder(mediaBuffer)); - buffer->setMediaBufferBase(NULL); + buffer->meta()->setObject("mediaBufferHolder", sp(nullptr)); } } else { flags = MediaCodec::BUFFER_FLAG_EOS; @@ -763,7 +768,7 @@ status_t Converter::doMoreWork() { if (mNeedToManuallyPrependSPSPPS && mIsH264 && (mFlags & FLAG_PREPEND_CSD_IF_NECESSARY) - && IsIDR(buffer)) { + && IsIDR(buffer->data(), buffer->size())) { buffer = prependCSD(buffer); } diff --git a/media/libstagefright/wifi-display/source/MediaPuller.cpp b/media/libstagefright/wifi-display/source/MediaPuller.cpp index ce07a4ec4c..7bdef1de39 100644 --- a/media/libstagefright/wifi-display/source/MediaPuller.cpp +++ b/media/libstagefright/wifi-display/source/MediaPuller.cpp @@ -23,9 +23,10 @@ #include #include #include -#include +#include #include #include +#include namespace android { @@ -138,7 +139,7 @@ void MediaPuller::onMessageReceived(const sp &msg) { break; } - MediaBuffer *mbuf; + MediaBufferBase *mbuf; status_t err = mSource->read(&mbuf); if (mPaused) { @@ -163,7 +164,7 @@ void MediaPuller::onMessageReceived(const sp &msg) { notify->post(); } else { int64_t timeUs; - CHECK(mbuf->meta_data()->findInt64(kKeyTime, &timeUs)); + CHECK(mbuf->meta_data().findInt64(kKeyTime, &timeUs)); sp accessUnit = new ABuffer(mbuf->range_length()); @@ -177,9 +178,9 @@ void MediaPuller::onMessageReceived(const sp &msg) { mbuf->release(); mbuf = NULL; } else { - // video encoder will release MediaBuffer when done + // video encoder will release MediaBufferBase when done // with underlying data. - accessUnit->setMediaBufferBase(mbuf); + accessUnit->meta()->setObject("mediaBufferHolder", new MediaBufferHolder(mbuf)); } sp notify = mNotify->dup(); diff --git a/media/libstagefright/wifi-display/source/PlaybackSession.cpp b/media/libstagefright/wifi-display/source/PlaybackSession.cpp index f1ecca0515..0240d2a552 100644 --- a/media/libstagefright/wifi-display/source/PlaybackSession.cpp +++ b/media/libstagefright/wifi-display/source/PlaybackSession.cpp @@ -23,13 +23,13 @@ #include "Converter.h" #include "MediaPuller.h" #include "RepeaterSource.h" -#include "include/avc_utils.h" #include "WifiDisplaySource.h" #include #include #include #include +#include #include #include #include @@ -43,6 +43,7 @@ #include #include #include +#include #include @@ -747,7 +748,7 @@ void WifiDisplaySource::PlaybackSession::onSinkFeedback(const sp &msg) status_t WifiDisplaySource::PlaybackSession::setupMediaPacketizer( bool enableAudio, bool enableVideo) { - mExtractor = new NuMediaExtractor; + mExtractor = new NuMediaExtractor(NuMediaExtractor::EntryPoint::OTHER); status_t err = mExtractor->setDataSource( NULL /* httpService */, mMediaPath.c_str()); @@ -1069,8 +1070,11 @@ status_t WifiDisplaySource::PlaybackSession::addVideoSource( } status_t WifiDisplaySource::PlaybackSession::addAudioSource(bool usePCMAudio) { + audio_attributes_t attr = AUDIO_ATTRIBUTES_INITIALIZER; + attr.source = AUDIO_SOURCE_REMOTE_SUBMIX; + sp audioSource = new AudioSource( - AUDIO_SOURCE_REMOTE_SUBMIX, + &attr, mOpPackageName, 48000 /* sampleRate */, 2 /* channelCount */); diff --git a/media/libstagefright/wifi-display/source/RepeaterSource.cpp b/media/libstagefright/wifi-display/source/RepeaterSource.cpp index af6b66337d..e225a02773 100644 --- a/media/libstagefright/wifi-display/source/RepeaterSource.cpp +++ b/media/libstagefright/wifi-display/source/RepeaterSource.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include namespace android { @@ -107,7 +107,7 @@ sp RepeaterSource::getFormat() { } status_t RepeaterSource::read( - MediaBuffer **buffer, const ReadOptions *options) { + MediaBufferBase **buffer, const ReadOptions *options) { int64_t seekTimeUs; ReadOptions::SeekMode seekMode; CHECK(options == NULL || !options->getSeekTo(&seekTimeUs, &seekMode)); @@ -155,7 +155,7 @@ status_t RepeaterSource::read( { mBuffer->add_ref(); *buffer = mBuffer; - (*buffer)->meta_data()->setInt64(kKeyTime, bufferTimeUs); + (*buffer)->meta_data().setInt64(kKeyTime, bufferTimeUs); ++mFrameCount; } } @@ -180,7 +180,7 @@ void RepeaterSource::onMessageReceived(const sp &msg) { switch (msg->what()) { case kWhatRead: { - MediaBuffer *buffer; + MediaBufferBase *buffer; status_t err = mSource->read(&buffer); ALOGV("read mbuf %p", buffer); diff --git a/media/libstagefright/wifi-display/source/RepeaterSource.h b/media/libstagefright/wifi-display/source/RepeaterSource.h index 8d414fd538..1647fb1c1f 100644 --- a/media/libstagefright/wifi-display/source/RepeaterSource.h +++ b/media/libstagefright/wifi-display/source/RepeaterSource.h @@ -20,7 +20,7 @@ struct RepeaterSource : public MediaSource { virtual sp getFormat(); virtual status_t read( - MediaBuffer **buffer, const ReadOptions *options); + MediaBufferBase **buffer, const ReadOptions *options); void onMessageReceived(const sp &msg); @@ -50,7 +50,7 @@ struct RepeaterSource : public MediaSource { sp mLooper; sp > mReflector; - MediaBuffer *mBuffer; + MediaBufferBase *mBuffer; status_t mResult; int64_t mLastBufferUpdateUs; diff --git a/media/libstagefright/wifi-display/source/TSPacketizer.cpp b/media/libstagefright/wifi-display/source/TSPacketizer.cpp index 865ba94e14..7d30ae208e 100644 --- a/media/libstagefright/wifi-display/source/TSPacketizer.cpp +++ b/media/libstagefright/wifi-display/source/TSPacketizer.cpp @@ -19,8 +19,8 @@ #include #include "TSPacketizer.h" -#include "include/avc_utils.h" +#include #include #include #include @@ -471,7 +471,7 @@ status_t TSPacketizer::packetize( const sp &track = mTracks.itemAt(trackIndex); if (track->isH264() && (flags & PREPEND_SPS_PPS_TO_IDR_FRAMES) - && IsIDR(accessUnit)) { + && IsIDR(accessUnit->data(), accessUnit->size())) { // prepend codec specific data, i.e. SPS and PPS. accessUnit = track->prependCSD(accessUnit); } else if (track->isAAC() && track->lacksADTSHeader()) { @@ -1039,7 +1039,7 @@ sp TSPacketizer::prependCSD( CHECK_LT(trackIndex, mTracks.size()); const sp &track = mTracks.itemAt(trackIndex); - CHECK(track->isH264() && IsIDR(accessUnit)); + CHECK(track->isH264() && IsIDR(accessUnit->data(), accessUnit->size())); int64_t timeUs; CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs)); diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp index 4695e5d289..159b113dcf 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp @@ -22,6 +22,7 @@ #include "PlaybackSession.h" #include "Parameters.h" #include "rtp/RTPSender.h" +#include #include #include From 76a59fc50b0d62827f544b3bc1c4e12fac90cbf1 Mon Sep 17 00:00:00 2001 From: zhangbo_a Date: Thu, 11 May 2017 11:50:57 +0800 Subject: [PATCH 08/20] stagefright: Fix SurfaceMediaSource getting handle from wrong position issue In function passMetadataBuffer_l, the bufferHandle(ANativeWindowBuffer) is saved to data (VideoNativeMetadata) but in function getMediaBufferHandle it gets the bufferHandle from (MediaBuffer*)buffer->data() + 4, which is a wrong position. To solve this problem, we should get handle from ANativeWindowBuffer, not from buffer->data() + 4. (If get bufferHandle from buffer->data() + 4, the function signalBufferReturned will print "returned buffer was not found in the current list" error. Test: Running wifi display, we can see the handle could be found in buffer list. Change-Id: I71ecf9e2bca1db67d8d6e862ac16b07e939bf521 Signed-off-by: zhangbo_a Signed-off-by: DennySPb Signed-off-by: Edwiin Kusuma Jaya --- media/libstagefright/SurfaceMediaSource.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/media/libstagefright/SurfaceMediaSource.cpp b/media/libstagefright/SurfaceMediaSource.cpp index 333c618fbd..3ba18b6631 100644 --- a/media/libstagefright/SurfaceMediaSource.cpp +++ b/media/libstagefright/SurfaceMediaSource.cpp @@ -375,7 +375,9 @@ static buffer_handle_t getMediaBufferHandle(MediaBufferBase *buffer) { // need to convert to char* for pointer arithmetic and then // copy the byte stream into our handle buffer_handle_t bufferHandle; - memcpy(&bufferHandle, (char*)(buffer->data()) + 4, sizeof(buffer_handle_t)); + VideoNativeMetadata *data = (VideoNativeMetadata *)buffer->data(); + ANativeWindowBuffer *anwbuffer = (ANativeWindowBuffer *)data->pBuffer; + bufferHandle = anwbuffer->handle; return bufferHandle; } From 2c7d0e12e922de601fd3d46b66ebfee3e7accc37 Mon Sep 17 00:00:00 2001 From: "Angelo G. Del Regno" Date: Sun, 30 Apr 2017 20:19:08 +0200 Subject: [PATCH 09/20] stagefright: Fix buffer handle retrieval in signalBufferReturned In commit 3e32878 the stagefright code was restructured to fix the logic for native handle source, but the change in the function SurfaceMediaSource::signalBufferReturned was probably missed. Try to compare the media buffer handle also to the current native buffer handle in this function when searching for correspondance. Change-Id: I352293e525f75dde500ac8e71ee49209710030c3 Signed-off-by: DennySPb Signed-off-by: Edwiin Kusuma Jaya --- media/libstagefright/SurfaceMediaSource.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/media/libstagefright/SurfaceMediaSource.cpp b/media/libstagefright/SurfaceMediaSource.cpp index 3ba18b6631..d7370551e0 100644 --- a/media/libstagefright/SurfaceMediaSource.cpp +++ b/media/libstagefright/SurfaceMediaSource.cpp @@ -389,9 +389,12 @@ void SurfaceMediaSource::signalBufferReturned(MediaBufferBase *buffer) { Mutex::Autolock lock(mMutex); buffer_handle_t bufferHandle = getMediaBufferHandle(buffer); + ANativeWindowBuffer* curNativeHandle = NULL; for (size_t i = 0; i < mCurrentBuffers.size(); i++) { - if (mCurrentBuffers[i]->handle == bufferHandle) { + curNativeHandle = mCurrentBuffers[i]->getNativeBuffer(); + if ((mCurrentBuffers[i]->handle == bufferHandle) || + ((buffer_handle_t)curNativeHandle == bufferHandle)) { mCurrentBuffers.removeAt(i); foundBuffer = true; break; @@ -407,7 +410,10 @@ void SurfaceMediaSource::signalBufferReturned(MediaBufferBase *buffer) { continue; } - if (bufferHandle == mSlots[id].mGraphicBuffer->handle) { + curNativeHandle = mSlots[id].mGraphicBuffer->getNativeBuffer(); + + if ((bufferHandle == mSlots[id].mGraphicBuffer->handle) || + (bufferHandle == (buffer_handle_t)curNativeHandle)) { ALOGV("Slot %d returned, matches handle = %p", id, mSlots[id].mGraphicBuffer->handle); From 61e7ecd4902bc04d0f11d91c31eab2a37b8951eb Mon Sep 17 00:00:00 2001 From: Vladimir Oltean Date: Sun, 20 Jan 2019 22:10:41 +0200 Subject: [PATCH 10/20] libstagefright_wfd: video encoder does not actually release MediaBufferBase when done * This fixes buffer flow SurfaceMediaSource -> MediaPuller -> Converted freezing at mMediaBuffersAvailableCondition.wait(), due to this condition never being broadcast. This was supposed to happen from within SurfaceMediaSource::signalBufferReturned(), but this was never called. The Converter class does feedEncoderInputBuffers(), and after the encoder does its job, it should return the video buffer to the SurfaceMediaSource in ACodec::BaseState::onOMXEmptyBufferDone(). * There (in ACodec class), the code for doing that used to be: // We're in "store-metadata-in-buffers" mode, the underlying // OMX component had access to data that's implicitly refcounted // by this "MediaBuffer" object. Now that the OMX component has // told us that it's done with the input buffer, we can decrement // the mediaBuffer's reference count. info->mData->setMediaBufferBase(NULL); This means that if there was already a MediaBufferBase assigned to this mediaBuffer, then it got released when explicitly setting it to NULL: void MediaCodecBuffer::setMediaBufferBase(MediaBufferBase *mediaBuffer) { if (mMediaBufferBase != NULL) { mMediaBufferBase->release(); } mMediaBufferBase = mediaBuffer; } Then in MediaBuffer::release(), which is a subclass of MediaBufferBase, there is code that does mObserver->signalBufferReturned(this); This should have went on to call SurfaceMediaSource::signalBufferReturned(), as it was setting itself as observer on the buffers sent to the video encoder. Stay tuned to find out why the call path was broken. * Now, after Mr. Dongwon Kang's commit "f03606d9 Move MediaBufferXXX from foundation to libmediaextractor", the setMediaBufferBase and getMediaBufferBase functions no longer exist, and reference counting on MediaBuffer's is different. The direct replacement of setMediaBufferBase(mbuf) is now meta()->setObject("mediaBufferHolder", new MediaBufferHolder(mbuf)). The reference counting seems to now be managed through the constructor and destructor of this new MediaBufferHolder class (the code for release() is now in the holder's destructor). Now the issue seems to be that the lifetime of these new MediaBufferHolder's is not quite what it should be, because their destructor never gets called, hence the buffers never get returned. * This might be an API problem that Mr. Dongwon Kang himself acknowledged, since in the aforementioned patch, he forcefully called mbuf->release() right below a comment where it clearly said that "video encoder will release MediaBuffer when done with underlying data": https://android.googlesource.com/platform/frameworks/av/+/f03606d9034730bea1a394e6803f9ebc36f3d2eb%5E%21/#F13 * Without addressing the root cause of the issue, in this commit we are simply mirroring a workaround for what appears to be broken media buffer reference counting. Change-Id: Ie540e6dcf5536f93091ced2af2e121b71f70bb83 Signed-off-by: Vladimir Oltean Signed-off-by: DennySPb Signed-off-by: Edwiin Kusuma Jaya --- media/libstagefright/wifi-display/source/MediaPuller.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/media/libstagefright/wifi-display/source/MediaPuller.cpp b/media/libstagefright/wifi-display/source/MediaPuller.cpp index 7bdef1de39..5bf893d1a7 100644 --- a/media/libstagefright/wifi-display/source/MediaPuller.cpp +++ b/media/libstagefright/wifi-display/source/MediaPuller.cpp @@ -181,6 +181,8 @@ void MediaPuller::onMessageReceived(const sp &msg) { // video encoder will release MediaBufferBase when done // with underlying data. accessUnit->meta()->setObject("mediaBufferHolder", new MediaBufferHolder(mbuf)); + mbuf->release(); + mbuf = NULL; } sp notify = mNotify->dup(); From debb8019034e229e7ff853a6836d7122ac752c86 Mon Sep 17 00:00:00 2001 From: myfluxi Date: Thu, 16 Apr 2020 12:53:23 +0000 Subject: [PATCH 11/20] audioflinger: Fix audio for WifiDisplay AudioFlinger is not able to determine the correct pid/tid for WifiDisplay and thus we do not pass checks for CAPTURE_AUDIO_OUTPUT and RECORD_AUDIO permissions. To fix audio for WifiDisplay, it should be safe to always allow a trusted calling uid (AID_MEDIA which has the same perms as AID_AUDIOSERVER). Change-Id: Ifa46d8e77a43027645cad02a04263b58e134c3ad Signed-off-by: Edwiin Kusuma Jaya --- services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp index 140cabb808..331aefad5b 100644 --- a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp +++ b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp @@ -615,7 +615,7 @@ Status AudioPolicyService::getInputForAttr(const media::AudioAttributesInternal& // type is API_INPUT_MIX_EXT_POLICY_REROUTE and by AudioService if a media projection // is used and input type is API_INPUT_MIX_PUBLIC_CAPTURE_PLAYBACK // - ECHO_REFERENCE source is controlled by captureAudioOutputAllowed() - if (!(recordingAllowed(attributionSource, inputSource) + if (!isAudioServerOrMediaServerOrSystemServerOrRootUid(IPCThreadState::self()->getCallingUid()) && !(recordingAllowed(attributionSource, inputSource) || inputSource == AUDIO_SOURCE_FM_TUNER || inputSource == AUDIO_SOURCE_REMOTE_SUBMIX || inputSource == AUDIO_SOURCE_ECHO_REFERENCE)) { @@ -696,7 +696,7 @@ Status AudioPolicyService::getInputForAttr(const media::AudioAttributesInternal& // FIXME: use the same permission as for remote submix for now. FALLTHROUGH_INTENDED; case AudioPolicyInterface::API_INPUT_MIX_CAPTURE: - if (!canCaptureOutput) { + if (!isAudioServerOrMediaServerUid(attributionSource.uid) && !canCaptureOutput) { ALOGE("%s permission denied: capture not allowed", __func__); status = PERMISSION_DENIED; } @@ -781,7 +781,7 @@ Status AudioPolicyService::startInput(int32_t portIdAidl) msg << "Audio recording on session " << client->session; // check calling permissions - if (!(startRecording(client->attributionSource, String16(msg.str().c_str()), + if (!isAudioServerOrMediaServerUid(IPCThreadState::self()->getCallingUid()) && !(startRecording(client->attributionSource, String16(msg.str().c_str()), client->attributes.source) || client->attributes.source == AUDIO_SOURCE_FM_TUNER || client->attributes.source == AUDIO_SOURCE_REMOTE_SUBMIX From dcd886d197941f60025d0561ca61a6f2ef1c06bd Mon Sep 17 00:00:00 2001 From: Nico Date: Fri, 1 Nov 2019 15:41:51 +0000 Subject: [PATCH 12/20] Partial revert "Move unused classes out of stagefright foundataion" This imports the old foundation code in the standard platform stagefright. The foundation variant is used in VNDK, we can't change the ABI there. This reverts commit 5ec3d6ac0c607e89d03ba5a9499e471d8559dc7e. Change-Id: Iebcf5d89a768fdb830bea90fbf2c2427a4c3d139 Signed-off-by: DennySPb Signed-off-by: Luca Stefani Signed-off-by: Edwiin Kusuma Jaya --- media/libmediaplayerservice/RemoteDisplay.cpp | 2 +- media/libstagefright/ANetworkSession.cpp | 1400 +++++++++++++++++ media/libstagefright/Android.bp | 2 + media/libstagefright/ParsedMessage.cpp | 301 ++++ .../media/stagefright/ANetworkSession.h | 135 ++ .../include/media/stagefright/ParsedMessage.h | 60 + .../wifi-display/MediaSender.cpp | 5 +- .../wifi-display/rtp/RTPSender.cpp | 2 +- .../wifi-display/source/WifiDisplaySource.cpp | 2 +- .../wifi-display/source/WifiDisplaySource.h | 2 +- 10 files changed, 1905 insertions(+), 6 deletions(-) create mode 100644 media/libstagefright/ANetworkSession.cpp create mode 100644 media/libstagefright/ParsedMessage.cpp create mode 100644 media/libstagefright/include/media/stagefright/ANetworkSession.h create mode 100644 media/libstagefright/include/media/stagefright/ParsedMessage.h diff --git a/media/libmediaplayerservice/RemoteDisplay.cpp b/media/libmediaplayerservice/RemoteDisplay.cpp index 0eb4b5dad0..80cca265f3 100644 --- a/media/libmediaplayerservice/RemoteDisplay.cpp +++ b/media/libmediaplayerservice/RemoteDisplay.cpp @@ -21,7 +21,7 @@ #include #include #include -#include +#include namespace android { diff --git a/media/libstagefright/ANetworkSession.cpp b/media/libstagefright/ANetworkSession.cpp new file mode 100644 index 0000000000..e3b77ec99a --- /dev/null +++ b/media/libstagefright/ANetworkSession.cpp @@ -0,0 +1,1400 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//#define LOG_NDEBUG 0 +#define LOG_TAG "NetworkSession" +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace android { + +static const size_t kMaxUDPSize = 1500; +static const int32_t kMaxUDPRetries = 200; + +struct ANetworkSession::NetworkThread : public Thread { + explicit NetworkThread(ANetworkSession *session); + +protected: + virtual ~NetworkThread(); + +private: + ANetworkSession *mSession; + + virtual bool threadLoop(); + + DISALLOW_EVIL_CONSTRUCTORS(NetworkThread); +}; + +struct ANetworkSession::Session : public RefBase { + enum Mode { + MODE_RTSP, + MODE_DATAGRAM, + MODE_WEBSOCKET, + }; + + enum State { + CONNECTING, + CONNECTED, + LISTENING_RTSP, + LISTENING_TCP_DGRAMS, + DATAGRAM, + }; + + Session(int32_t sessionID, + State state, + int s, + const sp ¬ify); + + int32_t sessionID() const; + int socket() const; + sp getNotificationMessage() const; + + bool isRTSPServer() const; + bool isTCPDatagramServer() const; + + bool wantsToRead(); + bool wantsToWrite(); + + status_t readMore(); + status_t writeMore(); + + status_t sendRequest( + const void *data, ssize_t size, bool timeValid, int64_t timeUs); + + void setMode(Mode mode); + + status_t switchToWebSocketMode(); + +protected: + virtual ~Session(); + +private: + enum { + FRAGMENT_FLAG_TIME_VALID = 1, + }; + struct Fragment { + uint32_t mFlags; + int64_t mTimeUs; + sp mBuffer; + }; + + int32_t mSessionID; + State mState; + Mode mMode; + int mSocket; + sp mNotify; + bool mSawReceiveFailure, mSawSendFailure; + int32_t mUDPRetries; + + List mOutFragments; + + AString mInBuffer; + + int64_t mLastStallReportUs; + + void notifyError(bool send, status_t err, const char *detail); + void notify(NotificationReason reason); + + void dumpFragmentStats(const Fragment &frag); + + DISALLOW_EVIL_CONSTRUCTORS(Session); +}; +//////////////////////////////////////////////////////////////////////////////// + +ANetworkSession::NetworkThread::NetworkThread(ANetworkSession *session) + : mSession(session) { +} + +ANetworkSession::NetworkThread::~NetworkThread() { +} + +bool ANetworkSession::NetworkThread::threadLoop() { + mSession->threadLoop(); + + return true; +} + +//////////////////////////////////////////////////////////////////////////////// + +ANetworkSession::Session::Session( + int32_t sessionID, + State state, + int s, + const sp ¬ify) + : mSessionID(sessionID), + mState(state), + mMode(MODE_DATAGRAM), + mSocket(s), + mNotify(notify), + mSawReceiveFailure(false), + mSawSendFailure(false), + mUDPRetries(kMaxUDPRetries), + mLastStallReportUs(-1ll) { + if (mState == CONNECTED) { + struct sockaddr_in localAddr; + socklen_t localAddrLen = sizeof(localAddr); + + int res = getsockname( + mSocket, (struct sockaddr *)&localAddr, &localAddrLen); + CHECK_GE(res, 0); + + struct sockaddr_in remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + res = getpeername( + mSocket, (struct sockaddr *)&remoteAddr, &remoteAddrLen); + CHECK_GE(res, 0); + + in_addr_t addr = ntohl(localAddr.sin_addr.s_addr); + AString localAddrString = AStringPrintf( + "%d.%d.%d.%d", + (addr >> 24), + (addr >> 16) & 0xff, + (addr >> 8) & 0xff, + addr & 0xff); + + addr = ntohl(remoteAddr.sin_addr.s_addr); + AString remoteAddrString = AStringPrintf( + "%d.%d.%d.%d", + (addr >> 24), + (addr >> 16) & 0xff, + (addr >> 8) & 0xff, + addr & 0xff); + + sp msg = mNotify->dup(); + msg->setInt32("sessionID", mSessionID); + msg->setInt32("reason", kWhatClientConnected); + msg->setString("server-ip", localAddrString.c_str()); + msg->setInt32("server-port", ntohs(localAddr.sin_port)); + msg->setString("client-ip", remoteAddrString.c_str()); + msg->setInt32("client-port", ntohs(remoteAddr.sin_port)); + msg->post(); + } +} + +ANetworkSession::Session::~Session() { + ALOGV("Session %d gone", mSessionID); + + close(mSocket); + mSocket = -1; +} + +int32_t ANetworkSession::Session::sessionID() const { + return mSessionID; +} + +int ANetworkSession::Session::socket() const { + return mSocket; +} + +void ANetworkSession::Session::setMode(Mode mode) { + mMode = mode; +} + +status_t ANetworkSession::Session::switchToWebSocketMode() { + if (mState != CONNECTED || mMode != MODE_RTSP) { + return INVALID_OPERATION; + } + + mMode = MODE_WEBSOCKET; + + return OK; +} + +sp ANetworkSession::Session::getNotificationMessage() const { + return mNotify; +} + +bool ANetworkSession::Session::isRTSPServer() const { + return mState == LISTENING_RTSP; +} + +bool ANetworkSession::Session::isTCPDatagramServer() const { + return mState == LISTENING_TCP_DGRAMS; +} + +bool ANetworkSession::Session::wantsToRead() { + return !mSawReceiveFailure && mState != CONNECTING; +} + +bool ANetworkSession::Session::wantsToWrite() { + return !mSawSendFailure + && (mState == CONNECTING + || (mState == CONNECTED && !mOutFragments.empty()) + || (mState == DATAGRAM && !mOutFragments.empty())); +} + +status_t ANetworkSession::Session::readMore() { + if (mState == DATAGRAM) { + CHECK_EQ(mMode, MODE_DATAGRAM); + + status_t err; + do { + sp buf = new ABuffer(kMaxUDPSize); + + struct sockaddr_in remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + ssize_t n; + do { + n = recvfrom( + mSocket, buf->data(), buf->capacity(), 0, + (struct sockaddr *)&remoteAddr, &remoteAddrLen); + } while (n < 0 && errno == EINTR); + + err = OK; + if (n < 0) { + err = -errno; + } else if (n == 0) { + err = -ECONNRESET; + } else { + buf->setRange(0, n); + + int64_t nowUs = ALooper::GetNowUs(); + buf->meta()->setInt64("arrivalTimeUs", nowUs); + + sp notify = mNotify->dup(); + notify->setInt32("sessionID", mSessionID); + notify->setInt32("reason", kWhatDatagram); + + uint32_t ip = ntohl(remoteAddr.sin_addr.s_addr); + notify->setString( + "fromAddr", + AStringPrintf( + "%u.%u.%u.%u", + ip >> 24, + (ip >> 16) & 0xff, + (ip >> 8) & 0xff, + ip & 0xff).c_str()); + + notify->setInt32("fromPort", ntohs(remoteAddr.sin_port)); + + notify->setBuffer("data", buf); + notify->post(); + } + } while (err == OK); + + if (err == -EAGAIN) { + err = OK; + } + + if (err != OK) { + if (!mUDPRetries) { + notifyError(false /* send */, err, "Recvfrom failed."); + mSawReceiveFailure = true; + } else { + mUDPRetries--; + ALOGE("Recvfrom failed, %d/%d retries left", + mUDPRetries, kMaxUDPRetries); + err = OK; + } + } else { + mUDPRetries = kMaxUDPRetries; + } + + return err; + } + + char tmp[512]; + ssize_t n; + do { + n = recv(mSocket, tmp, sizeof(tmp), 0); + } while (n < 0 && errno == EINTR); + + status_t err = OK; + + if (n > 0) { + mInBuffer.append(tmp, n); + +#if 0 + ALOGI("in:"); + hexdump(tmp, n); +#endif + } else if (n < 0) { + err = -errno; + } else { + err = -ECONNRESET; + } + + if (mMode == MODE_DATAGRAM) { + // TCP stream carrying 16-bit length-prefixed datagrams. + + while (mInBuffer.size() >= 2) { + size_t packetSize = U16_AT((const uint8_t *)mInBuffer.c_str()); + + if (mInBuffer.size() < packetSize + 2) { + break; + } + + sp packet = new ABuffer(packetSize); + memcpy(packet->data(), mInBuffer.c_str() + 2, packetSize); + + int64_t nowUs = ALooper::GetNowUs(); + packet->meta()->setInt64("arrivalTimeUs", nowUs); + + sp notify = mNotify->dup(); + notify->setInt32("sessionID", mSessionID); + notify->setInt32("reason", kWhatDatagram); + notify->setBuffer("data", packet); + notify->post(); + + mInBuffer.erase(0, packetSize + 2); + } + } else if (mMode == MODE_RTSP) { + for (;;) { + size_t length; + + if (mInBuffer.size() > 0 && mInBuffer.c_str()[0] == '$') { + if (mInBuffer.size() < 4) { + break; + } + + length = U16_AT((const uint8_t *)mInBuffer.c_str() + 2); + + if (mInBuffer.size() < 4 + length) { + break; + } + + sp notify = mNotify->dup(); + notify->setInt32("sessionID", mSessionID); + notify->setInt32("reason", kWhatBinaryData); + notify->setInt32("channel", mInBuffer.c_str()[1]); + + sp data = new ABuffer(length); + memcpy(data->data(), mInBuffer.c_str() + 4, length); + + int64_t nowUs = ALooper::GetNowUs(); + data->meta()->setInt64("arrivalTimeUs", nowUs); + + notify->setBuffer("data", data); + notify->post(); + + mInBuffer.erase(0, 4 + length); + continue; + } + + sp msg = + ParsedMessage::Parse( + mInBuffer.c_str(), mInBuffer.size(), err != OK, &length); + + if (msg == NULL) { + break; + } + + sp notify = mNotify->dup(); + notify->setInt32("sessionID", mSessionID); + notify->setInt32("reason", kWhatData); + notify->setObject("data", msg); + notify->post(); + +#if 1 + // XXX The (old) dongle sends the wrong content length header on a + // SET_PARAMETER request that signals a "wfd_idr_request". + // (17 instead of 19). + const char *content = msg->getContent(); + if (content + && !memcmp(content, "wfd_idr_request\r\n", 17) + && length >= 19 + && mInBuffer.c_str()[length] == '\r' + && mInBuffer.c_str()[length + 1] == '\n') { + length += 2; + } +#endif + + mInBuffer.erase(0, length); + + if (err != OK) { + break; + } + } + } else { + CHECK_EQ(mMode, MODE_WEBSOCKET); + + const uint8_t *data = (const uint8_t *)mInBuffer.c_str(); + // hexdump(data, mInBuffer.size()); + + while (mInBuffer.size() >= 2) { + size_t offset = 2; + + uint64_t payloadLen = data[1] & 0x7f; + if (payloadLen == 126) { + if (offset + 2 > mInBuffer.size()) { + break; + } + + payloadLen = U16_AT(&data[offset]); + offset += 2; + } else if (payloadLen == 127) { + if (offset + 8 > mInBuffer.size()) { + break; + } + + payloadLen = U64_AT(&data[offset]); + offset += 8; + } + + uint32_t mask = 0; + if (data[1] & 0x80) { + // MASK==1 + if (offset + 4 > mInBuffer.size()) { + break; + } + + mask = U32_AT(&data[offset]); + offset += 4; + } + + if (payloadLen > mInBuffer.size() || offset > mInBuffer.size() - payloadLen) { + break; + } + + // We have the full message. + + sp packet = new ABuffer(payloadLen); + memcpy(packet->data(), &data[offset], payloadLen); + + if (mask != 0) { + for (size_t i = 0; i < payloadLen; ++i) { + packet->data()[i] = + data[offset + i] + ^ ((mask >> (8 * (3 - (i % 4)))) & 0xff); + } + } + + sp notify = mNotify->dup(); + notify->setInt32("sessionID", mSessionID); + notify->setInt32("reason", kWhatWebSocketMessage); + notify->setBuffer("data", packet); + notify->setInt32("headerByte", data[0]); + notify->post(); + + mInBuffer.erase(0, offset + payloadLen); + } + } + + if (err != OK) { + notifyError(false /* send */, err, "Recv failed."); + mSawReceiveFailure = true; + } + + return err; +} + +void ANetworkSession::Session::dumpFragmentStats(const Fragment & /* frag */) { +#if 0 + int64_t nowUs = ALooper::GetNowUs(); + int64_t delayMs = (nowUs - frag.mTimeUs) / 1000ll; + + static const int64_t kMinDelayMs = 0; + static const int64_t kMaxDelayMs = 300; + + const char *kPattern = "########################################"; + size_t kPatternSize = strlen(kPattern); + + int n = (kPatternSize * (delayMs - kMinDelayMs)) + / (kMaxDelayMs - kMinDelayMs); + + if (n < 0) { + n = 0; + } else if ((size_t)n > kPatternSize) { + n = kPatternSize; + } + + ALOGI("[%lld]: (%4lld ms) %s\n", + frag.mTimeUs / 1000, + delayMs, + kPattern + kPatternSize - n); +#endif +} + +status_t ANetworkSession::Session::writeMore() { + if (mState == DATAGRAM) { + CHECK(!mOutFragments.empty()); + + status_t err; + do { + const Fragment &frag = *mOutFragments.begin(); + const sp &datagram = frag.mBuffer; + + int n; + do { + n = send(mSocket, datagram->data(), datagram->size(), 0); + } while (n < 0 && errno == EINTR); + + err = OK; + + if (n > 0) { + if (frag.mFlags & FRAGMENT_FLAG_TIME_VALID) { + dumpFragmentStats(frag); + } + + mOutFragments.erase(mOutFragments.begin()); + } else if (n < 0) { + err = -errno; + } else if (n == 0) { + err = -ECONNRESET; + } + } while (err == OK && !mOutFragments.empty()); + + if (err == -EAGAIN) { + if (!mOutFragments.empty()) { + ALOGI("%zu datagrams remain queued.", mOutFragments.size()); + } + err = OK; + } + + if (err != OK) { + if (!mUDPRetries) { + notifyError(true /* send */, err, "Send datagram failed."); + mSawSendFailure = true; + } else { + mUDPRetries--; + ALOGE("Send datagram failed, %d/%d retries left", + mUDPRetries, kMaxUDPRetries); + err = OK; + } + } else { + mUDPRetries = kMaxUDPRetries; + } + + return err; + } + + if (mState == CONNECTING) { + int err; + socklen_t optionLen = sizeof(err); + CHECK_EQ(getsockopt(mSocket, SOL_SOCKET, SO_ERROR, &err, &optionLen), 0); + CHECK_EQ(optionLen, (socklen_t)sizeof(err)); + + if (err != 0) { + notifyError(kWhatError, -err, "Connection failed"); + mSawSendFailure = true; + + return -err; + } + + mState = CONNECTED; + notify(kWhatConnected); + + return OK; + } + + CHECK_EQ(mState, CONNECTED); + CHECK(!mOutFragments.empty()); + + ssize_t n = -1; + while (!mOutFragments.empty()) { + const Fragment &frag = *mOutFragments.begin(); + + do { + n = send(mSocket, frag.mBuffer->data(), frag.mBuffer->size(), 0); + } while (n < 0 && errno == EINTR); + + if (n <= 0) { + break; + } + + frag.mBuffer->setRange( + frag.mBuffer->offset() + n, frag.mBuffer->size() - n); + + if (frag.mBuffer->size() > 0) { + break; + } + + if (frag.mFlags & FRAGMENT_FLAG_TIME_VALID) { + dumpFragmentStats(frag); + } + + mOutFragments.erase(mOutFragments.begin()); + } + + status_t err = OK; + + if (n < 0) { + err = -errno; + } else if (n == 0) { + err = -ECONNRESET; + } + + if (err != OK) { + notifyError(true /* send */, err, "Send failed."); + mSawSendFailure = true; + } + +#if 0 + int numBytesQueued; + int res = ioctl(mSocket, SIOCOUTQ, &numBytesQueued); + if (res == 0 && numBytesQueued > 50 * 1024) { + if (numBytesQueued > 409600) { + ALOGW("!!! numBytesQueued = %d", numBytesQueued); + } + + int64_t nowUs = ALooper::GetNowUs(); + + if (mLastStallReportUs < 0ll + || nowUs > mLastStallReportUs + 100000ll) { + sp msg = mNotify->dup(); + msg->setInt32("sessionID", mSessionID); + msg->setInt32("reason", kWhatNetworkStall); + msg->setSize("numBytesQueued", numBytesQueued); + msg->post(); + + mLastStallReportUs = nowUs; + } + } +#endif + + return err; +} + +status_t ANetworkSession::Session::sendRequest( + const void *data, ssize_t size, bool timeValid, int64_t timeUs) { + CHECK(mState == CONNECTED || mState == DATAGRAM); + + if (size < 0) { + size = strlen((const char *)data); + } + + if (size == 0) { + return OK; + } + + sp buffer; + + if (mState == CONNECTED && mMode == MODE_DATAGRAM) { + CHECK_LE(size, 65535); + + buffer = new ABuffer(size + 2); + buffer->data()[0] = size >> 8; + buffer->data()[1] = size & 0xff; + memcpy(buffer->data() + 2, data, size); + } else if (mState == CONNECTED && mMode == MODE_WEBSOCKET) { + static const bool kUseMask = false; // Chromium doesn't like it. + + size_t numHeaderBytes = 2 + (kUseMask ? 4 : 0); + if (size > 65535) { + numHeaderBytes += 8; + } else if (size > 125) { + numHeaderBytes += 2; + } + + buffer = new ABuffer(numHeaderBytes + size); + buffer->data()[0] = 0x81; // FIN==1 | opcode=1 (text) + buffer->data()[1] = kUseMask ? 0x80 : 0x00; + + if (size > 65535) { + buffer->data()[1] |= 127; + buffer->data()[2] = 0x00; + buffer->data()[3] = 0x00; + buffer->data()[4] = 0x00; + buffer->data()[5] = 0x00; + buffer->data()[6] = (size >> 24) & 0xff; + buffer->data()[7] = (size >> 16) & 0xff; + buffer->data()[8] = (size >> 8) & 0xff; + buffer->data()[9] = size & 0xff; + } else if (size > 125) { + buffer->data()[1] |= 126; + buffer->data()[2] = (size >> 8) & 0xff; + buffer->data()[3] = size & 0xff; + } else { + buffer->data()[1] |= size; + } + + if (kUseMask) { + uint32_t mask = rand(); + + buffer->data()[numHeaderBytes - 4] = (mask >> 24) & 0xff; + buffer->data()[numHeaderBytes - 3] = (mask >> 16) & 0xff; + buffer->data()[numHeaderBytes - 2] = (mask >> 8) & 0xff; + buffer->data()[numHeaderBytes - 1] = mask & 0xff; + + for (size_t i = 0; i < (size_t)size; ++i) { + buffer->data()[numHeaderBytes + i] = + ((const uint8_t *)data)[i] + ^ ((mask >> (8 * (3 - (i % 4)))) & 0xff); + } + } else { + memcpy(buffer->data() + numHeaderBytes, data, size); + } + } else { + buffer = new ABuffer(size); + memcpy(buffer->data(), data, size); + } + + Fragment frag; + + frag.mFlags = 0; + if (timeValid) { + frag.mFlags = FRAGMENT_FLAG_TIME_VALID; + frag.mTimeUs = timeUs; + } + + frag.mBuffer = buffer; + + mOutFragments.push_back(frag); + + return OK; +} + +void ANetworkSession::Session::notifyError( + bool send, status_t err, const char *detail) { + sp msg = mNotify->dup(); + msg->setInt32("sessionID", mSessionID); + msg->setInt32("reason", kWhatError); + msg->setInt32("send", send); + msg->setInt32("err", err); + msg->setString("detail", detail); + msg->post(); +} + +void ANetworkSession::Session::notify(NotificationReason reason) { + sp msg = mNotify->dup(); + msg->setInt32("sessionID", mSessionID); + msg->setInt32("reason", reason); + msg->post(); +} + +//////////////////////////////////////////////////////////////////////////////// + +ANetworkSession::ANetworkSession() + : mNextSessionID(1) { + mPipeFd[0] = mPipeFd[1] = -1; +} + +ANetworkSession::~ANetworkSession() { + stop(); +} + +status_t ANetworkSession::start() { + if (mThread != NULL) { + return INVALID_OPERATION; + } + + int res = pipe(mPipeFd); + if (res != 0) { + mPipeFd[0] = mPipeFd[1] = -1; + return -errno; + } + + mThread = new NetworkThread(this); + + status_t err = mThread->run("ANetworkSession", ANDROID_PRIORITY_AUDIO); + + if (err != OK) { + mThread.clear(); + + close(mPipeFd[0]); + close(mPipeFd[1]); + mPipeFd[0] = mPipeFd[1] = -1; + + return err; + } + + return OK; +} + +status_t ANetworkSession::stop() { + if (mThread == NULL) { + return INVALID_OPERATION; + } + + mThread->requestExit(); + interrupt(); + mThread->requestExitAndWait(); + + mThread.clear(); + + close(mPipeFd[0]); + close(mPipeFd[1]); + mPipeFd[0] = mPipeFd[1] = -1; + + return OK; +} + +status_t ANetworkSession::createRTSPClient( + const char *host, unsigned port, const sp ¬ify, + int32_t *sessionID) { + return createClientOrServer( + kModeCreateRTSPClient, + NULL /* addr */, + 0 /* port */, + host, + port, + notify, + sessionID); +} + +status_t ANetworkSession::createRTSPServer( + const struct in_addr &addr, unsigned port, + const sp ¬ify, int32_t *sessionID) { + return createClientOrServer( + kModeCreateRTSPServer, + &addr, + port, + NULL /* remoteHost */, + 0 /* remotePort */, + notify, + sessionID); +} + +status_t ANetworkSession::createUDPSession( + unsigned localPort, const sp ¬ify, int32_t *sessionID) { + return createUDPSession(localPort, NULL, 0, notify, sessionID); +} + +status_t ANetworkSession::createUDPSession( + unsigned localPort, + const char *remoteHost, + unsigned remotePort, + const sp ¬ify, + int32_t *sessionID) { + return createClientOrServer( + kModeCreateUDPSession, + NULL /* addr */, + localPort, + remoteHost, + remotePort, + notify, + sessionID); +} + +status_t ANetworkSession::createTCPDatagramSession( + const struct in_addr &addr, unsigned port, + const sp ¬ify, int32_t *sessionID) { + return createClientOrServer( + kModeCreateTCPDatagramSessionPassive, + &addr, + port, + NULL /* remoteHost */, + 0 /* remotePort */, + notify, + sessionID); +} + +status_t ANetworkSession::createTCPDatagramSession( + unsigned localPort, + const char *remoteHost, + unsigned remotePort, + const sp ¬ify, + int32_t *sessionID) { + return createClientOrServer( + kModeCreateTCPDatagramSessionActive, + NULL /* addr */, + localPort, + remoteHost, + remotePort, + notify, + sessionID); +} + +status_t ANetworkSession::destroySession(int32_t sessionID) { + Mutex::Autolock autoLock(mLock); + + ssize_t index = mSessions.indexOfKey(sessionID); + + if (index < 0) { + return -ENOENT; + } + + mSessions.removeItemsAt(index); + + interrupt(); + + return OK; +} + +// static +status_t ANetworkSession::MakeSocketNonBlocking(int s) { + int flags = fcntl(s, F_GETFL, 0); + if (flags < 0) { + flags = 0; + } + + int res = fcntl(s, F_SETFL, flags | O_NONBLOCK); + if (res < 0) { + return -errno; + } + + return OK; +} + +status_t ANetworkSession::createClientOrServer( + Mode mode, + const struct in_addr *localAddr, + unsigned port, + const char *remoteHost, + unsigned remotePort, + const sp ¬ify, + int32_t *sessionID) { + Mutex::Autolock autoLock(mLock); + + *sessionID = 0; + status_t err = OK; + int s, res; + sp session; + + s = socket( + AF_INET, + (mode == kModeCreateUDPSession) ? SOCK_DGRAM : SOCK_STREAM, + 0); + + if (s < 0) { + err = -errno; + goto bail; + } + + if (mode == kModeCreateRTSPServer + || mode == kModeCreateTCPDatagramSessionPassive) { + const int yes = 1; + res = setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + + if (res < 0) { + err = -errno; + goto bail2; + } + } + + if (mode == kModeCreateUDPSession) { + int size = 256 * 1024; + + res = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &size, sizeof(size)); + + if (res < 0) { + err = -errno; + goto bail2; + } + + res = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &size, sizeof(size)); + + if (res < 0) { + err = -errno; + goto bail2; + } + } else if (mode == kModeCreateTCPDatagramSessionActive) { + int flag = 1; + res = setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &flag, sizeof(flag)); + + if (res < 0) { + err = -errno; + goto bail2; + } + + int tos = 224; // VOICE + res = setsockopt(s, IPPROTO_IP, IP_TOS, &tos, sizeof(tos)); + + if (res < 0) { + err = -errno; + goto bail2; + } + } + + err = MakeSocketNonBlocking(s); + + if (err != OK) { + goto bail2; + } + + struct sockaddr_in addr; + memset(addr.sin_zero, 0, sizeof(addr.sin_zero)); + addr.sin_family = AF_INET; + + if (mode == kModeCreateRTSPClient + || mode == kModeCreateTCPDatagramSessionActive) { + struct hostent *ent= gethostbyname(remoteHost); + if (ent == NULL) { + err = -h_errno; + goto bail2; + } + + addr.sin_addr.s_addr = *(in_addr_t *)ent->h_addr; + addr.sin_port = htons(remotePort); + } else if (localAddr != NULL) { + addr.sin_addr = *localAddr; + addr.sin_port = htons(port); + } else { + addr.sin_addr.s_addr = htonl(INADDR_ANY); + addr.sin_port = htons(port); + } + + if (mode == kModeCreateRTSPClient + || mode == kModeCreateTCPDatagramSessionActive) { + in_addr_t x = ntohl(addr.sin_addr.s_addr); + ALOGI("connecting socket %d to %d.%d.%d.%d:%d", + s, + (x >> 24), + (x >> 16) & 0xff, + (x >> 8) & 0xff, + x & 0xff, + ntohs(addr.sin_port)); + + res = connect(s, (const struct sockaddr *)&addr, sizeof(addr)); + + CHECK_LT(res, 0); + if (errno == EINPROGRESS) { + res = 0; + } + } else { + res = bind(s, (const struct sockaddr *)&addr, sizeof(addr)); + + if (res == 0) { + if (mode == kModeCreateRTSPServer + || mode == kModeCreateTCPDatagramSessionPassive) { + res = listen(s, 4); + } else { + CHECK_EQ(mode, kModeCreateUDPSession); + + if (remoteHost != NULL) { + struct sockaddr_in remoteAddr; + memset(remoteAddr.sin_zero, 0, sizeof(remoteAddr.sin_zero)); + remoteAddr.sin_family = AF_INET; + remoteAddr.sin_port = htons(remotePort); + + struct hostent *ent= gethostbyname(remoteHost); + if (ent == NULL) { + err = -h_errno; + goto bail2; + } + + remoteAddr.sin_addr.s_addr = *(in_addr_t *)ent->h_addr; + + res = connect( + s, + (const struct sockaddr *)&remoteAddr, + sizeof(remoteAddr)); + } + } + } + } + + if (res < 0) { + err = -errno; + goto bail2; + } + + Session::State state; + switch (mode) { + case kModeCreateRTSPClient: + state = Session::CONNECTING; + break; + + case kModeCreateTCPDatagramSessionActive: + state = Session::CONNECTING; + break; + + case kModeCreateTCPDatagramSessionPassive: + state = Session::LISTENING_TCP_DGRAMS; + break; + + case kModeCreateRTSPServer: + state = Session::LISTENING_RTSP; + break; + + default: + CHECK_EQ(mode, kModeCreateUDPSession); + state = Session::DATAGRAM; + break; + } + + session = new Session( + mNextSessionID++, + state, + s, + notify); + + if (mode == kModeCreateTCPDatagramSessionActive) { + session->setMode(Session::MODE_DATAGRAM); + } else if (mode == kModeCreateRTSPClient) { + session->setMode(Session::MODE_RTSP); + } + + mSessions.add(session->sessionID(), session); + + interrupt(); + + *sessionID = session->sessionID(); + + goto bail; + +bail2: + close(s); + s = -1; + +bail: + return err; +} + +status_t ANetworkSession::connectUDPSession( + int32_t sessionID, const char *remoteHost, unsigned remotePort) { + Mutex::Autolock autoLock(mLock); + + ssize_t index = mSessions.indexOfKey(sessionID); + + if (index < 0) { + return -ENOENT; + } + + const sp session = mSessions.valueAt(index); + int s = session->socket(); + + struct sockaddr_in remoteAddr; + memset(remoteAddr.sin_zero, 0, sizeof(remoteAddr.sin_zero)); + remoteAddr.sin_family = AF_INET; + remoteAddr.sin_port = htons(remotePort); + + status_t err = OK; + struct hostent *ent = gethostbyname(remoteHost); + if (ent == NULL) { + err = -h_errno; + } else { + remoteAddr.sin_addr.s_addr = *(in_addr_t *)ent->h_addr; + + int res = connect( + s, + (const struct sockaddr *)&remoteAddr, + sizeof(remoteAddr)); + + if (res < 0) { + err = -errno; + } + } + + return err; +} + +status_t ANetworkSession::sendRequest( + int32_t sessionID, const void *data, ssize_t size, + bool timeValid, int64_t timeUs) { + Mutex::Autolock autoLock(mLock); + + ssize_t index = mSessions.indexOfKey(sessionID); + + if (index < 0) { + return -ENOENT; + } + + const sp session = mSessions.valueAt(index); + + status_t err = session->sendRequest(data, size, timeValid, timeUs); + + interrupt(); + + return err; +} + +status_t ANetworkSession::switchToWebSocketMode(int32_t sessionID) { + Mutex::Autolock autoLock(mLock); + + ssize_t index = mSessions.indexOfKey(sessionID); + + if (index < 0) { + return -ENOENT; + } + + const sp session = mSessions.valueAt(index); + return session->switchToWebSocketMode(); +} + +void ANetworkSession::interrupt() { + static const char dummy = 0; + + ssize_t n; + do { + n = write(mPipeFd[1], &dummy, 1); + } while (n < 0 && errno == EINTR); + + if (n < 0) { + ALOGW("Error writing to pipe (%s)", strerror(errno)); + } +} + +void ANetworkSession::threadLoop() { + fd_set rs, ws; + FD_ZERO(&rs); + FD_ZERO(&ws); + + FD_SET(mPipeFd[0], &rs); + int maxFd = mPipeFd[0]; + + { + Mutex::Autolock autoLock(mLock); + + for (size_t i = 0; i < mSessions.size(); ++i) { + const sp &session = mSessions.valueAt(i); + + int s = session->socket(); + + if (s < 0) { + continue; + } + + if (session->wantsToRead()) { + FD_SET(s, &rs); + if (s > maxFd) { + maxFd = s; + } + } + + if (session->wantsToWrite()) { + FD_SET(s, &ws); + if (s > maxFd) { + maxFd = s; + } + } + } + } + + int res = select(maxFd + 1, &rs, &ws, NULL, NULL /* tv */); + + if (res == 0) { + return; + } + + if (res < 0) { + if (errno == EINTR) { + return; + } + + ALOGE("select failed w/ error %d (%s)", errno, strerror(errno)); + return; + } + + if (FD_ISSET(mPipeFd[0], &rs)) { + char c; + ssize_t n; + do { + n = read(mPipeFd[0], &c, 1); + } while (n < 0 && errno == EINTR); + + if (n < 0) { + ALOGW("Error reading from pipe (%s)", strerror(errno)); + } + + --res; + } + + { + Mutex::Autolock autoLock(mLock); + + List > sessionsToAdd; + + for (size_t i = mSessions.size(); res > 0 && i > 0;) { + i--; + const sp &session = mSessions.valueAt(i); + + int s = session->socket(); + + if (s < 0) { + continue; + } + + if (FD_ISSET(s, &rs) || FD_ISSET(s, &ws)) { + --res; + } + + if (FD_ISSET(s, &rs)) { + if (session->isRTSPServer() || session->isTCPDatagramServer()) { + struct sockaddr_in remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + int clientSocket = accept( + s, (struct sockaddr *)&remoteAddr, &remoteAddrLen); + + if (clientSocket >= 0) { + status_t err = MakeSocketNonBlocking(clientSocket); + + if (err != OK) { + ALOGE("Unable to make client socket non blocking, " + "failed w/ error %d (%s)", + err, strerror(-err)); + + close(clientSocket); + clientSocket = -1; + } else { + in_addr_t addr = ntohl(remoteAddr.sin_addr.s_addr); + + ALOGI("incoming connection from %d.%d.%d.%d:%d " + "(socket %d)", + (addr >> 24), + (addr >> 16) & 0xff, + (addr >> 8) & 0xff, + addr & 0xff, + ntohs(remoteAddr.sin_port), + clientSocket); + + sp clientSession = + new Session( + mNextSessionID++, + Session::CONNECTED, + clientSocket, + session->getNotificationMessage()); + + clientSession->setMode( + session->isRTSPServer() + ? Session::MODE_RTSP + : Session::MODE_DATAGRAM); + + sessionsToAdd.push_back(clientSession); + } + } else { + ALOGE("accept returned error %d (%s)", + errno, strerror(errno)); + } + } else { + status_t err = session->readMore(); + if (err != OK) { + ALOGE("readMore on socket %d failed w/ error %d (%s)", + s, err, strerror(-err)); + } + } + } + + if (FD_ISSET(s, &ws)) { + status_t err = session->writeMore(); + if (err != OK) { + ALOGE("writeMore on socket %d failed w/ error %d (%s)", + s, err, strerror(-err)); + } + } + } + + while (!sessionsToAdd.empty()) { + sp session = *sessionsToAdd.begin(); + sessionsToAdd.erase(sessionsToAdd.begin()); + + mSessions.add(session->sessionID(), session); + + ALOGI("added clientSession %d", session->sessionID()); + } + } +} + +} // namespace android diff --git a/media/libstagefright/Android.bp b/media/libstagefright/Android.bp index 9af38fcb04..e71f3b5d29 100644 --- a/media/libstagefright/Android.bp +++ b/media/libstagefright/Android.bp @@ -296,6 +296,7 @@ cc_library { "ACodecBufferChannel.cpp", "AHierarchicalStateMachine.cpp", "AMRWriter.cpp", + "ANetworkSession.cpp", "AudioSource.cpp", "BufferImpl.cpp", "CallbackDataSource.cpp", @@ -325,6 +326,7 @@ cc_library { "OggWriter.cpp", "OMXClient.cpp", "OmxInfoBuilder.cpp", + "ParsedMessage.cpp", "RemoteMediaExtractor.cpp", "RemoteMediaSource.cpp", "SimpleDecodingSource.cpp", diff --git a/media/libstagefright/ParsedMessage.cpp b/media/libstagefright/ParsedMessage.cpp new file mode 100644 index 0000000000..4bfc454338 --- /dev/null +++ b/media/libstagefright/ParsedMessage.cpp @@ -0,0 +1,301 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +namespace android { + +// static +sp ParsedMessage::Parse( + const char *data, size_t size, bool noMoreData, size_t *length) { + sp msg = new ParsedMessage; + ssize_t res = msg->parse(data, size, noMoreData); + + if (res < 0) { + *length = 0; + return NULL; + } + + *length = res; + return msg; +} + +ParsedMessage::ParsedMessage() { +} + +ParsedMessage::~ParsedMessage() { +} + +bool ParsedMessage::findString(const char *name, AString *value) const { + AString key = name; + key.tolower(); + + ssize_t index = mDict.indexOfKey(key); + + if (index < 0) { + value->clear(); + + return false; + } + + *value = mDict.valueAt(index); + return true; +} + +bool ParsedMessage::findInt32(const char *name, int32_t *value) const { + AString stringValue; + + if (!findString(name, &stringValue)) { + return false; + } + + char *end; + *value = strtol(stringValue.c_str(), &end, 10); + + if (end == stringValue.c_str() || *end != '\0') { + *value = 0; + return false; + } + + return true; +} + +const char *ParsedMessage::getContent() const { + return mContent.c_str(); +} + +ssize_t ParsedMessage::parse(const char *data, size_t size, bool noMoreData) { + if (size == 0) { + return -1; + } + + ssize_t lastDictIndex = -1; + + size_t offset = 0; + bool headersComplete = false; + while (offset < size) { + size_t lineEndOffset = offset; + while (lineEndOffset + 1 < size + && (data[lineEndOffset] != '\r' + || data[lineEndOffset + 1] != '\n')) { + ++lineEndOffset; + } + + if (lineEndOffset + 1 >= size) { + return -1; + } + + AString line(&data[offset], lineEndOffset - offset); + + if (offset == 0) { + // Special handling for the request/status line. + + mDict.add(AString("_"), line); + offset = lineEndOffset + 2; + + continue; + } + + if (lineEndOffset == offset) { + // An empty line separates headers from body. + headersComplete = true; + offset += 2; + break; + } + + if (line.c_str()[0] == ' ' || line.c_str()[0] == '\t') { + // Support for folded header values. + + if (lastDictIndex >= 0) { + // Otherwise it's malformed since the first header line + // cannot continue anything... + + AString &value = mDict.editValueAt(lastDictIndex); + value.append(line); + } + + offset = lineEndOffset + 2; + continue; + } + + ssize_t colonPos = line.find(":"); + if (colonPos >= 0) { + AString key(line, 0, colonPos); + key.trim(); + key.tolower(); + + line.erase(0, colonPos + 1); + + lastDictIndex = mDict.add(key, line); + } + + offset = lineEndOffset + 2; + } + + if (!headersComplete && (!noMoreData || offset == 0)) { + // We either saw the empty line separating headers from body + // or we saw at least the status line and know that no more data + // is going to follow. + return -1; + } + + for (size_t i = 0; i < mDict.size(); ++i) { + mDict.editValueAt(i).trim(); + } + + int32_t contentLength; + if (!findInt32("content-length", &contentLength) || contentLength < 0) { + contentLength = 0; + } + + size_t totalLength = offset + contentLength; + + if (size < totalLength) { + return -1; + } + + mContent.setTo(&data[offset], contentLength); + + return totalLength; +} + +bool ParsedMessage::getRequestField(size_t index, AString *field) const { + AString line; + CHECK(findString("_", &line)); + + size_t prevOffset = 0; + size_t offset = 0; + for (size_t i = 0; i <= index; ++i) { + if (offset >= line.size()) { + return false; + } + + ssize_t spacePos = line.find(" ", offset); + + if (spacePos < 0) { + spacePos = line.size(); + } + + prevOffset = offset; + offset = spacePos + 1; + } + + field->setTo(line, prevOffset, offset - prevOffset - 1); + + return true; +} + +bool ParsedMessage::getStatusCode(int32_t *statusCode) const { + AString statusCodeString; + if (!getRequestField(1, &statusCodeString)) { + *statusCode = 0; + return false; + } + + char *end; + *statusCode = strtol(statusCodeString.c_str(), &end, 10); + + if (*end != '\0' || end == statusCodeString.c_str() + || (*statusCode) < 100 || (*statusCode) > 999) { + *statusCode = 0; + return false; + } + + return true; +} + +AString ParsedMessage::debugString() const { + AString line; + CHECK(findString("_", &line)); + + line.append("\n"); + + for (size_t i = 0; i < mDict.size(); ++i) { + const AString &key = mDict.keyAt(i); + const AString &value = mDict.valueAt(i); + + if (key == AString("_")) { + continue; + } + + line.append(key); + line.append(": "); + line.append(value); + line.append("\n"); + } + + line.append("\n"); + line.append(mContent); + + return line; +} + +// static +bool ParsedMessage::GetAttribute( + const char *s, const char *key, AString *value) { + value->clear(); + + size_t keyLen = strlen(key); + + for (;;) { + while (isspace(*s)) { + ++s; + } + + const char *colonPos = strchr(s, ';'); + + size_t len = + (colonPos == NULL) ? strlen(s) : colonPos - s; + + if (len >= keyLen + 1 && s[keyLen] == '=' && !strncmp(s, key, keyLen)) { + value->setTo(&s[keyLen + 1], len - keyLen - 1); + return true; + } + + if (colonPos == NULL) { + return false; + } + + s = colonPos + 1; + } +} + +// static +bool ParsedMessage::GetInt32Attribute( + const char *s, const char *key, int32_t *value) { + AString stringValue; + if (!GetAttribute(s, key, &stringValue)) { + *value = 0; + return false; + } + + char *end; + *value = strtol(stringValue.c_str(), &end, 10); + + if (end == stringValue.c_str() || *end != '\0') { + *value = 0; + return false; + } + + return true; +} + +} // namespace android + diff --git a/media/libstagefright/include/media/stagefright/ANetworkSession.h b/media/libstagefright/include/media/stagefright/ANetworkSession.h new file mode 100644 index 0000000000..fd3ebaaa28 --- /dev/null +++ b/media/libstagefright/include/media/stagefright/ANetworkSession.h @@ -0,0 +1,135 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef A_NETWORK_SESSION_H_ + +#define A_NETWORK_SESSION_H_ + +#include +#include +#include +#include + +#include + +namespace android { + +struct AMessage; + +// Helper class to manage a number of live sockets (datagram and stream-based) +// on a single thread. Clients are notified about activity through AMessages. +struct ANetworkSession : public RefBase { + ANetworkSession(); + + status_t start(); + status_t stop(); + + status_t createRTSPClient( + const char *host, unsigned port, const sp ¬ify, + int32_t *sessionID); + + status_t createRTSPServer( + const struct in_addr &addr, unsigned port, + const sp ¬ify, int32_t *sessionID); + + status_t createUDPSession( + unsigned localPort, const sp ¬ify, int32_t *sessionID); + + status_t createUDPSession( + unsigned localPort, + const char *remoteHost, + unsigned remotePort, + const sp ¬ify, + int32_t *sessionID); + + status_t connectUDPSession( + int32_t sessionID, const char *remoteHost, unsigned remotePort); + + // passive + status_t createTCPDatagramSession( + const struct in_addr &addr, unsigned port, + const sp ¬ify, int32_t *sessionID); + + // active + status_t createTCPDatagramSession( + unsigned localPort, + const char *remoteHost, + unsigned remotePort, + const sp ¬ify, + int32_t *sessionID); + + status_t destroySession(int32_t sessionID); + + status_t sendRequest( + int32_t sessionID, const void *data, ssize_t size = -1, + bool timeValid = false, int64_t timeUs = -1ll); + + status_t switchToWebSocketMode(int32_t sessionID); + + enum NotificationReason { + kWhatError, + kWhatConnected, + kWhatClientConnected, + kWhatData, + kWhatDatagram, + kWhatBinaryData, + kWhatWebSocketMessage, + kWhatNetworkStall, + }; + +protected: + virtual ~ANetworkSession(); + +private: + struct NetworkThread; + struct Session; + + Mutex mLock; + sp mThread; + + int32_t mNextSessionID; + + int mPipeFd[2]; + + KeyedVector > mSessions; + + enum Mode { + kModeCreateUDPSession, + kModeCreateTCPDatagramSessionPassive, + kModeCreateTCPDatagramSessionActive, + kModeCreateRTSPServer, + kModeCreateRTSPClient, + }; + status_t createClientOrServer( + Mode mode, + const struct in_addr *addr, + unsigned port, + const char *remoteHost, + unsigned remotePort, + const sp ¬ify, + int32_t *sessionID); + + void threadLoop(); + void interrupt(); + + static status_t MakeSocketNonBlocking(int s); + + DISALLOW_EVIL_CONSTRUCTORS(ANetworkSession); +}; + +} // namespace android + +#endif // A_NETWORK_SESSION_H_ diff --git a/media/libstagefright/include/media/stagefright/ParsedMessage.h b/media/libstagefright/include/media/stagefright/ParsedMessage.h new file mode 100644 index 0000000000..9d43a93319 --- /dev/null +++ b/media/libstagefright/include/media/stagefright/ParsedMessage.h @@ -0,0 +1,60 @@ +/* + * Copyright 2012, The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +namespace android { + +// Encapsulates an "HTTP/RTSP style" response, i.e. a status line, +// key/value pairs making up the headers and an optional body/content. +struct ParsedMessage : public RefBase { + static sp Parse( + const char *data, size_t size, bool noMoreData, size_t *length); + + bool findString(const char *name, AString *value) const; + bool findInt32(const char *name, int32_t *value) const; + + const char *getContent() const; + + bool getRequestField(size_t index, AString *field) const; + bool getStatusCode(int32_t *statusCode) const; + + AString debugString() const; + + static bool GetAttribute(const char *s, const char *key, AString *value); + + static bool GetInt32Attribute( + const char *s, const char *key, int32_t *value); + + +protected: + virtual ~ParsedMessage(); + +private: + KeyedVector mDict; + AString mContent; + + ParsedMessage(); + + ssize_t parse(const char *data, size_t size, bool noMoreData); + + DISALLOW_EVIL_CONSTRUCTORS(ParsedMessage); +}; + +} // namespace android diff --git a/media/libstagefright/wifi-display/MediaSender.cpp b/media/libstagefright/wifi-display/MediaSender.cpp index 6f1dbbfc87..6e743b5711 100644 --- a/media/libstagefright/wifi-display/MediaSender.cpp +++ b/media/libstagefright/wifi-display/MediaSender.cpp @@ -24,12 +24,13 @@ #include "source/TSPacketizer.h" #include -#include #include #include #include #include -#include +#include +#include +#include #include namespace android { diff --git a/media/libstagefright/wifi-display/rtp/RTPSender.cpp b/media/libstagefright/wifi-display/rtp/RTPSender.cpp index bfdd5fc491..ba01b740ea 100644 --- a/media/libstagefright/wifi-display/rtp/RTPSender.cpp +++ b/media/libstagefright/wifi-display/rtp/RTPSender.cpp @@ -24,9 +24,9 @@ #include #include #include -#include #include #include +#include #include #include diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp index 159b113dcf..96aceb1df4 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include #include diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.h b/media/libstagefright/wifi-display/source/WifiDisplaySource.h index c25a675ee2..47047f8f7b 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.h +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.h @@ -21,7 +21,7 @@ #include "VideoFormats.h" #include -#include +#include #include From 1f8feaf735f37af439e81a361f1e346038f6cb28 Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Wed, 2 Feb 2022 15:07:39 +0100 Subject: [PATCH 13/20] Add group audio to vendor.media.omx service Resolves Qualcomm libs trying to access e.g. /dev/msm_alac and failing. Referred from: https://source.codeaurora.org/quic/la/platform/frameworks/av/commit?id=150662d Test: run vts -m VtsHalMediaOmxV1_0TargetAudioDecTest Change-Id: I28dcc07c00f41ab542b24725d767227854f1b7c9 Signed-off-by: Edwiin Kusuma Jaya --- services/mediacodec/android.hardware.media.omx@1.0-service.rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/mediacodec/android.hardware.media.omx@1.0-service.rc b/services/mediacodec/android.hardware.media.omx@1.0-service.rc index 845e5cc8f0..b5f9ab0cc7 100644 --- a/services/mediacodec/android.hardware.media.omx@1.0-service.rc +++ b/services/mediacodec/android.hardware.media.omx@1.0-service.rc @@ -1,6 +1,6 @@ service vendor.media.omx /vendor/bin/hw/android.hardware.media.omx@1.0-service class main user mediacodec - group camera drmrpc mediadrm + group camera drmrpc mediadrm audio ioprio rt 4 task_profiles ProcessCapacityHigh From c6774d83eac79b674d322804312225f31a9c958c Mon Sep 17 00:00:00 2001 From: Jiajia Cong Date: Thu, 5 Aug 2021 16:33:35 +0800 Subject: [PATCH 14/20] codec2: add smoothfactor when checking n-th work pipeline watcher's queued work counter included smoothfactor, if we don't add it in elapsed(), it is easy to cause CCodec queue call timeout CRs-Fixed: 2984012 Test: Verified on OnePlus 9 that YouTube video does not unpause itself. Change-Id: Ia2dbf3a7cf0da4805398cf44780ad3dacbaf1a1b Signed-off-by: Edwiin Kusuma Jaya --- media/codec2/sfplugin/CCodecBufferChannel.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media/codec2/sfplugin/CCodecBufferChannel.cpp b/media/codec2/sfplugin/CCodecBufferChannel.cpp index c7ab360def..db939b2d13 100644 --- a/media/codec2/sfplugin/CCodecBufferChannel.cpp +++ b/media/codec2/sfplugin/CCodecBufferChannel.cpp @@ -2176,7 +2176,7 @@ PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() { size_t outputDelay = mOutput.lock()->outputDelay; { Mutexed::Locked input(mInput); - n = input->inputDelay + input->pipelineDelay + outputDelay; + n = input->inputDelay + input->pipelineDelay + outputDelay + kSmoothnessFactor; } return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n); } From b329a3839cb163845fb1b4838f2a8a0a28b18c12 Mon Sep 17 00:00:00 2001 From: Michael Bestas Date: Sat, 3 Sep 2022 20:49:09 +0300 Subject: [PATCH 15/20] Revert "Remove libavservices_minijail_vendor" Older device vendor blobs still depend on this. This reverts commit a03603523a5ec96c75153d39a2369c306f88fa70. Change-Id: I15a5d75a8731586e59a6600eb782332ab6844ad5 Signed-off-by: Edwiin Kusuma Jaya --- services/minijail/Android.bp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/services/minijail/Android.bp b/services/minijail/Android.bp index decc5feb70..ff29a9d2cb 100644 --- a/services/minijail/Android.bp +++ b/services/minijail/Android.bp @@ -36,6 +36,17 @@ cc_library_shared { export_include_dirs: ["."], } +// By adding "vendor_available: true" to "libavservices_minijail", we don't +// need to have "libavservices_minijail_vendor" any longer. +// "libavservices_minijail_vendor" will be removed, once we replace it with +// "libavservices_minijail" in all vendor modules. (b/146313710) +cc_library_shared { + name: "libavservices_minijail_vendor", + vendor: true, + defaults: ["libavservices_minijail_defaults"], + export_include_dirs: ["."], +} + // Unit tests. cc_test { name: "libavservices_minijail_unittest", From 815c302e80808c8584a168509ba80c54fff4a5f6 Mon Sep 17 00:00:00 2001 From: Chenyang Zhong Date: Sat, 13 Nov 2021 09:00:18 -0500 Subject: [PATCH 16/20] WifiDisplaySource: raise supported resolution to 1920x1080p60 Signed-off-by: Chenyang Zhong Change-Id: I5ab5e594c030be032353344600c67f6d0d36e7f2 Signed-off-by: Edwiin Kusuma Jaya --- .../wifi-display/source/WifiDisplaySource.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp index 96aceb1df4..4a31978b86 100644 --- a/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp +++ b/media/libstagefright/wifi-display/source/WifiDisplaySource.cpp @@ -79,11 +79,11 @@ WifiDisplaySource::WifiDisplaySource( mSupportedSourceVideoFormats.disableAll(); mSupportedSourceVideoFormats.setNativeResolution( - VideoFormats::RESOLUTION_CEA, 5); // 1280x720 p30 + VideoFormats::RESOLUTION_CEA, 8); // 1920x1080p60 - // Enable all resolutions up to 1280x720p30 + // Enable all resolutions up to 1920x1080p60 mSupportedSourceVideoFormats.enableResolutionUpto( - VideoFormats::RESOLUTION_CEA, 5, + VideoFormats::RESOLUTION_CEA, 8, VideoFormats::PROFILE_CHP, // Constrained High Profile VideoFormats::LEVEL_32); // Level 3.2 } From 4f8c2ffaae491e1fdb45164aaea788b9f747f428 Mon Sep 17 00:00:00 2001 From: Arian Date: Tue, 11 Oct 2022 22:18:21 +0200 Subject: [PATCH 17/20] Camera: Skip stream size check for whitelisted apps This is a squash of the following CLO commits: commit 84b2ea267f2b5a5b8925758ab8f9e9f0db4f6d53 Author: Pavan Kumar Mc Date: Fri Nov 12 21:21:46 2021 +0530 Camera: Avoid roundBufferDimensionsNearest for AIDE2 YUV streams - AIDE2 is vendor enhanced feature which needs special resolutions for full and downscaled YUV streams.Those resolutions are not populated in static capabilities. So avoid roundBufferDimensionsNearest for YUV streams in privileged client case. CRs-Fixed: 3012886 Change-Id: Ia1d134edd1fac2ffd454137aa2a4aa5b9d40c7a7 commit 35df5effc321d630e711161fba02614f0c61abdd Author: Pavan Kumar Mc Date: Tue Aug 17 10:28:21 2021 +0530 Camera: Support raw stream creation for priviledged APPs - Raw stream creation failing as priviledged APP check set to FALSE for new set of call flow to roundBufferDimensionNearest. Setting the condition to TRUE. CRs-Fixed: 3011444 Change-Id: Ia3d5e4744a78b06c00e99fb5be856b24acbff599 commit 573cb73a1c6c4a8bf13791b98c5d539ebffa2d0f Author: Pavan Kumar Mc Date: Thu Jun 24 11:19:35 2021 +0530 Camera: Master callback mode support for MCX raw - Raw stream creation failing for size greater than logical camera resolution as size cannot exceed max raw stream size in available stream configuration. - Allow creating raw streams of greater size for logical camera CRs-Fixed: 2968932 Change-Id: Ibacc3c72e34b2f137546288f67bcdb4e0a38aa23 commit 15de50a2fe471e7d3cc498286f0de31282303fb0 Author: Pavan Kumar Mc Date: Wed Apr 21 17:59:46 2021 +0530 Camera: Master callback mode support for MCX raw - Raw stream creation failing for size greater than logical camera resolution as size cannot exceed max raw stream size in available stream configuration. - Allow creating raw streams of greater size for logical camera CRs-Fixed: 2860300 Change-Id: Ic0f99e55b1519596e49ff2af14895cea1a19c752 commit cee145eaaf32faf0b99f46f957a9fb6ad66abccf Author: Gaoxiang Chen Date: Wed Jun 21 12:57:08 2017 +0530 Camera: Skip stream size check for whitelisted apps. Issue: For quadracfa capture, Blob/YUV output streams need to be configured with custom dimensions which will not be available in advertised stream configurations map. Fix: Skip the stream size check for whitelisted apps to allow configuration of streams with custom dimensions. Setprop to be used: adb shell setprop persist.vendor.camera.privapp.list Change-Id: Id94b40c94f42bf4579dc6d8bb6273003312ea669 Change-Id: If12f9696fbc78edbd5fb4345a48818fa5d74752f Signed-off-by: Adithya R Signed-off-by: Edwiin Kusuma Jaya --- .../api2/CameraDeviceClient.cpp | 14 +++++-- .../api2/CameraDeviceClient.h | 1 + .../device3/Camera3Device.cpp | 5 --- .../utils/SessionConfigurationUtils.cpp | 41 ++++++++++++++++--- .../utils/SessionConfigurationUtils.h | 6 +-- 5 files changed, 51 insertions(+), 16 deletions(-) diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp index 1f8641473c..df687ef162 100644 --- a/services/camera/libcameraservice/api2/CameraDeviceClient.cpp +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.cpp @@ -106,8 +106,16 @@ CameraDeviceClient::CameraDeviceClient(const sp& cameraService, mInputStream(), mStreamingRequestId(REQUEST_ID_NONE), mRequestIdCounter(0), + mPrivilegedClient(false), mOverrideForPerfClass(overrideForPerfClass) { + char value[PROPERTY_VALUE_MAX]; + property_get("persist.vendor.camera.privapp.list", value, ""); + String16 packagelist(value); + if (packagelist.contains(clientPackageName.string())) { + mPrivilegedClient = true; + } + ATRACE_CALL(); ALOGI("CameraDeviceClient %s: Opened", cameraId.string()); } @@ -926,7 +934,7 @@ binder::Status CameraDeviceClient::createStream( res = SessionConfigurationUtils::createSurfaceFromGbp(streamInfo, isStreamInfoValid, surface, bufferProducer, mCameraIdStr, mDevice->infoPhysical(physicalCameraId), sensorPixelModesUsed, dynamicRangeProfile, - streamUseCase, timestampBase, mirrorMode); + streamUseCase, timestampBase, mirrorMode, mPrivilegedClient); if (!res.isOk()) return res; @@ -1286,7 +1294,7 @@ binder::Status CameraDeviceClient::updateOutputConfiguration(int streamId, res = SessionConfigurationUtils::createSurfaceFromGbp(outInfo, /*isStreamInfoValid*/ false, surface, newOutputsMap.valueAt(i), mCameraIdStr, mDevice->infoPhysical(physicalCameraId), sensorPixelModesUsed, dynamicRangeProfile, - streamUseCase, timestampBase, mirrorMode); + streamUseCase, timestampBase, mirrorMode, mPrivilegedClient); if (!res.isOk()) return res; @@ -1660,7 +1668,7 @@ binder::Status CameraDeviceClient::finalizeOutputConfigurations(int32_t streamId res = SessionConfigurationUtils::createSurfaceFromGbp(mStreamInfoMap[streamId], true /*isStreamInfoValid*/, surface, bufferProducer, mCameraIdStr, mDevice->infoPhysical(physicalId), sensorPixelModesUsed, dynamicRangeProfile, - streamUseCase, timestampBase, mirrorMode); + streamUseCase, timestampBase, mirrorMode, mPrivilegedClient); if (!res.isOk()) return res; diff --git a/services/camera/libcameraservice/api2/CameraDeviceClient.h b/services/camera/libcameraservice/api2/CameraDeviceClient.h index c95bb4a470..d1a2f001aa 100644 --- a/services/camera/libcameraservice/api2/CameraDeviceClient.h +++ b/services/camera/libcameraservice/api2/CameraDeviceClient.h @@ -331,6 +331,7 @@ class CameraDeviceClient : static const int32_t REQUEST_ID_NONE = -1; int32_t mRequestIdCounter; + bool mPrivilegedClient; std::vector mPhysicalCameraIds; diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp index e55d724b9f..6fb7dfd71f 100644 --- a/services/camera/libcameraservice/device3/Camera3Device.cpp +++ b/services/camera/libcameraservice/device3/Camera3Device.cpp @@ -441,11 +441,6 @@ ssize_t Camera3Device::getJpegBufferSize(const CameraMetadata &info, uint32_t wi (chosenMaxJpegResolution.width * chosenMaxJpegResolution.height); ssize_t jpegBufferSize = scaleFactor * (maxJpegBufferSize - kMinJpegBufferSize) + kMinJpegBufferSize; - if (jpegBufferSize > maxJpegBufferSize) { - ALOGI("%s: jpeg buffer size calculated is > maxJpeg bufferSize(%zd), clamping", - __FUNCTION__, maxJpegBufferSize); - jpegBufferSize = maxJpegBufferSize; - } return jpegBufferSize; } diff --git a/services/camera/libcameraservice/utils/SessionConfigurationUtils.cpp b/services/camera/libcameraservice/utils/SessionConfigurationUtils.cpp index 7dde268f07..b49f86c0e7 100644 --- a/services/camera/libcameraservice/utils/SessionConfigurationUtils.cpp +++ b/services/camera/libcameraservice/utils/SessionConfigurationUtils.cpp @@ -150,7 +150,7 @@ int64_t euclidDistSquare(int32_t x0, int32_t y0, int32_t x1, int32_t y1) { bool roundBufferDimensionNearest(int32_t width, int32_t height, int32_t format, android_dataspace dataSpace, const CameraMetadata& info, bool maxResolution, /*out*/int32_t* outWidth, - /*out*/int32_t* outHeight) { + /*out*/int32_t* outHeight, bool isPriviledgedClient) { const int32_t depthSizesTag = getAppropriateModeTag(ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS, maxResolution); @@ -191,6 +191,37 @@ bool roundBufferDimensionNearest(int32_t width, int32_t height, } } + if (isPriviledgedClient == true && bestWidth == -1 && + (format == HAL_PIXEL_FORMAT_RAW10 || format == HAL_PIXEL_FORMAT_RAW12 || + format == HAL_PIXEL_FORMAT_RAW16 || format == HAL_PIXEL_FORMAT_RAW_OPAQUE)) { + bool isLogicalCamera = false; + auto entry = info.find(ANDROID_REQUEST_AVAILABLE_CAPABILITIES); + for (size_t i = 0; i < entry.count; ++i) { + uint8_t capability = entry.data.u8[i]; + if (capability == ANDROID_REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA) { + isLogicalCamera = true; + break; + } + } + + if (isLogicalCamera == true) { + bestWidth = width; + bestHeight = height; + } + } + + // Avoid roundBufferDimensionsNearest for privileged client YUV streams to meet the AIDE2 + // requirement. AIDE2 is vendor enhanced feature which requires special resolutions and + // those are not populated in static capabilities. + if (isPriviledgedClient == true && format == HAL_PIXEL_FORMAT_YCbCr_420_888) { + ALOGI("Bypass roundBufferDimensionNearest for privilegedClient YUV streams " + "width %d height %d", + width, height); + + bestWidth = width; + bestHeight = height; + } + if (bestWidth == -1) { // Return false if no configurations for this format were listed return false; @@ -336,7 +367,7 @@ binder::Status createSurfaceFromGbp( sp& surface, const sp& gbp, const String8 &logicalCameraId, const CameraMetadata &physicalCameraMetadata, const std::vector &sensorPixelModesUsed, int64_t dynamicRangeProfile, - int64_t streamUseCase, int timestampBase, int mirrorMode) { + int64_t streamUseCase, int timestampBase, int mirrorMode, bool isPriviledgedClient) { // bufferProducer must be non-null if (gbp == nullptr) { String8 msg = String8::format("Camera %s: Surface is NULL", logicalCameraId.string()); @@ -427,7 +458,7 @@ binder::Status createSurfaceFromGbp( if (flexibleConsumer && isPublicFormat(format) && !SessionConfigurationUtils::roundBufferDimensionNearest(width, height, format, dataSpace, physicalCameraMetadata, foundInMaxRes, /*out*/&width, - /*out*/&height)) { + /*out*/&height, isPriviledgedClient)) { String8 msg = String8::format("Camera %s: No supported stream configurations with " "format %#x defined, failed to create output stream", logicalCameraId.string(), format); @@ -564,7 +595,7 @@ convertToHALStreamCombination( const String8 &logicalCameraId, const CameraMetadata &deviceInfo, metadataGetter getMetadata, const std::vector &physicalCameraIds, aidl::android::hardware::camera::device::StreamConfiguration &streamConfiguration, - bool overrideForPerfClass, bool *earlyExit) { + bool overrideForPerfClass, bool *earlyExit, bool isPriviledgedClient) { using SensorPixelMode = aidl::android::hardware::camera::metadata::SensorPixelMode; auto operatingMode = sessionConfiguration.getOperatingMode(); binder::Status res = checkOperatingMode(operatingMode, deviceInfo, logicalCameraId); @@ -693,7 +724,7 @@ convertToHALStreamCombination( sp surface; res = createSurfaceFromGbp(streamInfo, isStreamInfoValid, surface, bufferProducer, logicalCameraId, metadataChosen, sensorPixelModesUsed, dynamicRangeProfile, - streamUseCase, timestampBase, mirrorMode); + streamUseCase, timestampBase, mirrorMode, isPriviledgedClient); if (!res.isOk()) return res; diff --git a/services/camera/libcameraservice/utils/SessionConfigurationUtils.h b/services/camera/libcameraservice/utils/SessionConfigurationUtils.h index a127c7bdcb..2bc55a88bb 100644 --- a/services/camera/libcameraservice/utils/SessionConfigurationUtils.h +++ b/services/camera/libcameraservice/utils/SessionConfigurationUtils.h @@ -86,7 +86,7 @@ int64_t euclidDistSquare(int32_t x0, int32_t y0, int32_t x1, int32_t y1); // a width <= ROUNDING_WIDTH_CAP bool roundBufferDimensionNearest(int32_t width, int32_t height, int32_t format, android_dataspace dataSpace, const CameraMetadata& info, bool maxResolution, - /*out*/int32_t* outWidth, /*out*/int32_t* outHeight); + /*out*/int32_t* outWidth, /*out*/int32_t* outHeight, bool isPriviledgedClient); // check if format is not custom format bool isPublicFormat(int32_t format); @@ -98,7 +98,7 @@ binder::Status createSurfaceFromGbp( sp& surface, const sp& gbp, const String8 &logicalCameraId, const CameraMetadata &physicalCameraMetadata, const std::vector &sensorPixelModesUsed, int64_t dynamicRangeProfile, - int64_t streamUseCase, int timestampBase, int mirrorMode); + int64_t streamUseCase, int timestampBase, int mirrorMode, bool isPriviledgedClient=false); //check if format is 10-bit output compatible bool is10bitCompatibleFormat(int32_t format); @@ -133,7 +133,7 @@ convertToHALStreamCombination( const String8 &logicalCameraId, const CameraMetadata &deviceInfo, metadataGetter getMetadata, const std::vector &physicalCameraIds, aidl::android::hardware::camera::device::StreamConfiguration &streamConfiguration, - bool overrideForPerfClass, bool *earlyExit); + bool overrideForPerfClass, bool *earlyExit, bool isPriviledgedClient = false); StreamConfigurationPair getStreamConfigurationPair(const CameraMetadata &metadata); From 6228e1ebdd83eb5691edd9486390545add182c75 Mon Sep 17 00:00:00 2001 From: SGCMarkus Date: Sun, 9 Oct 2022 16:12:50 +0200 Subject: [PATCH 18/20] Camera: Avoid roundBufferDimensionsNearest also for HAL_PIXEL_FORMAT_BLOB This fixes the Ultra-Res feature with Motorolas MotCamera3 It uses multiple camera streams, with only the blob format using the full sensor size (e.g. on berlin: 12000 x 9000 for 108 MP) [ghostrider-reborn] - Xiaomi devices also seem to use this, with MiuiCamera (tested with 64 MP mode on lisa, 48 MP on ginkgo etc.) Change-Id: I025e068b2d3ff1123599ae5655655876ee28f39b Signed-off-by: Adithya R Signed-off-by: Edwiin Kusuma Jaya --- .../utils/SessionConfigurationUtils.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/services/camera/libcameraservice/utils/SessionConfigurationUtils.cpp b/services/camera/libcameraservice/utils/SessionConfigurationUtils.cpp index b49f86c0e7..9c6aacd9e2 100644 --- a/services/camera/libcameraservice/utils/SessionConfigurationUtils.cpp +++ b/services/camera/libcameraservice/utils/SessionConfigurationUtils.cpp @@ -210,13 +210,14 @@ bool roundBufferDimensionNearest(int32_t width, int32_t height, } } - // Avoid roundBufferDimensionsNearest for privileged client YUV streams to meet the AIDE2 - // requirement. AIDE2 is vendor enhanced feature which requires special resolutions and - // those are not populated in static capabilities. - if (isPriviledgedClient == true && format == HAL_PIXEL_FORMAT_YCbCr_420_888) { - ALOGI("Bypass roundBufferDimensionNearest for privilegedClient YUV streams " - "width %d height %d", - width, height); + // Avoid roundBufferDimensionsNearest for privileged client Blob/YUV streams to meet the + // AIDE2 requirement. AIDE2 is vendor enhanced feature which requires special resolutions + // and those are not populated in static capabilities. + if (isPriviledgedClient == true && + (format == HAL_PIXEL_FORMAT_YCbCr_420_888 || format == HAL_PIXEL_FORMAT_BLOB)) { + ALOGI("Bypass roundBufferDimensionNearest for privilegedClient Blob/YUV streams " + "width %d height %d format %d", + width, height, format); bestWidth = width; bestHeight = height; From afd6b4d00cd37120fe78e44971b9824da8a2a56e Mon Sep 17 00:00:00 2001 From: basamaryan Date: Sun, 20 Nov 2022 04:27:59 +0000 Subject: [PATCH 19/20] Revert "Revert "Remove libavservices_minijail_vendor"" This reverts commit b02a11a545bc80b09c51d386c9f278907da1247d. Signed-off-by: Edwiin Kusuma Jaya --- services/minijail/Android.bp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/services/minijail/Android.bp b/services/minijail/Android.bp index ff29a9d2cb..decc5feb70 100644 --- a/services/minijail/Android.bp +++ b/services/minijail/Android.bp @@ -36,17 +36,6 @@ cc_library_shared { export_include_dirs: ["."], } -// By adding "vendor_available: true" to "libavservices_minijail", we don't -// need to have "libavservices_minijail_vendor" any longer. -// "libavservices_minijail_vendor" will be removed, once we replace it with -// "libavservices_minijail" in all vendor modules. (b/146313710) -cc_library_shared { - name: "libavservices_minijail_vendor", - vendor: true, - defaults: ["libavservices_minijail_defaults"], - export_include_dirs: ["."], -} - // Unit tests. cc_test { name: "libavservices_minijail_unittest", From b16f9f82f92b305a9bbee2f4e3dfde6cc892646d Mon Sep 17 00:00:00 2001 From: Edwiin Kusuma Jaya Date: Mon, 19 Jun 2023 10:34:16 +0000 Subject: [PATCH 20/20] av: fix callinguid Signed-off-by: Edwiin Kusuma Jaya --- services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp index 331aefad5b..da812b3c52 100644 --- a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp +++ b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp @@ -615,7 +615,8 @@ Status AudioPolicyService::getInputForAttr(const media::AudioAttributesInternal& // type is API_INPUT_MIX_EXT_POLICY_REROUTE and by AudioService if a media projection // is used and input type is API_INPUT_MIX_PUBLIC_CAPTURE_PLAYBACK // - ECHO_REFERENCE source is controlled by captureAudioOutputAllowed() - if (!isAudioServerOrMediaServerOrSystemServerOrRootUid(IPCThreadState::self()->getCallingUid()) && !(recordingAllowed(attributionSource, inputSource) + if (!isAudioServerOrMediaServerUid(attributionSource.uid) + && !(recordingAllowed(attributionSource, inputSource) || inputSource == AUDIO_SOURCE_FM_TUNER || inputSource == AUDIO_SOURCE_REMOTE_SUBMIX || inputSource == AUDIO_SOURCE_ECHO_REFERENCE)) { @@ -781,7 +782,8 @@ Status AudioPolicyService::startInput(int32_t portIdAidl) msg << "Audio recording on session " << client->session; // check calling permissions - if (!isAudioServerOrMediaServerUid(IPCThreadState::self()->getCallingUid()) && !(startRecording(client->attributionSource, String16(msg.str().c_str()), + if (!isAudioServerOrMediaServerUid(client->attributionSource.uid) + && !(startRecording(client->attributionSource, String16(msg.str().c_str()), client->attributes.source) || client->attributes.source == AUDIO_SOURCE_FM_TUNER || client->attributes.source == AUDIO_SOURCE_REMOTE_SUBMIX @@ -2376,4 +2378,4 @@ Status AudioPolicyService::getDirectProfilesForAttributes( return Status::ok(); } -} // namespace android +} // namespace android \ No newline at end of file