From 1a95f2ee163601d91e812c276aebc4affb06e51f Mon Sep 17 00:00:00 2001 From: Johannes Kron Date: Thu, 24 Feb 2022 15:07:27 +0000 Subject: [PATCH] Use spatialScalability instead of scalabilityMode for decodeInfo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An upcoming spec change to the MediaCapabilities API replaces scalabilityMode with spatialScalability when calling decodeInfo. This CL implements that change by adding spatialScalability to VideoConfiguration. See https://github.com/w3c/media-capabilities/pull/189 Bug: chromium:1299424 Change-Id: I46215603640cc7f4cf97e7309be934f8adb1b358 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3484211 Reviewed-by: Will Cassella Reviewed-by: Henrik Boström Commit-Queue: Johannes Kron Cr-Commit-Position: refs/heads/main@{#974639} --- .../media_capabilities/media_capabilities.cc | 25 +++++--- .../media_capabilities_test.cc | 60 +++++++++++++++++++ .../video_configuration.idl | 1 + .../webrtc_decoding_info_handler.cc | 32 ++-------- .../webrtc_decoding_info_handler.h | 2 +- .../webrtc_decoding_info_handler_test.cc | 38 +++++------- .../platform/webrtc/webrtc_video_utils.cc | 45 -------------- .../platform/webrtc/webrtc_video_utils.h | 12 ---- 8 files changed, 100 insertions(+), 115 deletions(-) diff --git a/third_party/blink/renderer/modules/media_capabilities/media_capabilities.cc b/third_party/blink/renderer/modules/media_capabilities/media_capabilities.cc index 0a9c341bf30b30..f06b0b1dc69e17 100644 --- a/third_party/blink/renderer/modules/media_capabilities/media_capabilities.cc +++ b/third_party/blink/renderer/modules/media_capabilities/media_capabilities.cc @@ -276,6 +276,7 @@ bool IsValidMediaConfiguration(const MediaConfiguration* configuration) { } bool IsValidVideoConfiguration(const VideoConfiguration* configuration, + bool is_decode, bool is_webrtc) { DCHECK(configuration->hasContentType()); if (!IsValidMimeType(configuration->contentType(), kVideoMimeTypePrefix, @@ -286,6 +287,13 @@ bool IsValidVideoConfiguration(const VideoConfiguration* configuration, if (!IsValidFrameRate(configuration->framerate())) return false; + // scalabilityMode only valid for WebRTC encode configuration. + if ((!is_webrtc || is_decode) && configuration->hasScalabilityMode()) + return false; + // spatialScalability only valid for WebRTC decode configuration. + if ((!is_webrtc || !is_decode) && configuration->hasSpatialScalability()) + return false; + return true; } @@ -337,7 +345,8 @@ bool IsValidMediaDecodingConfiguration( } if (configuration->hasVideo() && - !IsValidVideoConfiguration(configuration->video(), is_webrtc)) { + !IsValidVideoConfiguration(configuration->video(), /*is_decode=*/true, + is_webrtc)) { *message = "The video configuration dictionary is not valid."; return false; } @@ -363,7 +372,8 @@ bool IsValidMediaEncodingConfiguration( } if (configuration->hasVideo() && - !IsValidVideoConfiguration(configuration->video(), is_webrtc)) { + !IsValidVideoConfiguration(configuration->video(), /*is_decode=*/false, + is_webrtc)) { *message = "The video configuration dictionary is not valid."; return false; } @@ -818,7 +828,7 @@ ScriptPromise MediaCapabilities::decodingInfo( : absl::nullopt; absl::optional sdp_video_format; - absl::optional scalability_mode; + bool spatial_scalability = false; media::VideoCodecProfile codec_profile = media::VIDEO_CODEC_PROFILE_UNKNOWN; int video_pixels = 0; @@ -826,10 +836,9 @@ ScriptPromise MediaCapabilities::decodingInfo( if (config->hasVideo()) { sdp_video_format = absl::make_optional(ToSdpVideoFormat(config->video())); - scalability_mode = - config->video()->hasScalabilityMode() - ? absl::make_optional(config->video()->scalabilityMode()) - : absl::nullopt; + spatial_scalability = config->video()->hasSpatialScalability() + ? config->video()->spatialScalability() + : false; // Additional information needed for lookup in WebrtcVideoPerfHistory. codec_profile = @@ -845,7 +854,7 @@ ScriptPromise MediaCapabilities::decodingInfo( video_pixels, /*hardware_accelerated=*/false); handler->DecodingInfo(sdp_audio_format, sdp_video_format, - scalability_mode, + spatial_scalability, WTF::Bind(&MediaCapabilities::OnWebrtcSupportInfo, WrapPersistent(this), callback_id, std::move(features), frames_per_second)); diff --git a/third_party/blink/renderer/modules/media_capabilities/media_capabilities_test.cc b/third_party/blink/renderer/modules/media_capabilities/media_capabilities_test.cc index 52857be0e3a76b..5c0d93460efade 100644 --- a/third_party/blink/renderer/modules/media_capabilities/media_capabilities_test.cc +++ b/third_party/blink/renderer/modules/media_capabilities/media_capabilities_test.cc @@ -1250,6 +1250,37 @@ TEST(MediaCapabilitiesTests, WebrtcDecodingUnsupportedVideo) { EXPECT_FALSE(info->powerEfficient()); } +TEST(MediaCapabilitiesTests, WebrtcDecodingSpatialScalability) { + MediaCapabilitiesTestContext context; + auto* decoding_config = CreateWebrtcDecodingConfig(); + auto* video_config = decoding_config->getVideoOr(nullptr); + video_config->setSpatialScalability(false); + const media::mojom::blink::WebrtcPredictionFeatures kFeatures = + CreateWebrtcFeatures(/*is_decode=*/true); + + // WebrtcPerfHistoryService should be queried for smoothness. Signal + // smooth=true. + EXPECT_CALL(*context.GetWebrtcPerfHistoryService(), GetPerfInfo(_, _, _)) + .WillOnce(WebrtcDbCallback(kFeatures, kFramerate, /*is_smooth=*/true)); + MediaCapabilitiesInfo* info = DecodingInfo(decoding_config, &context); + EXPECT_TRUE(info->supported()); + EXPECT_TRUE(info->smooth()); + EXPECT_FALSE(info->powerEfficient()); + + // Verify DB call was made. + testing::Mock::VerifyAndClearExpectations( + context.GetWebrtcPerfHistoryService()); + + // Repeat test with spatialScalability=true. + video_config->setSpatialScalability(true); + EXPECT_CALL(*context.GetWebrtcPerfHistoryService(), GetPerfInfo(_, _, _)) + .WillOnce(WebrtcDbCallback(kFeatures, kFramerate, /*is_smooth=*/false)); + info = DecodingInfo(decoding_config, &context); + EXPECT_TRUE(info->supported()); + EXPECT_FALSE(info->smooth()); + EXPECT_FALSE(info->powerEfficient()); +} + // WebRTC encodingInfo tests. TEST(MediaCapabilitiesTests, WebrtcEncodingBasicAudio) { MediaCapabilitiesTestContext context; @@ -1326,4 +1357,33 @@ TEST(MediaCapabilitiesTests, WebrtcEncodingUnsupportedVideo) { EXPECT_FALSE(info->powerEfficient()); } +TEST(MediaCapabilitiesTests, WebrtcEncodingScalabilityMode) { + MediaCapabilitiesTestContext context; + auto* encoding_config = CreateWebrtcEncodingConfig(); + auto* video_config = encoding_config->getVideoOr(nullptr); + video_config->setScalabilityMode("L3T3_KEY"); + const media::mojom::blink::WebrtcPredictionFeatures kFeatures = + CreateWebrtcFeatures(/*is_decode=*/false); + + // WebrtcPerfHistoryService should be queried for smoothness. Signal + // smooth=true. + EXPECT_CALL(*context.GetWebrtcPerfHistoryService(), GetPerfInfo(_, _, _)) + .WillOnce(WebrtcDbCallback(kFeatures, kFramerate, /*is_smooth=*/true)); + MediaCapabilitiesInfo* info = EncodingInfo(encoding_config, &context); + EXPECT_TRUE(info->supported()); + EXPECT_TRUE(info->smooth()); + EXPECT_FALSE(info->powerEfficient()); + + // Verify DB call was made. + testing::Mock::VerifyAndClearExpectations( + context.GetWebrtcPerfHistoryService()); + + // Repeat with unsupported mode. + video_config->setScalabilityMode("L3T2_Foo"); + info = EncodingInfo(encoding_config, &context); + EXPECT_FALSE(info->supported()); + EXPECT_FALSE(info->smooth()); + EXPECT_FALSE(info->powerEfficient()); +} + } // namespace blink diff --git a/third_party/blink/renderer/modules/media_capabilities/video_configuration.idl b/third_party/blink/renderer/modules/media_capabilities/video_configuration.idl index cf9e35c6b8ae2f..7d951dff97ce4a 100644 --- a/third_party/blink/renderer/modules/media_capabilities/video_configuration.idl +++ b/third_party/blink/renderer/modules/media_capabilities/video_configuration.idl @@ -18,4 +18,5 @@ dictionary VideoConfiguration { [RuntimeEnabled=MediaCapabilitiesDynamicRange] ColorGamut colorGamut; [RuntimeEnabled=MediaCapabilitiesDynamicRange] TransferFunction transferFunction; [RuntimeEnabled=MediaCapabilitiesWebRtc] DOMString scalabilityMode; + [RuntimeEnabled=MediaCapabilitiesWebRtc] boolean spatialScalability; }; diff --git a/third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.cc b/third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.cc index 100207b6688f2f..de53fb5f4c638d 100644 --- a/third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.cc +++ b/third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.cc @@ -12,7 +12,6 @@ #include "third_party/blink/public/platform/platform.h" #include "third_party/blink/renderer/platform/peerconnection/audio_codec_factory.h" #include "third_party/blink/renderer/platform/peerconnection/video_codec_factory.h" -#include "third_party/blink/renderer/platform/webrtc/webrtc_video_utils.h" #include "third_party/blink/renderer/platform/wtf/text/string_hash.h" #include "third_party/webrtc/api/audio_codecs/audio_decoder_factory.h" #include "third_party/webrtc/api/audio_codecs/audio_format.h" @@ -54,7 +53,7 @@ WebrtcDecodingInfoHandler::~WebrtcDecodingInfoHandler() = default; void WebrtcDecodingInfoHandler::DecodingInfo( const absl::optional sdp_audio_format, const absl::optional sdp_video_format, - const absl::optional video_scalability_mode, + const bool video_spatial_scalability, OnMediaCapabilitiesDecodingInfoCallback callback) const { DCHECK(sdp_audio_format || sdp_video_format); @@ -75,30 +74,11 @@ void WebrtcDecodingInfoHandler::DecodingInfo( // Only check video configuration if the audio configuration was supported (or // not specified). if (sdp_video_format && supported) { - bool reference_scaling = false; - if (video_scalability_mode) { - absl::optional dependent_spatial_layers = - WebRtcScalabilityModeDependentSpatialLayers( - video_scalability_mode->Utf8()); - if (dependent_spatial_layers) { - // Reference scaling must be supported to be able to decode a stream - // with more than one dependent spatial layers. - reference_scaling = *dependent_spatial_layers > 1; - } else { - // We were not able to decode the scalability mode string. Return - // unsupported. - supported = false; - power_efficient = false; - } - } - - if (supported) { - webrtc::VideoDecoderFactory::CodecSupport support = - video_decoder_factory_->QueryCodecSupport(*sdp_video_format, - reference_scaling); - supported = support.is_supported; - power_efficient = support.is_power_efficient; - } + webrtc::VideoDecoderFactory::CodecSupport support = + video_decoder_factory_->QueryCodecSupport(*sdp_video_format, + video_spatial_scalability); + supported = support.is_supported; + power_efficient = support.is_power_efficient; DVLOG(1) << "Video:" << sdp_video_format->name << " supported:" << supported << " power_efficient:" << power_efficient; } diff --git a/third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.h b/third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.h index 53f6f392d0712b..e2eb11cf3777d9 100644 --- a/third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.h +++ b/third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.h @@ -43,7 +43,7 @@ class PLATFORM_EXPORT WebrtcDecodingInfoHandler { void DecodingInfo( const absl::optional sdp_audio_format, const absl::optional sdp_video_format, - const absl::optional video_scalability_mode, + const bool video_spatial_scalability, OnMediaCapabilitiesDecodingInfoCallback callback) const; private: diff --git a/third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler_test.cc b/third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler_test.cc index 53d623d985012c..f51a03300d3aa8 100644 --- a/third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler_test.cc +++ b/third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler_test.cc @@ -45,7 +45,7 @@ class MockVideoDecoderFactory : public webrtc::VideoDecoderFactory { (const)); MOCK_METHOD(webrtc::VideoDecoderFactory::CodecSupport, QueryCodecSupport, - (const webrtc::SdpVideoFormat& format, bool reference_scaling), + (const webrtc::SdpVideoFormat& format, bool spatial_scalability), (const, override)); }; @@ -88,17 +88,16 @@ class WebrtcDecodingInfoHandlerTests : public ::testing::Test { void VerifyDecodingInfo( const absl::optional sdp_audio_format, const absl::optional sdp_video_format, - const absl::optional video_scalability_mode, - const bool expected_reference_scaling, + const bool video_spatial_scalability, const CodecSupport support) { if (sdp_video_format) { ON_CALL(*mock_video_decoder_factory_, QueryCodecSupport) .WillByDefault( - testing::Invoke([sdp_video_format, expected_reference_scaling, + testing::Invoke([sdp_video_format, video_spatial_scalability, support](const webrtc::SdpVideoFormat& format, - bool reference_scaling) { + bool spatial_scalability) { EXPECT_TRUE(format.IsSameCodec(*sdp_video_format)); - EXPECT_EQ(reference_scaling, expected_reference_scaling); + EXPECT_EQ(spatial_scalability, video_spatial_scalability); return support; })); EXPECT_CALL(*mock_video_decoder_factory_, QueryCodecSupport) @@ -109,7 +108,7 @@ class WebrtcDecodingInfoHandlerTests : public ::testing::Test { MediaCapabilitiesDecodingInfoCallback decoding_info_callback; decoding_info_handler.DecodingInfo( - sdp_audio_format, sdp_video_format, video_scalability_mode, + sdp_audio_format, sdp_video_format, video_spatial_scalability, base::BindOnce( &MediaCapabilitiesDecodingInfoCallback::OnWebrtcDecodingInfoSupport, base::Unretained(&decoding_info_callback))); @@ -131,16 +130,14 @@ class WebrtcDecodingInfoHandlerTests : public ::testing::Test { TEST_F(WebrtcDecodingInfoHandlerTests, BasicAudio) { VerifyDecodingInfo( kAudioFormatOpus, /*sdp_video_format=*/absl::nullopt, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/true}); } TEST_F(WebrtcDecodingInfoHandlerTests, UnsupportedAudio) { VerifyDecodingInfo( kAudioFormatFoo, /*sdp_video_format=*/absl::nullopt, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/false, /*is_power_efficient=*/false}); } @@ -151,47 +148,42 @@ TEST_F(WebrtcDecodingInfoHandlerTests, UnsupportedAudio) { TEST_F(WebrtcDecodingInfoHandlerTests, BasicVideo) { VerifyDecodingInfo( /*sdp _audio_format=*/absl::nullopt, kVideoFormatVp9, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/false}); } TEST_F(WebrtcDecodingInfoHandlerTests, BasicVideoPowerEfficient) { VerifyDecodingInfo( /*sdp _audio_format=*/absl::nullopt, kVideoFormatVp9, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/true}); } TEST_F(WebrtcDecodingInfoHandlerTests, UnsupportedVideo) { VerifyDecodingInfo( /*sdp _audio_format=*/absl::nullopt, kVideoFormatFoo, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/false}); } TEST_F(WebrtcDecodingInfoHandlerTests, VideoWithReferenceScaling) { VerifyDecodingInfo( - /*sdp _audio_format=*/absl::nullopt, kVideoFormatVp9, "L3T3", - /*expected_reference_scaling=*/true, + /*sdp _audio_format=*/absl::nullopt, kVideoFormatVp9, + /*video_spatial_scalability=*/true, CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/false}); } TEST_F(WebrtcDecodingInfoHandlerTests, SupportedAudioUnsupportedVideo) { VerifyDecodingInfo( kAudioFormatOpus, kVideoFormatFoo, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/false, /*is_power_efficient=*/false}); } TEST_F(WebrtcDecodingInfoHandlerTests, SupportedVideoUnsupportedAudio) { VerifyDecodingInfo( kAudioFormatFoo, kVideoFormatVp9, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/false, /*is_power_efficient=*/false}); } diff --git a/third_party/blink/renderer/platform/webrtc/webrtc_video_utils.cc b/third_party/blink/renderer/platform/webrtc/webrtc_video_utils.cc index fcddb238a38fd3..b3a43878494f80 100644 --- a/third_party/blink/renderer/platform/webrtc/webrtc_video_utils.cc +++ b/third_party/blink/renderer/platform/webrtc/webrtc_video_utils.cc @@ -9,32 +9,6 @@ #include "third_party/webrtc/api/video_codecs/vp9_profile.h" namespace blink { -namespace { -struct ScalabilityModeSpatialLayers { - const char* name; - int spatial_layers; - int dependent_spatial_layers; -}; - -constexpr ScalabilityModeSpatialLayers kSvcSpatialLayers[] = { - // See https://www.w3.org/TR/webrtc-svc/ for a list of scalability modes. - {"L1T2", 1, 1}, {"L1T3", 1, 1}, - {"L2T1", 2, 2}, {"L2T2", 2, 2}, - {"L2T3", 2, 2}, {"L2T1h", 2, 2}, - {"L2T2h", 2, 2}, {"L2T3h", 2, 2}, - {"S2T1", 2, 1}, {"S2T2", 2, 1}, - {"S2T3", 2, 1}, {"S2T1h", 2, 1}, - {"S2T2h", 2, 1}, {"S2T3h", 2, 1}, - {"L3T1", 3, 3}, {"L3T2", 3, 3}, - {"L3T3", 3, 3}, {"S3T1", 3, 1}, - {"S3T2", 3, 1}, {"S3T3", 3, 1}, - {"S3T1h", 3, 1}, {"S3T2h", 3, 1}, - {"S3T3h", 3, 1}, {"L2T2_KEY", 2, 2}, - {"L2T2_KEY_SHIFT", 2, 2}, {"L2T3_KEY", 2, 2}, - {"L2T3_KEY_SHIFT", 2, 2}, {"L3T2_KEY", 3, 3}, - {"L3T2_KEY_SHIFT", 3, 3}, {"L3T3_KEY", 3, 3}}; - -} // namespace media::VideoRotation WebRtcToMediaVideoRotation( webrtc::VideoRotation rotation) { @@ -267,23 +241,4 @@ media::VideoColorSpace WebRtcToMediaVideoColorSpace( return media::VideoColorSpace(primaries, transfer, matrix, range); } -absl::optional WebRtcScalabilityModeSpatialLayers( - const std::string& scalability_mode) { - for (const auto& entry : kSvcSpatialLayers) { - if (entry.name == scalability_mode) { - return entry.spatial_layers; - } - } - return absl::nullopt; -} - -absl::optional WebRtcScalabilityModeDependentSpatialLayers( - const std::string& scalability_mode) { - for (const auto& entry : kSvcSpatialLayers) { - if (entry.name == scalability_mode) { - return entry.dependent_spatial_layers; - } - } - return absl::nullopt; -} } // namespace blink diff --git a/third_party/blink/renderer/platform/webrtc/webrtc_video_utils.h b/third_party/blink/renderer/platform/webrtc/webrtc_video_utils.h index 1db00d53c9cb51..5edb0b3879fe0e 100644 --- a/third_party/blink/renderer/platform/webrtc/webrtc_video_utils.h +++ b/third_party/blink/renderer/platform/webrtc/webrtc_video_utils.h @@ -32,18 +32,6 @@ WebRtcVideoFormatToMediaVideoCodecProfile(const webrtc::SdpVideoFormat& format); media::VideoColorSpace PLATFORM_EXPORT WebRtcToMediaVideoColorSpace(const webrtc::ColorSpace& color_space); -// Returns the number of spatial layers that is used in the specified -// scalability mode. An empty return value means that the specified scalability -// mode is unsupported. -absl::optional PLATFORM_EXPORT -WebRtcScalabilityModeSpatialLayers(const std::string& scalability_mode); - -// Returns the number of dependent spatial layers that is used in the specified -// scalability mode. An empty return value means that the specified scalability -// mode is unsupported. -absl::optional PLATFORM_EXPORT WebRtcScalabilityModeDependentSpatialLayers( - const std::string& scalability_mode); - } // namespace blink #endif // THIRD_PARTY_BLINK_RENDERER_PLATFORM_WEBRTC_WEBRTC_VIDEO_UTILS_H_