diff --git a/blink/renderer/modules/media_capabilities/media_capabilities.cc b/blink/renderer/modules/media_capabilities/media_capabilities.cc index 0a9c341bf30..f06b0b1dc69 100644 --- a/blink/renderer/modules/media_capabilities/media_capabilities.cc +++ b/blink/renderer/modules/media_capabilities/media_capabilities.cc @@ -276,6 +276,7 @@ bool IsValidMediaConfiguration(const MediaConfiguration* configuration) { } bool IsValidVideoConfiguration(const VideoConfiguration* configuration, + bool is_decode, bool is_webrtc) { DCHECK(configuration->hasContentType()); if (!IsValidMimeType(configuration->contentType(), kVideoMimeTypePrefix, @@ -286,6 +287,13 @@ bool IsValidVideoConfiguration(const VideoConfiguration* configuration, if (!IsValidFrameRate(configuration->framerate())) return false; + // scalabilityMode only valid for WebRTC encode configuration. + if ((!is_webrtc || is_decode) && configuration->hasScalabilityMode()) + return false; + // spatialScalability only valid for WebRTC decode configuration. + if ((!is_webrtc || !is_decode) && configuration->hasSpatialScalability()) + return false; + return true; } @@ -337,7 +345,8 @@ bool IsValidMediaDecodingConfiguration( } if (configuration->hasVideo() && - !IsValidVideoConfiguration(configuration->video(), is_webrtc)) { + !IsValidVideoConfiguration(configuration->video(), /*is_decode=*/true, + is_webrtc)) { *message = "The video configuration dictionary is not valid."; return false; } @@ -363,7 +372,8 @@ bool IsValidMediaEncodingConfiguration( } if (configuration->hasVideo() && - !IsValidVideoConfiguration(configuration->video(), is_webrtc)) { + !IsValidVideoConfiguration(configuration->video(), /*is_decode=*/false, + is_webrtc)) { *message = "The video configuration dictionary is not valid."; return false; } @@ -818,7 +828,7 @@ ScriptPromise MediaCapabilities::decodingInfo( : absl::nullopt; absl::optional sdp_video_format; - absl::optional scalability_mode; + bool spatial_scalability = false; media::VideoCodecProfile codec_profile = media::VIDEO_CODEC_PROFILE_UNKNOWN; int video_pixels = 0; @@ -826,10 +836,9 @@ ScriptPromise MediaCapabilities::decodingInfo( if (config->hasVideo()) { sdp_video_format = absl::make_optional(ToSdpVideoFormat(config->video())); - scalability_mode = - config->video()->hasScalabilityMode() - ? absl::make_optional(config->video()->scalabilityMode()) - : absl::nullopt; + spatial_scalability = config->video()->hasSpatialScalability() + ? config->video()->spatialScalability() + : false; // Additional information needed for lookup in WebrtcVideoPerfHistory. codec_profile = @@ -845,7 +854,7 @@ ScriptPromise MediaCapabilities::decodingInfo( video_pixels, /*hardware_accelerated=*/false); handler->DecodingInfo(sdp_audio_format, sdp_video_format, - scalability_mode, + spatial_scalability, WTF::Bind(&MediaCapabilities::OnWebrtcSupportInfo, WrapPersistent(this), callback_id, std::move(features), frames_per_second)); diff --git a/blink/renderer/modules/media_capabilities/media_capabilities_test.cc b/blink/renderer/modules/media_capabilities/media_capabilities_test.cc index 52857be0e3a..5c0d93460ef 100644 --- a/blink/renderer/modules/media_capabilities/media_capabilities_test.cc +++ b/blink/renderer/modules/media_capabilities/media_capabilities_test.cc @@ -1250,6 +1250,37 @@ TEST(MediaCapabilitiesTests, WebrtcDecodingUnsupportedVideo) { EXPECT_FALSE(info->powerEfficient()); } +TEST(MediaCapabilitiesTests, WebrtcDecodingSpatialScalability) { + MediaCapabilitiesTestContext context; + auto* decoding_config = CreateWebrtcDecodingConfig(); + auto* video_config = decoding_config->getVideoOr(nullptr); + video_config->setSpatialScalability(false); + const media::mojom::blink::WebrtcPredictionFeatures kFeatures = + CreateWebrtcFeatures(/*is_decode=*/true); + + // WebrtcPerfHistoryService should be queried for smoothness. Signal + // smooth=true. + EXPECT_CALL(*context.GetWebrtcPerfHistoryService(), GetPerfInfo(_, _, _)) + .WillOnce(WebrtcDbCallback(kFeatures, kFramerate, /*is_smooth=*/true)); + MediaCapabilitiesInfo* info = DecodingInfo(decoding_config, &context); + EXPECT_TRUE(info->supported()); + EXPECT_TRUE(info->smooth()); + EXPECT_FALSE(info->powerEfficient()); + + // Verify DB call was made. + testing::Mock::VerifyAndClearExpectations( + context.GetWebrtcPerfHistoryService()); + + // Repeat test with spatialScalability=true. + video_config->setSpatialScalability(true); + EXPECT_CALL(*context.GetWebrtcPerfHistoryService(), GetPerfInfo(_, _, _)) + .WillOnce(WebrtcDbCallback(kFeatures, kFramerate, /*is_smooth=*/false)); + info = DecodingInfo(decoding_config, &context); + EXPECT_TRUE(info->supported()); + EXPECT_FALSE(info->smooth()); + EXPECT_FALSE(info->powerEfficient()); +} + // WebRTC encodingInfo tests. TEST(MediaCapabilitiesTests, WebrtcEncodingBasicAudio) { MediaCapabilitiesTestContext context; @@ -1326,4 +1357,33 @@ TEST(MediaCapabilitiesTests, WebrtcEncodingUnsupportedVideo) { EXPECT_FALSE(info->powerEfficient()); } +TEST(MediaCapabilitiesTests, WebrtcEncodingScalabilityMode) { + MediaCapabilitiesTestContext context; + auto* encoding_config = CreateWebrtcEncodingConfig(); + auto* video_config = encoding_config->getVideoOr(nullptr); + video_config->setScalabilityMode("L3T3_KEY"); + const media::mojom::blink::WebrtcPredictionFeatures kFeatures = + CreateWebrtcFeatures(/*is_decode=*/false); + + // WebrtcPerfHistoryService should be queried for smoothness. Signal + // smooth=true. + EXPECT_CALL(*context.GetWebrtcPerfHistoryService(), GetPerfInfo(_, _, _)) + .WillOnce(WebrtcDbCallback(kFeatures, kFramerate, /*is_smooth=*/true)); + MediaCapabilitiesInfo* info = EncodingInfo(encoding_config, &context); + EXPECT_TRUE(info->supported()); + EXPECT_TRUE(info->smooth()); + EXPECT_FALSE(info->powerEfficient()); + + // Verify DB call was made. + testing::Mock::VerifyAndClearExpectations( + context.GetWebrtcPerfHistoryService()); + + // Repeat with unsupported mode. + video_config->setScalabilityMode("L3T2_Foo"); + info = EncodingInfo(encoding_config, &context); + EXPECT_FALSE(info->supported()); + EXPECT_FALSE(info->smooth()); + EXPECT_FALSE(info->powerEfficient()); +} + } // namespace blink diff --git a/blink/renderer/modules/media_capabilities/video_configuration.idl b/blink/renderer/modules/media_capabilities/video_configuration.idl index cf9e35c6b8a..7d951dff97c 100644 --- a/blink/renderer/modules/media_capabilities/video_configuration.idl +++ b/blink/renderer/modules/media_capabilities/video_configuration.idl @@ -18,4 +18,5 @@ dictionary VideoConfiguration { [RuntimeEnabled=MediaCapabilitiesDynamicRange] ColorGamut colorGamut; [RuntimeEnabled=MediaCapabilitiesDynamicRange] TransferFunction transferFunction; [RuntimeEnabled=MediaCapabilitiesWebRtc] DOMString scalabilityMode; + [RuntimeEnabled=MediaCapabilitiesWebRtc] boolean spatialScalability; }; diff --git a/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.cc b/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.cc index 100207b6688..de53fb5f4c6 100644 --- a/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.cc +++ b/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.cc @@ -12,7 +12,6 @@ #include "third_party/blink/public/platform/platform.h" #include "third_party/blink/renderer/platform/peerconnection/audio_codec_factory.h" #include "third_party/blink/renderer/platform/peerconnection/video_codec_factory.h" -#include "third_party/blink/renderer/platform/webrtc/webrtc_video_utils.h" #include "third_party/blink/renderer/platform/wtf/text/string_hash.h" #include "third_party/webrtc/api/audio_codecs/audio_decoder_factory.h" #include "third_party/webrtc/api/audio_codecs/audio_format.h" @@ -54,7 +53,7 @@ WebrtcDecodingInfoHandler::~WebrtcDecodingInfoHandler() = default; void WebrtcDecodingInfoHandler::DecodingInfo( const absl::optional sdp_audio_format, const absl::optional sdp_video_format, - const absl::optional video_scalability_mode, + const bool video_spatial_scalability, OnMediaCapabilitiesDecodingInfoCallback callback) const { DCHECK(sdp_audio_format || sdp_video_format); @@ -75,30 +74,11 @@ void WebrtcDecodingInfoHandler::DecodingInfo( // Only check video configuration if the audio configuration was supported (or // not specified). if (sdp_video_format && supported) { - bool reference_scaling = false; - if (video_scalability_mode) { - absl::optional dependent_spatial_layers = - WebRtcScalabilityModeDependentSpatialLayers( - video_scalability_mode->Utf8()); - if (dependent_spatial_layers) { - // Reference scaling must be supported to be able to decode a stream - // with more than one dependent spatial layers. - reference_scaling = *dependent_spatial_layers > 1; - } else { - // We were not able to decode the scalability mode string. Return - // unsupported. - supported = false; - power_efficient = false; - } - } - - if (supported) { - webrtc::VideoDecoderFactory::CodecSupport support = - video_decoder_factory_->QueryCodecSupport(*sdp_video_format, - reference_scaling); - supported = support.is_supported; - power_efficient = support.is_power_efficient; - } + webrtc::VideoDecoderFactory::CodecSupport support = + video_decoder_factory_->QueryCodecSupport(*sdp_video_format, + video_spatial_scalability); + supported = support.is_supported; + power_efficient = support.is_power_efficient; DVLOG(1) << "Video:" << sdp_video_format->name << " supported:" << supported << " power_efficient:" << power_efficient; } diff --git a/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.h b/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.h index 53f6f392d07..e2eb11cf377 100644 --- a/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.h +++ b/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.h @@ -43,7 +43,7 @@ class PLATFORM_EXPORT WebrtcDecodingInfoHandler { void DecodingInfo( const absl::optional sdp_audio_format, const absl::optional sdp_video_format, - const absl::optional video_scalability_mode, + const bool video_spatial_scalability, OnMediaCapabilitiesDecodingInfoCallback callback) const; private: diff --git a/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler_test.cc b/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler_test.cc index 53d623d9850..f51a03300d3 100644 --- a/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler_test.cc +++ b/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler_test.cc @@ -45,7 +45,7 @@ class MockVideoDecoderFactory : public webrtc::VideoDecoderFactory { (const)); MOCK_METHOD(webrtc::VideoDecoderFactory::CodecSupport, QueryCodecSupport, - (const webrtc::SdpVideoFormat& format, bool reference_scaling), + (const webrtc::SdpVideoFormat& format, bool spatial_scalability), (const, override)); }; @@ -88,17 +88,16 @@ class WebrtcDecodingInfoHandlerTests : public ::testing::Test { void VerifyDecodingInfo( const absl::optional sdp_audio_format, const absl::optional sdp_video_format, - const absl::optional video_scalability_mode, - const bool expected_reference_scaling, + const bool video_spatial_scalability, const CodecSupport support) { if (sdp_video_format) { ON_CALL(*mock_video_decoder_factory_, QueryCodecSupport) .WillByDefault( - testing::Invoke([sdp_video_format, expected_reference_scaling, + testing::Invoke([sdp_video_format, video_spatial_scalability, support](const webrtc::SdpVideoFormat& format, - bool reference_scaling) { + bool spatial_scalability) { EXPECT_TRUE(format.IsSameCodec(*sdp_video_format)); - EXPECT_EQ(reference_scaling, expected_reference_scaling); + EXPECT_EQ(spatial_scalability, video_spatial_scalability); return support; })); EXPECT_CALL(*mock_video_decoder_factory_, QueryCodecSupport) @@ -109,7 +108,7 @@ class WebrtcDecodingInfoHandlerTests : public ::testing::Test { MediaCapabilitiesDecodingInfoCallback decoding_info_callback; decoding_info_handler.DecodingInfo( - sdp_audio_format, sdp_video_format, video_scalability_mode, + sdp_audio_format, sdp_video_format, video_spatial_scalability, base::BindOnce( &MediaCapabilitiesDecodingInfoCallback::OnWebrtcDecodingInfoSupport, base::Unretained(&decoding_info_callback))); @@ -131,16 +130,14 @@ class WebrtcDecodingInfoHandlerTests : public ::testing::Test { TEST_F(WebrtcDecodingInfoHandlerTests, BasicAudio) { VerifyDecodingInfo( kAudioFormatOpus, /*sdp_video_format=*/absl::nullopt, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/true}); } TEST_F(WebrtcDecodingInfoHandlerTests, UnsupportedAudio) { VerifyDecodingInfo( kAudioFormatFoo, /*sdp_video_format=*/absl::nullopt, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/false, /*is_power_efficient=*/false}); } @@ -151,47 +148,42 @@ TEST_F(WebrtcDecodingInfoHandlerTests, UnsupportedAudio) { TEST_F(WebrtcDecodingInfoHandlerTests, BasicVideo) { VerifyDecodingInfo( /*sdp _audio_format=*/absl::nullopt, kVideoFormatVp9, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/false}); } TEST_F(WebrtcDecodingInfoHandlerTests, BasicVideoPowerEfficient) { VerifyDecodingInfo( /*sdp _audio_format=*/absl::nullopt, kVideoFormatVp9, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/true}); } TEST_F(WebrtcDecodingInfoHandlerTests, UnsupportedVideo) { VerifyDecodingInfo( /*sdp _audio_format=*/absl::nullopt, kVideoFormatFoo, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/false}); } TEST_F(WebrtcDecodingInfoHandlerTests, VideoWithReferenceScaling) { VerifyDecodingInfo( - /*sdp _audio_format=*/absl::nullopt, kVideoFormatVp9, "L3T3", - /*expected_reference_scaling=*/true, + /*sdp _audio_format=*/absl::nullopt, kVideoFormatVp9, + /*video_spatial_scalability=*/true, CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/false}); } TEST_F(WebrtcDecodingInfoHandlerTests, SupportedAudioUnsupportedVideo) { VerifyDecodingInfo( kAudioFormatOpus, kVideoFormatFoo, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/false, /*is_power_efficient=*/false}); } TEST_F(WebrtcDecodingInfoHandlerTests, SupportedVideoUnsupportedAudio) { VerifyDecodingInfo( kAudioFormatFoo, kVideoFormatVp9, - /*video_scalability_mode=*/absl::nullopt, - /*expected_reference_scaling=*/false, + /*video_spatial_scalability=*/false, CodecSupport{/*is_supported=*/false, /*is_power_efficient=*/false}); } diff --git a/blink/renderer/platform/webrtc/webrtc_video_utils.cc b/blink/renderer/platform/webrtc/webrtc_video_utils.cc index fcddb238a38..b3a43878494 100644 --- a/blink/renderer/platform/webrtc/webrtc_video_utils.cc +++ b/blink/renderer/platform/webrtc/webrtc_video_utils.cc @@ -9,32 +9,6 @@ #include "third_party/webrtc/api/video_codecs/vp9_profile.h" namespace blink { -namespace { -struct ScalabilityModeSpatialLayers { - const char* name; - int spatial_layers; - int dependent_spatial_layers; -}; - -constexpr ScalabilityModeSpatialLayers kSvcSpatialLayers[] = { - // See https://www.w3.org/TR/webrtc-svc/ for a list of scalability modes. - {"L1T2", 1, 1}, {"L1T3", 1, 1}, - {"L2T1", 2, 2}, {"L2T2", 2, 2}, - {"L2T3", 2, 2}, {"L2T1h", 2, 2}, - {"L2T2h", 2, 2}, {"L2T3h", 2, 2}, - {"S2T1", 2, 1}, {"S2T2", 2, 1}, - {"S2T3", 2, 1}, {"S2T1h", 2, 1}, - {"S2T2h", 2, 1}, {"S2T3h", 2, 1}, - {"L3T1", 3, 3}, {"L3T2", 3, 3}, - {"L3T3", 3, 3}, {"S3T1", 3, 1}, - {"S3T2", 3, 1}, {"S3T3", 3, 1}, - {"S3T1h", 3, 1}, {"S3T2h", 3, 1}, - {"S3T3h", 3, 1}, {"L2T2_KEY", 2, 2}, - {"L2T2_KEY_SHIFT", 2, 2}, {"L2T3_KEY", 2, 2}, - {"L2T3_KEY_SHIFT", 2, 2}, {"L3T2_KEY", 3, 3}, - {"L3T2_KEY_SHIFT", 3, 3}, {"L3T3_KEY", 3, 3}}; - -} // namespace media::VideoRotation WebRtcToMediaVideoRotation( webrtc::VideoRotation rotation) { @@ -267,23 +241,4 @@ media::VideoColorSpace WebRtcToMediaVideoColorSpace( return media::VideoColorSpace(primaries, transfer, matrix, range); } -absl::optional WebRtcScalabilityModeSpatialLayers( - const std::string& scalability_mode) { - for (const auto& entry : kSvcSpatialLayers) { - if (entry.name == scalability_mode) { - return entry.spatial_layers; - } - } - return absl::nullopt; -} - -absl::optional WebRtcScalabilityModeDependentSpatialLayers( - const std::string& scalability_mode) { - for (const auto& entry : kSvcSpatialLayers) { - if (entry.name == scalability_mode) { - return entry.dependent_spatial_layers; - } - } - return absl::nullopt; -} } // namespace blink diff --git a/blink/renderer/platform/webrtc/webrtc_video_utils.h b/blink/renderer/platform/webrtc/webrtc_video_utils.h index 1db00d53c9c..5edb0b3879f 100644 --- a/blink/renderer/platform/webrtc/webrtc_video_utils.h +++ b/blink/renderer/platform/webrtc/webrtc_video_utils.h @@ -32,18 +32,6 @@ WebRtcVideoFormatToMediaVideoCodecProfile(const webrtc::SdpVideoFormat& format); media::VideoColorSpace PLATFORM_EXPORT WebRtcToMediaVideoColorSpace(const webrtc::ColorSpace& color_space); -// Returns the number of spatial layers that is used in the specified -// scalability mode. An empty return value means that the specified scalability -// mode is unsupported. -absl::optional PLATFORM_EXPORT -WebRtcScalabilityModeSpatialLayers(const std::string& scalability_mode); - -// Returns the number of dependent spatial layers that is used in the specified -// scalability mode. An empty return value means that the specified scalability -// mode is unsupported. -absl::optional PLATFORM_EXPORT WebRtcScalabilityModeDependentSpatialLayers( - const std::string& scalability_mode); - } // namespace blink #endif // THIRD_PARTY_BLINK_RENDERER_PLATFORM_WEBRTC_WEBRTC_VIDEO_UTILS_H_