Skip to content

Commit

Permalink
Use spatialScalability instead of scalabilityMode for decodeInfo
Browse files Browse the repository at this point in the history 
An upcoming spec change to the MediaCapabilities API replaces
scalabilityMode with spatialScalability when calling
decodeInfo. This CL implements that change by adding
spatialScalability to VideoConfiguration.

See w3c/media-capabilities#189

Bug: chromium:1299424
Change-Id: I46215603640cc7f4cf97e7309be934f8adb1b358
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/3484211
Reviewed-by: Will Cassella <[email protected]>
Reviewed-by: Henrik Boström <[email protected]>
Commit-Queue: Johannes Kron <[email protected]>
Cr-Commit-Position: refs/heads/main@{#974639}
  • Loading branch information
Johannes Kron authored and Chromium LUCI CQ committed Feb 24, 2022
1 parent f84257a commit 1a95f2e
Show file tree
Hide file tree
Showing 8 changed files with 100 additions and 115 deletions.
25 changes: 17 additions & 8 deletions third_party/blink/renderer/modules/media_capabilities/media_capabilities.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,7 @@ bool IsValidMediaConfiguration(const MediaConfiguration* configuration) {
}

bool IsValidVideoConfiguration(const VideoConfiguration* configuration,
bool is_decode,
bool is_webrtc) {
DCHECK(configuration->hasContentType());
if (!IsValidMimeType(configuration->contentType(), kVideoMimeTypePrefix,
Expand All @@ -286,6 +287,13 @@ bool IsValidVideoConfiguration(const VideoConfiguration* configuration,
if (!IsValidFrameRate(configuration->framerate()))
return false;

// scalabilityMode only valid for WebRTC encode configuration.
if ((!is_webrtc || is_decode) && configuration->hasScalabilityMode())
return false;
// spatialScalability only valid for WebRTC decode configuration.
if ((!is_webrtc || !is_decode) && configuration->hasSpatialScalability())
return false;

return true;
}

Expand Down Expand Up @@ -337,7 +345,8 @@ bool IsValidMediaDecodingConfiguration(
}

if (configuration->hasVideo() &&
!IsValidVideoConfiguration(configuration->video(), is_webrtc)) {
!IsValidVideoConfiguration(configuration->video(), /*is_decode=*/true,
is_webrtc)) {
*message = "The video configuration dictionary is not valid.";
return false;
}
Expand All @@ -363,7 +372,8 @@ bool IsValidMediaEncodingConfiguration(
}

if (configuration->hasVideo() &&
!IsValidVideoConfiguration(configuration->video(), is_webrtc)) {
!IsValidVideoConfiguration(configuration->video(), /*is_decode=*/false,
is_webrtc)) {
*message = "The video configuration dictionary is not valid.";
return false;
}
Expand Down Expand Up @@ -818,18 +828,17 @@ ScriptPromise MediaCapabilities::decodingInfo(
: absl::nullopt;

absl::optional<webrtc::SdpVideoFormat> sdp_video_format;
absl::optional<String> scalability_mode;
bool spatial_scalability = false;
media::VideoCodecProfile codec_profile =
media::VIDEO_CODEC_PROFILE_UNKNOWN;
int video_pixels = 0;
int frames_per_second = 0;
if (config->hasVideo()) {
sdp_video_format =
absl::make_optional(ToSdpVideoFormat(config->video()));
scalability_mode =
config->video()->hasScalabilityMode()
? absl::make_optional(config->video()->scalabilityMode())
: absl::nullopt;
spatial_scalability = config->video()->hasSpatialScalability()
? config->video()->spatialScalability()
: false;

// Additional information needed for lookup in WebrtcVideoPerfHistory.
codec_profile =
Expand All @@ -845,7 +854,7 @@ ScriptPromise MediaCapabilities::decodingInfo(
video_pixels, /*hardware_accelerated=*/false);

handler->DecodingInfo(sdp_audio_format, sdp_video_format,
scalability_mode,
spatial_scalability,
WTF::Bind(&MediaCapabilities::OnWebrtcSupportInfo,
WrapPersistent(this), callback_id,
std::move(features), frames_per_second));
Expand Down
60 changes: 60 additions & 0 deletions third_party/blink/renderer/modules/media_capabilities/media_capabilities_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1250,6 +1250,37 @@ TEST(MediaCapabilitiesTests, WebrtcDecodingUnsupportedVideo) {
EXPECT_FALSE(info->powerEfficient());
}

TEST(MediaCapabilitiesTests, WebrtcDecodingSpatialScalability) {
MediaCapabilitiesTestContext context;
auto* decoding_config = CreateWebrtcDecodingConfig();
auto* video_config = decoding_config->getVideoOr(nullptr);
video_config->setSpatialScalability(false);
const media::mojom::blink::WebrtcPredictionFeatures kFeatures =
CreateWebrtcFeatures(/*is_decode=*/true);

// WebrtcPerfHistoryService should be queried for smoothness. Signal
// smooth=true.
EXPECT_CALL(*context.GetWebrtcPerfHistoryService(), GetPerfInfo(_, _, _))
.WillOnce(WebrtcDbCallback(kFeatures, kFramerate, /*is_smooth=*/true));
MediaCapabilitiesInfo* info = DecodingInfo(decoding_config, &context);
EXPECT_TRUE(info->supported());
EXPECT_TRUE(info->smooth());
EXPECT_FALSE(info->powerEfficient());

// Verify DB call was made.
testing::Mock::VerifyAndClearExpectations(
context.GetWebrtcPerfHistoryService());

// Repeat test with spatialScalability=true.
video_config->setSpatialScalability(true);
EXPECT_CALL(*context.GetWebrtcPerfHistoryService(), GetPerfInfo(_, _, _))
.WillOnce(WebrtcDbCallback(kFeatures, kFramerate, /*is_smooth=*/false));
info = DecodingInfo(decoding_config, &context);
EXPECT_TRUE(info->supported());
EXPECT_FALSE(info->smooth());
EXPECT_FALSE(info->powerEfficient());
}

// WebRTC encodingInfo tests.
TEST(MediaCapabilitiesTests, WebrtcEncodingBasicAudio) {
MediaCapabilitiesTestContext context;
Expand Down Expand Up @@ -1326,4 +1357,33 @@ TEST(MediaCapabilitiesTests, WebrtcEncodingUnsupportedVideo) {
EXPECT_FALSE(info->powerEfficient());
}

TEST(MediaCapabilitiesTests, WebrtcEncodingScalabilityMode) {
MediaCapabilitiesTestContext context;
auto* encoding_config = CreateWebrtcEncodingConfig();
auto* video_config = encoding_config->getVideoOr(nullptr);
video_config->setScalabilityMode("L3T3_KEY");
const media::mojom::blink::WebrtcPredictionFeatures kFeatures =
CreateWebrtcFeatures(/*is_decode=*/false);

// WebrtcPerfHistoryService should be queried for smoothness. Signal
// smooth=true.
EXPECT_CALL(*context.GetWebrtcPerfHistoryService(), GetPerfInfo(_, _, _))
.WillOnce(WebrtcDbCallback(kFeatures, kFramerate, /*is_smooth=*/true));
MediaCapabilitiesInfo* info = EncodingInfo(encoding_config, &context);
EXPECT_TRUE(info->supported());
EXPECT_TRUE(info->smooth());
EXPECT_FALSE(info->powerEfficient());

// Verify DB call was made.
testing::Mock::VerifyAndClearExpectations(
context.GetWebrtcPerfHistoryService());

// Repeat with unsupported mode.
video_config->setScalabilityMode("L3T2_Foo");
info = EncodingInfo(encoding_config, &context);
EXPECT_FALSE(info->supported());
EXPECT_FALSE(info->smooth());
EXPECT_FALSE(info->powerEfficient());
}

} // namespace blink
1 change: 1 addition & 0 deletions third_party/blink/renderer/modules/media_capabilities/video_configuration.idl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,5 @@ dictionary VideoConfiguration {
[RuntimeEnabled=MediaCapabilitiesDynamicRange] ColorGamut colorGamut;
[RuntimeEnabled=MediaCapabilitiesDynamicRange] TransferFunction transferFunction;
[RuntimeEnabled=MediaCapabilitiesWebRtc] DOMString scalabilityMode;
[RuntimeEnabled=MediaCapabilitiesWebRtc] boolean spatialScalability;
};
32 changes: 6 additions & 26 deletions third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
#include "third_party/blink/public/platform/platform.h"
#include "third_party/blink/renderer/platform/peerconnection/audio_codec_factory.h"
#include "third_party/blink/renderer/platform/peerconnection/video_codec_factory.h"
#include "third_party/blink/renderer/platform/webrtc/webrtc_video_utils.h"
#include "third_party/blink/renderer/platform/wtf/text/string_hash.h"
#include "third_party/webrtc/api/audio_codecs/audio_decoder_factory.h"
#include "third_party/webrtc/api/audio_codecs/audio_format.h"
Expand Down Expand Up @@ -54,7 +53,7 @@ WebrtcDecodingInfoHandler::~WebrtcDecodingInfoHandler() = default;
void WebrtcDecodingInfoHandler::DecodingInfo(
const absl::optional<webrtc::SdpAudioFormat> sdp_audio_format,
const absl::optional<webrtc::SdpVideoFormat> sdp_video_format,
const absl::optional<String> video_scalability_mode,
const bool video_spatial_scalability,
OnMediaCapabilitiesDecodingInfoCallback callback) const {
DCHECK(sdp_audio_format || sdp_video_format);

Expand All @@ -75,30 +74,11 @@ void WebrtcDecodingInfoHandler::DecodingInfo(
// Only check video configuration if the audio configuration was supported (or
// not specified).
if (sdp_video_format && supported) {
bool reference_scaling = false;
if (video_scalability_mode) {
absl::optional<int> dependent_spatial_layers =
WebRtcScalabilityModeDependentSpatialLayers(
video_scalability_mode->Utf8());
if (dependent_spatial_layers) {
// Reference scaling must be supported to be able to decode a stream
// with more than one dependent spatial layers.
reference_scaling = *dependent_spatial_layers > 1;
} else {
// We were not able to decode the scalability mode string. Return
// unsupported.
supported = false;
power_efficient = false;
}
}

if (supported) {
webrtc::VideoDecoderFactory::CodecSupport support =
video_decoder_factory_->QueryCodecSupport(*sdp_video_format,
reference_scaling);
supported = support.is_supported;
power_efficient = support.is_power_efficient;
}
webrtc::VideoDecoderFactory::CodecSupport support =
video_decoder_factory_->QueryCodecSupport(*sdp_video_format,
video_spatial_scalability);
supported = support.is_supported;
power_efficient = support.is_power_efficient;
DVLOG(1) << "Video:" << sdp_video_format->name << " supported:" << supported
<< " power_efficient:" << power_efficient;
}
Expand Down
2 changes: 1 addition & 1 deletion third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ class PLATFORM_EXPORT WebrtcDecodingInfoHandler {
void DecodingInfo(
const absl::optional<webrtc::SdpAudioFormat> sdp_audio_format,
const absl::optional<webrtc::SdpVideoFormat> sdp_video_format,
const absl::optional<String> video_scalability_mode,
const bool video_spatial_scalability,
OnMediaCapabilitiesDecodingInfoCallback callback) const;

private:
Expand Down
38 changes: 15 additions & 23 deletions third_party/blink/renderer/platform/peerconnection/webrtc_decoding_info_handler_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ class MockVideoDecoderFactory : public webrtc::VideoDecoderFactory {
(const));
MOCK_METHOD(webrtc::VideoDecoderFactory::CodecSupport,
QueryCodecSupport,
(const webrtc::SdpVideoFormat& format, bool reference_scaling),
(const webrtc::SdpVideoFormat& format, bool spatial_scalability),
(const, override));
};

Expand Down Expand Up @@ -88,17 +88,16 @@ class WebrtcDecodingInfoHandlerTests : public ::testing::Test {
void VerifyDecodingInfo(
const absl::optional<webrtc::SdpAudioFormat> sdp_audio_format,
const absl::optional<webrtc::SdpVideoFormat> sdp_video_format,
const absl::optional<String> video_scalability_mode,
const bool expected_reference_scaling,
const bool video_spatial_scalability,
const CodecSupport support) {
if (sdp_video_format) {
ON_CALL(*mock_video_decoder_factory_, QueryCodecSupport)
.WillByDefault(
testing::Invoke([sdp_video_format, expected_reference_scaling,
testing::Invoke([sdp_video_format, video_spatial_scalability,
support](const webrtc::SdpVideoFormat& format,
bool reference_scaling) {
bool spatial_scalability) {
EXPECT_TRUE(format.IsSameCodec(*sdp_video_format));
EXPECT_EQ(reference_scaling, expected_reference_scaling);
EXPECT_EQ(spatial_scalability, video_spatial_scalability);
return support;
}));
EXPECT_CALL(*mock_video_decoder_factory_, QueryCodecSupport)
Expand All @@ -109,7 +108,7 @@ class WebrtcDecodingInfoHandlerTests : public ::testing::Test {
MediaCapabilitiesDecodingInfoCallback decoding_info_callback;

decoding_info_handler.DecodingInfo(
sdp_audio_format, sdp_video_format, video_scalability_mode,
sdp_audio_format, sdp_video_format, video_spatial_scalability,
base::BindOnce(
&MediaCapabilitiesDecodingInfoCallback::OnWebrtcDecodingInfoSupport,
base::Unretained(&decoding_info_callback)));
Expand All @@ -131,16 +130,14 @@ class WebrtcDecodingInfoHandlerTests : public ::testing::Test {
TEST_F(WebrtcDecodingInfoHandlerTests, BasicAudio) {
VerifyDecodingInfo(
kAudioFormatOpus, /*sdp_video_format=*/absl::nullopt,
/*video_scalability_mode=*/absl::nullopt,
/*expected_reference_scaling=*/false,
/*video_spatial_scalability=*/false,
CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/true});
}

TEST_F(WebrtcDecodingInfoHandlerTests, UnsupportedAudio) {
VerifyDecodingInfo(
kAudioFormatFoo, /*sdp_video_format=*/absl::nullopt,
/*video_scalability_mode=*/absl::nullopt,
/*expected_reference_scaling=*/false,
/*video_spatial_scalability=*/false,
CodecSupport{/*is_supported=*/false, /*is_power_efficient=*/false});
}

Expand All @@ -151,47 +148,42 @@ TEST_F(WebrtcDecodingInfoHandlerTests, UnsupportedAudio) {
TEST_F(WebrtcDecodingInfoHandlerTests, BasicVideo) {
VerifyDecodingInfo(
/*sdp _audio_format=*/absl::nullopt, kVideoFormatVp9,
/*video_scalability_mode=*/absl::nullopt,
/*expected_reference_scaling=*/false,
/*video_spatial_scalability=*/false,
CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/false});
}

TEST_F(WebrtcDecodingInfoHandlerTests, BasicVideoPowerEfficient) {
VerifyDecodingInfo(
/*sdp _audio_format=*/absl::nullopt, kVideoFormatVp9,
/*video_scalability_mode=*/absl::nullopt,
/*expected_reference_scaling=*/false,
/*video_spatial_scalability=*/false,
CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/true});
}

TEST_F(WebrtcDecodingInfoHandlerTests, UnsupportedVideo) {
VerifyDecodingInfo(
/*sdp _audio_format=*/absl::nullopt, kVideoFormatFoo,
/*video_scalability_mode=*/absl::nullopt,
/*expected_reference_scaling=*/false,
/*video_spatial_scalability=*/false,
CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/false});
}

TEST_F(WebrtcDecodingInfoHandlerTests, VideoWithReferenceScaling) {
VerifyDecodingInfo(
/*sdp _audio_format=*/absl::nullopt, kVideoFormatVp9, "L3T3",
/*expected_reference_scaling=*/true,
/*sdp _audio_format=*/absl::nullopt, kVideoFormatVp9,
/*video_spatial_scalability=*/true,
CodecSupport{/*is_supported=*/true, /*is_power_efficient=*/false});
}

TEST_F(WebrtcDecodingInfoHandlerTests, SupportedAudioUnsupportedVideo) {
VerifyDecodingInfo(
kAudioFormatOpus, kVideoFormatFoo,
/*video_scalability_mode=*/absl::nullopt,
/*expected_reference_scaling=*/false,
/*video_spatial_scalability=*/false,
CodecSupport{/*is_supported=*/false, /*is_power_efficient=*/false});
}

TEST_F(WebrtcDecodingInfoHandlerTests, SupportedVideoUnsupportedAudio) {
VerifyDecodingInfo(
kAudioFormatFoo, kVideoFormatVp9,
/*video_scalability_mode=*/absl::nullopt,
/*expected_reference_scaling=*/false,
/*video_spatial_scalability=*/false,
CodecSupport{/*is_supported=*/false, /*is_power_efficient=*/false});
}

Expand Down
45 changes: 0 additions & 45 deletions third_party/blink/renderer/platform/webrtc/webrtc_video_utils.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,32 +9,6 @@
#include "third_party/webrtc/api/video_codecs/vp9_profile.h"

namespace blink {
namespace {
struct ScalabilityModeSpatialLayers {
const char* name;
int spatial_layers;
int dependent_spatial_layers;
};

constexpr ScalabilityModeSpatialLayers kSvcSpatialLayers[] = {
// See https://www.w3.org/TR/webrtc-svc/ for a list of scalability modes.
{"L1T2", 1, 1}, {"L1T3", 1, 1},
{"L2T1", 2, 2}, {"L2T2", 2, 2},
{"L2T3", 2, 2}, {"L2T1h", 2, 2},
{"L2T2h", 2, 2}, {"L2T3h", 2, 2},
{"S2T1", 2, 1}, {"S2T2", 2, 1},
{"S2T3", 2, 1}, {"S2T1h", 2, 1},
{"S2T2h", 2, 1}, {"S2T3h", 2, 1},
{"L3T1", 3, 3}, {"L3T2", 3, 3},
{"L3T3", 3, 3}, {"S3T1", 3, 1},
{"S3T2", 3, 1}, {"S3T3", 3, 1},
{"S3T1h", 3, 1}, {"S3T2h", 3, 1},
{"S3T3h", 3, 1}, {"L2T2_KEY", 2, 2},
{"L2T2_KEY_SHIFT", 2, 2}, {"L2T3_KEY", 2, 2},
{"L2T3_KEY_SHIFT", 2, 2}, {"L3T2_KEY", 3, 3},
{"L3T2_KEY_SHIFT", 3, 3}, {"L3T3_KEY", 3, 3}};

} // namespace

media::VideoRotation WebRtcToMediaVideoRotation(
webrtc::VideoRotation rotation) {
Expand Down Expand Up @@ -267,23 +241,4 @@ media::VideoColorSpace WebRtcToMediaVideoColorSpace(
return media::VideoColorSpace(primaries, transfer, matrix, range);
}

absl::optional<int> WebRtcScalabilityModeSpatialLayers(
const std::string& scalability_mode) {
for (const auto& entry : kSvcSpatialLayers) {
if (entry.name == scalability_mode) {
return entry.spatial_layers;
}
}
return absl::nullopt;
}

absl::optional<int> WebRtcScalabilityModeDependentSpatialLayers(
const std::string& scalability_mode) {
for (const auto& entry : kSvcSpatialLayers) {
if (entry.name == scalability_mode) {
return entry.dependent_spatial_layers;
}
}
return absl::nullopt;
}
} // namespace blink
Loading

0 comments on commit 1a95f2e

Please sign in to comment.