Change how alignment requirements are processed

Software fallback wrapper now reports least common multiple of requirements
for two encoders.

SimulcastEncoderAdapter queries actual encoder before InitEncode call
and requests alignment for all layers if simulcast is not supported by
any of the encoders.

Bug: chromium:1084702
Change-Id: Iaed8190737125d447036b6c664b863be72556a5d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/225881
Reviewed-by: Niels Moller <[email protected]>
Commit-Queue: Ilya Nikolaevskiy <[email protected]>
Cr-Commit-Position: refs/heads/master@{#34466}

Author: Ilya Nikolaevskiy

api/video_codecs/video_encoder_software_fallback_wrapper.cc

@@ -25,6 +25,7 @@
 #include "api/video/video_frame.h"
 #include "api/video_codecs/video_codec.h"
 #include "api/video_codecs/video_encoder.h"
+#include "media/base/video_common.h"
 #include "modules/video_coding/include/video_error_codes.h"
 #include "modules/video_coding/utility/simulcast_utility.h"
 #include "rtc_base/checks.h"
@@ -417,6 +418,13 @@ VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo()
   EncoderInfo info =
       IsFallbackActive() ? fallback_encoder_info : default_encoder_info;
 
+  info.requested_resolution_alignment = cricket::LeastCommonMultiple(
+      fallback_encoder_info.requested_resolution_alignment,
+      default_encoder_info.requested_resolution_alignment);
+  info.apply_alignment_to_all_simulcast_layers =
+      fallback_encoder_info.apply_alignment_to_all_simulcast_layers ||
+      default_encoder_info.apply_alignment_to_all_simulcast_layers;
+
   if (fallback_params_.has_value()) {
     const auto settings = (encoder_state_ == EncoderState::kForcedFallback)
                               ? fallback_encoder_info.scaling_settings

media/engine/simulcast_encoder_adapter.cc

@@ -149,9 +149,13 @@ namespace webrtc {
 
 SimulcastEncoderAdapter::EncoderContext::EncoderContext(
     std::unique_ptr<VideoEncoder> encoder,
-    bool prefer_temporal_support)
+    bool prefer_temporal_support,
+    VideoEncoder::EncoderInfo primary_info,
+    VideoEncoder::EncoderInfo fallback_info)
     : encoder_(std::move(encoder)),
-      prefer_temporal_support_(prefer_temporal_support) {}
+      prefer_temporal_support_(prefer_temporal_support),
+      primary_info_(std::move(primary_info)),
+      fallback_info_(std::move(fallback_info)) {}
 
 void SimulcastEncoderAdapter::EncoderContext::Release() {
   if (encoder_) {
@@ -690,7 +694,7 @@ void SimulcastEncoderAdapter::DestroyStoredEncoders() {
 
 std::unique_ptr<SimulcastEncoderAdapter::EncoderContext>
 SimulcastEncoderAdapter::FetchOrCreateEncoderContext(
-    bool is_lowest_quality_stream) {
+    bool is_lowest_quality_stream) const {
   bool prefer_temporal_support = fallback_encoder_factory_ != nullptr &&
                                  is_lowest_quality_stream &&
                                  prefer_temporal_support_on_base_layer_;
@@ -712,14 +716,20 @@ SimulcastEncoderAdapter::FetchOrCreateEncoderContext(
   } else {
     std::unique_ptr<VideoEncoder> encoder =
         primary_encoder_factory_->CreateVideoEncoder(video_format_);
+    VideoEncoder::EncoderInfo primary_info = encoder->GetEncoderInfo();
+    VideoEncoder::EncoderInfo fallback_info = primary_info;
     if (fallback_encoder_factory_ != nullptr) {
+      std::unique_ptr<VideoEncoder> fallback_encoder =
+          fallback_encoder_factory_->CreateVideoEncoder(video_format_);
+      fallback_info = fallback_encoder->GetEncoderInfo();
       encoder = CreateVideoEncoderSoftwareFallbackWrapper(
-          fallback_encoder_factory_->CreateVideoEncoder(video_format_),
-          std::move(encoder), prefer_temporal_support);
+          std::move(fallback_encoder), std::move(encoder),
+          prefer_temporal_support);
     }
 
     encoder_context = std::make_unique<SimulcastEncoderAdapter::EncoderContext>(
-        std::move(encoder), prefer_temporal_support);
+        std::move(encoder), prefer_temporal_support, primary_info,
+        fallback_info);
   }
 
   encoder_context->encoder().RegisterEncodeCompleteCallback(
@@ -789,9 +799,11 @@ webrtc::VideoCodec SimulcastEncoderAdapter::MakeStreamCodec(
 void SimulcastEncoderAdapter::OverrideFromFieldTrial(
     VideoEncoder::EncoderInfo* info) const {
   if (encoder_info_override_.requested_resolution_alignment()) {
-    info->requested_resolution_alignment =
-        *encoder_info_override_.requested_resolution_alignment();
+    info->requested_resolution_alignment = cricket::LeastCommonMultiple(
+        info->requested_resolution_alignment,
+        *encoder_info_override_.requested_resolution_alignment());
     info->apply_alignment_to_all_simulcast_layers =
+        info->apply_alignment_to_all_simulcast_layers ||
         encoder_info_override_.apply_alignment_to_all_simulcast_layers();
   }
   if (!encoder_info_override_.resolution_bitrate_limits().empty()) {
@@ -815,7 +827,34 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
   encoder_info.apply_alignment_to_all_simulcast_layers = false;
   encoder_info.supports_native_handle = true;
   encoder_info.scaling_settings.thresholds = absl::nullopt;
+
   if (stream_contexts_.empty()) {
+    // GetEncoderInfo queried before InitEncode. Only alignment info is needed
+    // to be filled.
+    // Create one encoder and query it.
+
+    std::unique_ptr<SimulcastEncoderAdapter::EncoderContext> encoder_context =
+        FetchOrCreateEncoderContext(true);
+
+    const VideoEncoder::EncoderInfo& primary_info =
+        encoder_context->PrimaryInfo();
+    const VideoEncoder::EncoderInfo& fallback_info =
+        encoder_context->FallbackInfo();
+
+    encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple(
+        primary_info.requested_resolution_alignment,
+        fallback_info.requested_resolution_alignment);
+
+    encoder_info.apply_alignment_to_all_simulcast_layers =
+        primary_info.apply_alignment_to_all_simulcast_layers ||
+        fallback_info.apply_alignment_to_all_simulcast_layers;
+
+    if (!primary_info.supports_simulcast || !fallback_info.supports_simulcast) {
+      encoder_info.apply_alignment_to_all_simulcast_layers = true;
+    }
+
+    cached_encoder_contexts_.emplace_back(std::move(encoder_context));
+
     OverrideFromFieldTrial(&encoder_info);
     return encoder_info;
   }
@@ -825,7 +864,6 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
   for (size_t i = 0; i < stream_contexts_.size(); ++i) {
     VideoEncoder::EncoderInfo encoder_impl_info =
         stream_contexts_[i].encoder().GetEncoderInfo();
-
     if (i == 0) {
       // Encoder name indicates names of all sub-encoders.
       encoder_info.implementation_name += " (";
@@ -864,7 +902,12 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
     encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple(
         encoder_info.requested_resolution_alignment,
         encoder_impl_info.requested_resolution_alignment);
-    if (encoder_impl_info.apply_alignment_to_all_simulcast_layers) {
+    // request alignment on all layers if any of the encoders may need it, or
+    // if any non-top layer encoder requests a non-trivial alignment.
+    if (encoder_impl_info.apply_alignment_to_all_simulcast_layers ||
+        (encoder_impl_info.requested_resolution_alignment > 1 &&
+         (codec_.simulcastStream[i].height < codec_.height ||
+          codec_.simulcastStream[i].width < codec_.width))) {
       encoder_info.apply_alignment_to_all_simulcast_layers = true;
     }
   }

media/engine/simulcast_encoder_adapter.h

@@ -71,16 +71,24 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
   class EncoderContext {
    public:
     EncoderContext(std::unique_ptr<VideoEncoder> encoder,
-                   bool prefer_temporal_support);
+                   bool prefer_temporal_support,
+                   VideoEncoder::EncoderInfo primary_info,
+                   VideoEncoder::EncoderInfo fallback_info);
     EncoderContext& operator=(EncoderContext&&) = delete;
 
     VideoEncoder& encoder() { return *encoder_; }
     bool prefer_temporal_support() { return prefer_temporal_support_; }
     void Release();
 
+    const VideoEncoder::EncoderInfo& PrimaryInfo() { return primary_info_; }
+
+    const VideoEncoder::EncoderInfo& FallbackInfo() { return fallback_info_; }
+
    private:
     std::unique_ptr<VideoEncoder> encoder_;
     bool prefer_temporal_support_;
+    const VideoEncoder::EncoderInfo primary_info_;
+    const VideoEncoder::EncoderInfo fallback_info_;
   };
 
   class StreamContext : public EncodedImageCallback {
@@ -138,8 +146,11 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
 
   void DestroyStoredEncoders();
 
+  // This method creates encoder. May reuse previously created encoders from
+  // |cached_encoder_contexts_|. It's const because it's used from
+  // const GetEncoderInfo().
   std::unique_ptr<EncoderContext> FetchOrCreateEncoderContext(
-      bool is_lowest_quality_stream);
+      bool is_lowest_quality_stream) const;
 
   webrtc::VideoCodec MakeStreamCodec(const webrtc::VideoCodec& codec,
                                      int stream_idx,
@@ -169,9 +180,11 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
   // Used for checking the single-threaded access of the encoder interface.
   RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_;
 
-  // Store encoders in between calls to Release and InitEncode, so they don't
-  // have to be recreated. Remaining encoders are destroyed by the destructor.
-  std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_;
+  // Store previously created and released encoders , so they don't have to be
+  // recreated. Remaining encoders are destroyed by the destructor.
+  // Marked as |mutable| becuase we may need to temporarily create encoder in
+  // GetEncoderInfo(), which is const.
+  mutable std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_;
 
   const absl::optional<unsigned int> experimental_boosted_screenshare_qp_;
   const bool boost_base_layer_quality_;

media/engine/simulcast_encoder_adapter_unittest.cc

@@ -900,8 +900,6 @@ TEST_F(TestSimulcastEncoderAdapterFake, SetRatesUnderMinBitrate) {
 }
 
 TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) {
-  EXPECT_EQ("SimulcastEncoderAdapter",
-            adapter_->GetEncoderInfo().implementation_name);
   SimulcastTestFixtureImpl::DefaultSettings(
       &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
       kVideoCodecVP8);
@@ -910,6 +908,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) {
   encoder_names.push_back("codec2");
   encoder_names.push_back("codec3");
   helper_->factory()->SetEncoderNames(encoder_names);
+  EXPECT_EQ("SimulcastEncoderAdapter",
+            adapter_->GetEncoderInfo().implementation_name);
   EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
   EXPECT_EQ("SimulcastEncoderAdapter (codec1, codec2, codec3)",
             adapter_->GetEncoderInfo().implementation_name);