2013-07-10 02:45:36 +02:00
|
|
|
// libjingle
|
|
|
|
// Copyright 2010 Google Inc.
|
|
|
|
//
|
|
|
|
// Redistribution and use in source and binary forms, with or without
|
|
|
|
// modification, are permitted provided that the following conditions are met:
|
|
|
|
//
|
|
|
|
// 1. Redistributions of source code must retain the above copyright notice,
|
|
|
|
// this list of conditions and the following disclaimer.
|
|
|
|
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
|
|
// this list of conditions and the following disclaimer in the documentation
|
|
|
|
// and/or other materials provided with the distribution.
|
|
|
|
// 3. The name of the author may not be used to endorse or promote products
|
|
|
|
// derived from this software without specific prior written permission.
|
|
|
|
//
|
|
|
|
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
|
|
|
|
// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
|
|
|
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
|
|
|
// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
|
|
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
|
|
|
|
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
|
|
|
// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
|
|
|
// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
|
|
|
|
// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
|
|
|
// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
|
|
|
|
#ifndef TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT
|
|
|
|
#define TALK_MEDIA_BASE_VIDEOADAPTER_H_
|
|
|
|
|
|
|
|
#include "talk/base/common.h" // For ASSERT
|
|
|
|
#include "talk/base/criticalsection.h"
|
|
|
|
#include "talk/base/scoped_ptr.h"
|
|
|
|
#include "talk/base/sigslot.h"
|
|
|
|
#include "talk/media/base/videocommon.h"
|
|
|
|
|
|
|
|
namespace cricket {
|
|
|
|
|
|
|
|
class VideoFrame;
|
|
|
|
|
|
|
|
// VideoAdapter adapts an input video frame to an output frame based on the
|
|
|
|
// specified input and output formats. The adaptation includes dropping frames
|
|
|
|
// to reduce frame rate and scaling frames. VideoAdapter is thread safe.
|
|
|
|
class VideoAdapter {
|
|
|
|
public:
|
|
|
|
VideoAdapter();
|
|
|
|
virtual ~VideoAdapter();
|
|
|
|
|
2013-09-07 00:56:28 +02:00
|
|
|
virtual void SetInputFormat(const VideoFormat& format);
|
2013-07-10 02:45:36 +02:00
|
|
|
void SetOutputFormat(const VideoFormat& format);
|
|
|
|
// Constrain output resolution to this many pixels overall
|
|
|
|
void SetOutputNumPixels(int num_pixels);
|
|
|
|
int GetOutputNumPixels() const;
|
|
|
|
|
|
|
|
const VideoFormat& input_format();
|
2014-02-27 18:52:04 +01:00
|
|
|
// Returns true if the adapter is dropping frames in calls to AdaptFrame.
|
|
|
|
bool drops_all_frames() const;
|
2013-07-10 02:45:36 +02:00
|
|
|
const VideoFormat& output_format();
|
|
|
|
// If the parameter black is true, the adapted frames will be black.
|
|
|
|
void SetBlackOutput(bool black);
|
|
|
|
|
|
|
|
// Adapt the input frame from the input format to the output format. Return
|
|
|
|
// true and set the output frame to NULL if the input frame is dropped. Return
|
|
|
|
// true and set the out frame to output_frame_ if the input frame is adapted
|
|
|
|
// successfully. Return false otherwise.
|
2014-06-04 03:31:14 +02:00
|
|
|
// Note that, if no adaptation is required, |out_frame| will refer directly
|
|
|
|
// in_frame. If a copy is always required, the caller must do an explicit
|
|
|
|
// copy.
|
|
|
|
// If a copy has taken place, |output_frame_| is owned by the VideoAdapter
|
|
|
|
// and will remain usable until the adapter is destroyed or AdaptFrame is
|
|
|
|
// called again.
|
|
|
|
bool AdaptFrame(VideoFrame* in_frame, VideoFrame** out_frame);
|
2013-07-10 02:45:36 +02:00
|
|
|
|
2014-02-22 00:43:24 +01:00
|
|
|
void set_scale_third(bool enable);
|
2013-08-30 23:24:16 +02:00
|
|
|
bool scale_third() const { return scale_third_; }
|
|
|
|
|
2014-06-24 09:24:49 +02:00
|
|
|
int adaptation_changes() const { return adaption_changes_; }
|
|
|
|
|
2013-07-10 02:45:36 +02:00
|
|
|
protected:
|
|
|
|
float FindClosestScale(int width, int height, int target_num_pixels);
|
2013-08-30 23:24:16 +02:00
|
|
|
float FindClosestViewScale(int width, int height, int target_num_pixels);
|
2013-07-10 02:45:36 +02:00
|
|
|
float FindLowerScale(int width, int height, int target_num_pixels);
|
|
|
|
|
|
|
|
private:
|
2013-08-30 23:24:16 +02:00
|
|
|
const float* GetViewScaleFactors() const;
|
|
|
|
float FindScale(const float* scale_factors,
|
|
|
|
const float upbias, int width, int height,
|
|
|
|
int target_num_pixels);
|
2013-07-10 02:45:36 +02:00
|
|
|
bool StretchToOutputFrame(const VideoFrame* in_frame);
|
|
|
|
|
|
|
|
VideoFormat input_format_;
|
|
|
|
VideoFormat output_format_;
|
|
|
|
int output_num_pixels_;
|
2013-08-30 23:24:16 +02:00
|
|
|
bool scale_third_; // True if adapter allows scaling to 1/3 and 2/3.
|
2014-02-07 20:03:26 +01:00
|
|
|
int frames_in_; // Number of input frames.
|
|
|
|
int frames_out_; // Number of output frames.
|
|
|
|
int frames_scaled_; // Number of frames scaled.
|
2013-08-30 23:24:16 +02:00
|
|
|
int adaption_changes_; // Number of changes in scale factor.
|
2014-02-03 17:57:16 +01:00
|
|
|
size_t previous_width_; // Previous adapter output width.
|
|
|
|
size_t previous_height_; // Previous adapter output height.
|
2013-07-10 02:45:36 +02:00
|
|
|
bool black_output_; // Flag to tell if we need to black output_frame_.
|
|
|
|
bool is_black_; // Flag to tell if output_frame_ is currently black.
|
|
|
|
int64 interval_next_frame_;
|
|
|
|
talk_base::scoped_ptr<VideoFrame> output_frame_;
|
|
|
|
// The critical section to protect the above variables.
|
|
|
|
talk_base::CriticalSection critical_section_;
|
|
|
|
|
|
|
|
DISALLOW_COPY_AND_ASSIGN(VideoAdapter);
|
|
|
|
};
|
|
|
|
|
|
|
|
// CoordinatedVideoAdapter adapts the video input to the encoder by coordinating
|
|
|
|
// the format request from the server, the resolution request from the encoder,
|
|
|
|
// and the CPU load.
|
|
|
|
class CoordinatedVideoAdapter
|
|
|
|
: public VideoAdapter, public sigslot::has_slots<> {
|
|
|
|
public:
|
|
|
|
enum AdaptRequest { UPGRADE, KEEP, DOWNGRADE };
|
2013-11-20 22:49:41 +01:00
|
|
|
enum AdaptReasonEnum {
|
2014-02-21 16:51:43 +01:00
|
|
|
ADAPTREASON_NONE = 0,
|
2013-07-10 02:45:36 +02:00
|
|
|
ADAPTREASON_CPU = 1,
|
|
|
|
ADAPTREASON_BANDWIDTH = 2,
|
|
|
|
ADAPTREASON_VIEW = 4
|
|
|
|
};
|
|
|
|
typedef int AdaptReason;
|
|
|
|
|
|
|
|
CoordinatedVideoAdapter();
|
|
|
|
virtual ~CoordinatedVideoAdapter() {}
|
|
|
|
|
2013-09-07 00:56:28 +02:00
|
|
|
virtual void SetInputFormat(const VideoFormat& format);
|
|
|
|
|
2013-07-10 02:45:36 +02:00
|
|
|
// Enable or disable video adaptation due to the change of the CPU load.
|
|
|
|
void set_cpu_adaptation(bool enable) { cpu_adaptation_ = enable; }
|
|
|
|
bool cpu_adaptation() const { return cpu_adaptation_; }
|
2013-07-22 23:07:49 +02:00
|
|
|
// Enable or disable smoothing when doing CPU adaptation. When smoothing is
|
|
|
|
// enabled, system CPU load is tracked using an exponential weighted
|
|
|
|
// average.
|
2014-02-22 00:43:24 +01:00
|
|
|
void set_cpu_smoothing(bool enable);
|
2013-07-22 23:07:49 +02:00
|
|
|
bool cpu_smoothing() const { return cpu_smoothing_; }
|
2013-07-10 02:45:36 +02:00
|
|
|
// Enable or disable video adaptation due to the change of the GD
|
|
|
|
void set_gd_adaptation(bool enable) { gd_adaptation_ = enable; }
|
|
|
|
bool gd_adaptation() const { return gd_adaptation_; }
|
|
|
|
// Enable or disable video adaptation due to the change of the View
|
|
|
|
void set_view_adaptation(bool enable) { view_adaptation_ = enable; }
|
|
|
|
bool view_adaptation() const { return view_adaptation_; }
|
|
|
|
// Enable or disable video adaptation to fast switch View
|
|
|
|
void set_view_switch(bool enable) { view_switch_ = enable; }
|
|
|
|
bool view_switch() const { return view_switch_; }
|
|
|
|
|
|
|
|
CoordinatedVideoAdapter::AdaptReason adapt_reason() const {
|
|
|
|
return adapt_reason_;
|
|
|
|
}
|
|
|
|
|
|
|
|
// When the video is decreased, set the waiting time for CPU adaptation to
|
|
|
|
// decrease video again.
|
2014-02-22 00:43:24 +01:00
|
|
|
void set_cpu_load_min_samples(int cpu_load_min_samples);
|
2014-02-03 17:57:16 +01:00
|
|
|
int cpu_load_min_samples() const { return cpu_load_min_samples_; }
|
2013-07-10 02:45:36 +02:00
|
|
|
// CPU system load high threshold for reducing resolution. e.g. 0.85f
|
2014-02-22 00:43:24 +01:00
|
|
|
void set_high_system_threshold(float high_system_threshold);
|
2013-07-10 02:45:36 +02:00
|
|
|
float high_system_threshold() const { return high_system_threshold_; }
|
|
|
|
// CPU system load low threshold for increasing resolution. e.g. 0.70f
|
2014-02-22 00:43:24 +01:00
|
|
|
void set_low_system_threshold(float low_system_threshold);
|
2013-07-10 02:45:36 +02:00
|
|
|
float low_system_threshold() const { return low_system_threshold_; }
|
|
|
|
// CPU process load threshold for reducing resolution. e.g. 0.10f
|
2014-02-22 00:43:24 +01:00
|
|
|
void set_process_threshold(float process_threshold);
|
2013-07-10 02:45:36 +02:00
|
|
|
float process_threshold() const { return process_threshold_; }
|
|
|
|
|
|
|
|
// Handle the format request from the server via Jingle update message.
|
|
|
|
void OnOutputFormatRequest(const VideoFormat& format);
|
|
|
|
// Handle the resolution request from the encoder due to bandwidth changes.
|
|
|
|
void OnEncoderResolutionRequest(int width, int height, AdaptRequest request);
|
2013-08-30 23:24:16 +02:00
|
|
|
// Handle the resolution request for CPU overuse.
|
|
|
|
void OnCpuResolutionRequest(AdaptRequest request);
|
2013-07-10 02:45:36 +02:00
|
|
|
// Handle the CPU load provided by a CPU monitor.
|
|
|
|
void OnCpuLoadUpdated(int current_cpus, int max_cpus,
|
|
|
|
float process_load, float system_load);
|
|
|
|
|
|
|
|
sigslot::signal0<> SignalCpuAdaptationUnable;
|
|
|
|
|
|
|
|
private:
|
|
|
|
// Adapt to the minimum of the formats the server requests, the CPU wants, and
|
2013-07-22 23:07:49 +02:00
|
|
|
// the encoder wants. Returns true if resolution changed.
|
2013-07-10 02:45:36 +02:00
|
|
|
bool AdaptToMinimumFormat(int* new_width, int* new_height);
|
|
|
|
bool IsMinimumFormat(int pixels);
|
|
|
|
void StepPixelCount(CoordinatedVideoAdapter::AdaptRequest request,
|
|
|
|
int* num_pixels);
|
|
|
|
CoordinatedVideoAdapter::AdaptRequest FindCpuRequest(
|
|
|
|
int current_cpus, int max_cpus,
|
|
|
|
float process_load, float system_load);
|
|
|
|
|
|
|
|
bool cpu_adaptation_; // True if cpu adaptation is enabled.
|
2013-07-22 23:07:49 +02:00
|
|
|
bool cpu_smoothing_; // True if cpu smoothing is enabled (with adaptation).
|
2013-07-10 02:45:36 +02:00
|
|
|
bool gd_adaptation_; // True if gd adaptation is enabled.
|
|
|
|
bool view_adaptation_; // True if view adaptation is enabled.
|
|
|
|
bool view_switch_; // True if view switch is enabled.
|
|
|
|
int cpu_downgrade_count_;
|
2014-02-03 17:57:16 +01:00
|
|
|
int cpu_load_min_samples_;
|
|
|
|
int cpu_load_num_samples_;
|
2013-07-10 02:45:36 +02:00
|
|
|
// cpu system load thresholds relative to max cpus.
|
|
|
|
float high_system_threshold_;
|
|
|
|
float low_system_threshold_;
|
|
|
|
// cpu process load thresholds relative to current cpus.
|
|
|
|
float process_threshold_;
|
|
|
|
// Video formats that the server view requests, the CPU wants, and the encoder
|
|
|
|
// wants respectively. The adapted output format is the minimum of these.
|
|
|
|
int view_desired_num_pixels_;
|
|
|
|
int64 view_desired_interval_;
|
|
|
|
int encoder_desired_num_pixels_;
|
|
|
|
int cpu_desired_num_pixels_;
|
|
|
|
CoordinatedVideoAdapter::AdaptReason adapt_reason_;
|
|
|
|
// The critical section to protect handling requests.
|
|
|
|
talk_base::CriticalSection request_critical_section_;
|
|
|
|
|
2013-07-22 23:07:49 +02:00
|
|
|
// The weighted average of cpu load over time. It's always updated (if cpu
|
|
|
|
// adaptation is on), but only used if cpu_smoothing_ is set.
|
|
|
|
float system_load_average_;
|
|
|
|
|
2013-07-10 02:45:36 +02:00
|
|
|
DISALLOW_COPY_AND_ASSIGN(CoordinatedVideoAdapter);
|
|
|
|
};
|
|
|
|
|
|
|
|
} // namespace cricket
|
|
|
|
|
|
|
|
#endif // TALK_MEDIA_BASE_VIDEOADAPTER_H_ // NOLINT
|