generic-library/webrtc
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Hardware VP8 encoding: Use QP as metric for resize.

Browse Source 
Add vp8 frame header parser to get QP from vp8 bitstream.

BUG= 4273
R=glaznev@webrtc.org, marpan@google.com, pbos@webrtc.org
TBR=stefan@webrtc.org

Review URL: https://webrtc-codereview.appspot.com/49259004

Cr-Commit-Position: refs/heads/master@{#9256}
This commit is contained in:
jackychen
2015-05-21 11:12:02 -07:00
parent 5fdcdf66d0
commit 98d8cf58ee
9 changed files with 305 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
talk/app/webrtc/java/jni/androidmediaencoder_jni.cc
View File

@@ -35,6 +35,7 @@
#include "webrtc/base/thread.h"
#include "webrtc/modules/video_coding/codecs/interface/video_codec_interface.h"
#include "webrtc/modules/video_coding/utility/include/quality_scaler.h"
#include "webrtc/modules/video_coding/utility/include/vp8_header_parser.h"
#include "webrtc/system_wrappers/interface/logcat_trace_context.h"
#include "third_party/libyuv/include/libyuv/convert.h"
#include "third_party/libyuv/include/libyuv/convert_from.h"
@@ -196,8 +197,6 @@ class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
// Global references; must be deleted in Release().
std::vector<jobject> input_buffers_;
scoped_ptr<webrtc::QualityScaler> quality_scaler_;
// Target frame size in bytes.
int target_framesize_;
// Dynamic resolution change, off by default.
bool scale_;
};
@@ -280,6 +279,11 @@ int32_t MediaCodecVideoEncoder::InitEncode(
size_t /* max_payload_size */) {
const int kMinWidth = 320;
const int kMinHeight = 180;
// QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
// (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is
// always = 127. Note that in SW, QP is that of the user-level range [0, 63].
const int kMaxQP = 127;
const int kLowQpThresholdDenominator = 3;
if (codec_settings == NULL) {
ALOGE("NULL VideoCodec instance");
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
@@ -290,14 +294,10 @@ int32_t MediaCodecVideoEncoder::InitEncode(
ALOGD("InitEncode request");
scale_ = false;
quality_scaler_->Init(0);
quality_scaler_->SetMinResolution(kMinWidth, kMinHeight);
quality_scaler_->ReportFramerate(codec_settings->maxFramerate);
if (codec_settings->maxFramerate > 0) {
target_framesize_ = codec_settings->startBitrate * 1000 /
codec_settings->maxFramerate / 8;
} else {
target_framesize_ = 0;
if (codecType_ == kVideoCodecVP8) {
quality_scaler_->Init(kMaxQP / kLowQpThresholdDenominator);
quality_scaler_->SetMinResolution(kMinWidth, kMinHeight);
quality_scaler_->ReportFramerate(codec_settings->maxFramerate);
}
return codec_thread_->Invoke<int32_t>(
Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread,
@@ -337,12 +337,8 @@ int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */,
int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate,
uint32_t frame_rate) {
quality_scaler_->ReportFramerate(frame_rate);
if (frame_rate > 0) {
target_framesize_ = new_bit_rate * 1000 / frame_rate / 8;
} else {
target_framesize_ = 0;
}
if (codecType_ == kVideoCodecVP8)
quality_scaler_->ReportFramerate(frame_rate);
return codec_thread_->Invoke<int32_t>(
Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread,
this,
@@ -716,16 +712,8 @@ bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
last_input_timestamp_ms_ - last_output_timestamp_ms_,
frame_encoding_time_ms);
if (payload_size) {
double framesize_deviation = 0.0;
if (target_framesize_ > 0) {
framesize_deviation =
(double)abs((int)payload_size - target_framesize_) /
target_framesize_;
}
quality_scaler_->ReportNormalizedFrameSizeFluctuation(
framesize_deviation);
}
if (payload_size && codecType_ == kVideoCodecVP8)
quality_scaler_->ReportQP(webrtc::vp8::GetQP(payload));
// Calculate and print encoding statistics - every 3 seconds.
frames_encoded_++;
@@ -872,7 +860,8 @@ int32_t MediaCodecVideoEncoder::NextNaluPosition(
}
void MediaCodecVideoEncoder::OnDroppedFrame() {
quality_scaler_->ReportDroppedFrame();
if (codecType_ == kVideoCodecVP8)
quality_scaler_->ReportDroppedFrame();
}
MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() {

2
webrtc/modules/video_coding/BUILD.gn
View File

@@ -96,7 +96,9 @@ source_set("video_coding_utility") {
"utility/include/frame_dropper.h",
"utility/include/moving_average.h",
"utility/include/quality_scaler.h",
"utility/include/vp8_header_parser.h",
"utility/quality_scaler.cc",
"utility/vp8_header_parser.cc",
]
configs += [ "../..:common_config" ]

4
webrtc/modules/video_coding/codecs/vp8/vp8_impl.cc
View File

@@ -603,7 +603,7 @@ int VP8EncoderImpl::InitEncode(const VideoCodec* inst,
}
rps_.Init();
quality_scaler_.Init(codec_.qpMax);
quality_scaler_.Init(codec_.qpMax / kDefaultLowQpDenominator);
quality_scaler_.ReportFramerate(codec_.maxFramerate);
return InitAndSetControlSettings();
@@ -1035,7 +1035,7 @@ int VP8EncoderImpl::GetEncodedPartitions(
if (encoded_images_[0]._length > 0) {
int qp;
vpx_codec_control(&encoders_[0], VP8E_GET_LAST_QUANTIZER_64, &qp);
quality_scaler_.ReportNormalizedQP(qp);
quality_scaler_.ReportQP(qp);
} else {
quality_scaler_.ReportDroppedFrame();
}

13
webrtc/modules/video_coding/utility/include/quality_scaler.h
View File

@@ -15,6 +15,7 @@
#include "webrtc/modules/video_coding/utility/include/moving_average.h"
namespace webrtc {
const int kDefaultLowQpDenominator = 3;
class QualityScaler {
public:
struct Resolution {
@@ -23,15 +24,10 @@ class QualityScaler {
};
QualityScaler();
void Init(int max_qp);
void Init(int low_qp_threshold);
void SetMinResolution(int min_width, int min_height);
void ReportFramerate(int framerate);
// Report QP for SW encoder, report framesize fluctuation for HW encoder,
// only one of these two functions should be called, framesize fluctuation
// is to be used only if qp isn't available.
void ReportNormalizedQP(int qp);
void ReportNormalizedFrameSizeFluctuation(double framesize_deviation);
void ReportQP(int qp);
void ReportDroppedFrame();
void Reset(int framerate, int bitrate, int width, int height);
Resolution GetScaledResolution(const I420VideoFrame& frame);
@@ -45,10 +41,9 @@ class QualityScaler {
I420VideoFrame scaled_frame_;
size_t num_samples_;
int target_framesize_;
int low_qp_threshold_;
MovingAverage<int> framedrop_percent_;
MovingAverage<double> frame_quality_;
MovingAverage<int> average_qp_;
int downscale_shift_;
int min_width_;

75
webrtc/modules/video_coding/utility/include/vp8_header_parser.h Normal file
View File

@@ -0,0 +1,75 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef WEBRTC_MODULES_VIDEO_CODING_UTILITY_VP8_PARSE_HEADER_H_
#define WEBRTC_MODULES_VIDEO_CODING_UTILITY_VP8_PARSE_HEADER_H_
namespace webrtc {
namespace vp8 {
enum {
MB_FEATURE_TREE_PROBS = 3,
NUM_MB_SEGMENTS = 4,
NUM_REF_LF_DELTAS = 4,
NUM_MODE_LF_DELTAS = 4,
};
typedef struct VP8BitReader VP8BitReader;
struct VP8BitReader {
// Boolean decoder.
uint32_t value_; // Current value.
uint32_t range_; // Current range minus 1. In [127, 254] interval.
int bits_; // Number of valid bits left.
// Read buffer.
const uint8_t* buf_; // Next byte to be read.
const uint8_t* buf_end_; // End of read buffer.
int eof_; // True if input is exhausted.
};
const uint8_t kVP8Log2Range[128] = {
7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4,
3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
0
};
// range = ((range - 1) << kVP8Log2Range[range]) + 1
const uint8_t kVP8NewRange[128] = {
127, 127, 191, 127, 159, 191, 223, 127,
143, 159, 175, 191, 207, 223, 239, 127,
135, 143, 151, 159, 167, 175, 183, 191,
199, 207, 215, 223, 231, 239, 247, 127,
131, 135, 139, 143, 147, 151, 155, 159,
163, 167, 171, 175, 179, 183, 187, 191,
195, 199, 203, 207, 211, 215, 219, 223,
227, 231, 235, 239, 243, 247, 251, 127,
129, 131, 133, 135, 137, 139, 141, 143,
145, 147, 149, 151, 153, 155, 157, 159,
161, 163, 165, 167, 169, 171, 173, 175,
177, 179, 181, 183, 185, 187, 189, 191,
193, 195, 197, 199, 201, 203, 205, 207,
209, 211, 213, 215, 217, 219, 221, 223,
225, 227, 229, 231, 233, 235, 237, 239,
241, 243, 245, 247, 249, 251, 253, 127
};
int GetQP(uint8_t* buf);
} // namespace vp8
} // namespace webrtc
#endif // WEBRTC_MODULES_VIDEO_CODING_UTILITY_VP8_PARSE_HEADER_H_

30
webrtc/modules/video_coding/utility/quality_scaler.cc
View File

@@ -14,17 +14,15 @@ namespace webrtc {
static const int kMinFps = 10;
static const int kMeasureSeconds = 5;
static const int kFramedropPercentThreshold = 60;
static const int kLowQpThresholdDenominator = 3;
static const double kFramesizeFlucThreshold = 0.11;
QualityScaler::QualityScaler()
: num_samples_(0), low_qp_threshold_(-1), downscale_shift_(0),
min_width_(0), min_height_(0) {
}
void QualityScaler::Init(int max_qp) {
void QualityScaler::Init(int low_qp_threshold) {
ClearSamples();
low_qp_threshold_ = max_qp / kLowQpThresholdDenominator;
low_qp_threshold_ = low_qp_threshold;
}
void QualityScaler::SetMinResolution(int min_width, int min_height) {
@@ -32,24 +30,15 @@ void QualityScaler::SetMinResolution(int min_width, int min_height) {
min_height_ = min_height;
}
// TODO(jackychen): target_framesize should be calculated from average bitrate
// in the measured period of time.
// Report framerate(fps) and target_bitrate(kbit/s) to estimate # of samples
// and get target_framesize_.
// Report framerate(fps) to estimate # of samples.
void QualityScaler::ReportFramerate(int framerate) {
num_samples_ = static_cast<size_t>(
kMeasureSeconds * (framerate < kMinFps ? kMinFps : framerate));
}
void QualityScaler::ReportNormalizedQP(int qp) {
void QualityScaler::ReportQP(int qp) {
framedrop_percent_.AddSample(0);
frame_quality_.AddSample(static_cast<double>(qp) / low_qp_threshold_);
}
void QualityScaler::ReportNormalizedFrameSizeFluctuation(
double framesize_deviation) {
framedrop_percent_.AddSample(0);
frame_quality_.AddSample(framesize_deviation / kFramesizeFlucThreshold);
average_qp_.AddSample(qp);
}
void QualityScaler::ReportDroppedFrame() {
@@ -67,13 +56,12 @@ QualityScaler::Resolution QualityScaler::GetScaledResolution(
res.height = frame.height();
// Update scale factor.
int avg_drop;
double avg_quality;
int avg_drop, avg_qp;
if (framedrop_percent_.GetAverage(num_samples_, &avg_drop) &&
avg_drop >= kFramedropPercentThreshold) {
AdjustScale(false);
} else if (frame_quality_.GetAverage(num_samples_, &avg_quality) &&
avg_quality <= 1.0) {
} else if (average_qp_.GetAverage(num_samples_, &avg_qp) &&
avg_qp <= low_qp_threshold_) {
AdjustScale(true);
}
@@ -119,7 +107,7 @@ const I420VideoFrame& QualityScaler::GetScaledFrame(
void QualityScaler::ClearSamples() {
framedrop_percent_.Reset();
frame_quality_.Reset();
average_qp_.Reset();
}
void QualityScaler::AdjustScale(bool up) {

12
webrtc/modules/video_coding/utility/quality_scaler_unittest.cc
View File

@@ -31,7 +31,7 @@ class QualityScalerTest : public ::testing::Test {
QualityScalerTest() {
input_frame_.CreateEmptyFrame(
kWidth, kHeight, kWidth, kHalfWidth, kHalfWidth);
qs_.Init(kMaxQp);
qs_.Init(kMaxQp / kDefaultLowQpDenominator);
qs_.ReportFramerate(kFramerate);
}
@@ -40,7 +40,7 @@ class QualityScalerTest : public ::testing::Test {
for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
switch (scale_direction) {
case kScaleUp:
qs_.ReportNormalizedQP(kLowQp);
qs_.ReportQP(kLowQp);
break;
case kScaleDown:
qs_.ReportDroppedFrame();
@@ -93,7 +93,7 @@ TEST_F(QualityScalerTest, DownscalesAfterContinuousFramedrop) {
TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
for (int i = 0; i < kFramerate * kNumSeconds / 3; ++i) {
qs_.ReportNormalizedQP(kNormalQp);
qs_.ReportQP(kNormalQp);
qs_.ReportDroppedFrame();
qs_.ReportDroppedFrame();
if (qs_.GetScaledResolution(input_frame_).width < input_frame_.width())
@@ -105,7 +105,7 @@ TEST_F(QualityScalerTest, DownscalesAfterTwoThirdsFramedrop) {
TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) {
for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
qs_.ReportNormalizedQP(kNormalQp);
qs_.ReportQP(kNormalQp);
ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width)
<< "Unexpected scale on half framedrop.";
}
@@ -113,7 +113,7 @@ TEST_F(QualityScalerTest, DoesNotDownscaleOnNormalQp) {
TEST_F(QualityScalerTest, DoesNotDownscaleAfterHalfFramedrop) {
for (int i = 0; i < kFramerate * kNumSeconds / 2; ++i) {
qs_.ReportNormalizedQP(kNormalQp);
qs_.ReportQP(kNormalQp);
ASSERT_EQ(input_frame_.width(), qs_.GetScaledResolution(input_frame_).width)
<< "Unexpected scale on half framedrop.";
@@ -153,7 +153,7 @@ void QualityScalerTest::ContinuouslyDownscalesByHalfDimensionsAndBackUp() {
// Verify we don't start upscaling after further low use.
for (int i = 0; i < kFramerate * kNumSeconds; ++i) {
qs_.ReportNormalizedQP(kLowQp);
qs_.ReportQP(kLowQp);
ExpectOriginalFrame();
}
}

4
webrtc/modules/video_coding/utility/video_coding_utility.gyp
View File

@@ -21,8 +21,10 @@
'frame_dropper.cc',
'include/frame_dropper.h',
'include/moving_average.h',
'quality_scaler.cc',
'include/quality_scaler.h',
'include/vp8_header_parser.h',
'quality_scaler.cc',
'vp8_header_parser.cc',
],
},
], # targets

188
webrtc/modules/video_coding/utility/vp8_header_parser.cc Normal file
View File

@@ -0,0 +1,188 @@
/*
* Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stdint.h>
#include <stdio.h>
#include "webrtc/modules/video_coding/utility/include/vp8_header_parser.h"
namespace webrtc {
namespace vp8 {
static uint32_t BSwap32(uint32_t x) {
return (x >> 24) | ((x >> 8) & 0xff00) | ((x << 8) & 0xff0000) | (x << 24);
}
static void VP8LoadFinalBytes(VP8BitReader* const br) {
// Only read 8bits at a time.
if (br->buf_ < br->buf_end_) {
br->bits_ += 8;
br->value_ = static_cast<uint32_t>(*br->buf_++) | (br->value_ << 8);
} else if (!br->eof_) {
br->value_ <<= 8;
br->bits_ += 8;
br->eof_ = 1;
}
}
static void VP8LoadNewBytes(VP8BitReader* const br) {
int BITS = 24;
// Read 'BITS' bits at a time.
if (br->buf_ + sizeof(uint32_t) <= br->buf_end_) {
uint32_t bits;
const uint32_t in_bits = *(const uint32_t*)(br->buf_);
br->buf_ += BITS >> 3;
#if defined(WEBRTC_ARCH_BIG_ENDIAN)
bits = static_cast<uint32_t>(in_bits);
if (BITS != 8 * sizeof(uint32_t))
bits >>= (8 * sizeof(uint32_t) - BITS);
#else
bits = BSwap32(in_bits);
bits >>= 32 - BITS;
#endif
br->value_ = bits | (br->value_ << BITS);
br->bits_ += BITS;
} else {
VP8LoadFinalBytes(br);
}
}
static void VP8InitBitReader(VP8BitReader* const br,
const uint8_t* const start,
const uint8_t* const end) {
br->range_ = 255 - 1;
br->buf_ = start;
br->buf_end_ = end;
br->value_ = 0;
br->bits_ = -8; // To load the very first 8bits.
br->eof_ = 0;
VP8LoadNewBytes(br);
}
// Read a bit with proba 'prob'.
static int VP8GetBit(VP8BitReader* const br, int prob) {
uint8_t range = br->range_;
if (br->bits_ < 0) {
VP8LoadNewBytes(br);
}
const int pos = br->bits_;
const uint8_t split = (range * prob) >> 8;
const uint8_t value = static_cast<uint8_t>(br->value_ >> pos);
int bit;
if (value > split) {
range -= split + 1;
br->value_ -= static_cast<uint32_t>(split + 1) << pos;
bit = 1;
} else {
range = split;
bit = 0;
}
if (range <= static_cast<uint8_t>(0x7e)) {
const int shift = kVP8Log2Range[range];
range = kVP8NewRange[range];
br->bits_ -= shift;
}
br->range_ = range;
return bit;
}
static uint32_t VP8GetValue(VP8BitReader* const br, int bits) {
uint32_t v = 0;
while (bits-- > 0) {
v |= VP8GetBit(br, 0x80) << bits;
}
return v;
}
static uint32_t VP8Get(VP8BitReader* const br) {
return VP8GetValue(br, 1);
}
static int32_t VP8GetSignedValue(VP8BitReader* const br, int bits) {
const int value = VP8GetValue(br, bits);
return VP8Get(br) ? -value : value;
}
static void ParseSegmentHeader(VP8BitReader* br) {
int use_segment = VP8Get(br);
if (use_segment) {
int update_map = VP8Get(br);
if (VP8Get(br)) {
int s;
VP8Get(br);
for (s = 0; s < NUM_MB_SEGMENTS; ++s) {
VP8Get(br) ? VP8GetSignedValue(br, 7) : 0;
}
for (s = 0; s < NUM_MB_SEGMENTS; ++s) {
VP8Get(br) ? VP8GetSignedValue(br, 6) : 0;
}
}
if (update_map) {
int s;
for (s = 0; s < MB_FEATURE_TREE_PROBS; ++s) {
VP8Get(br) ? VP8GetValue(br, 8) : 255;
}
}
}
}
static void ParseFilterHeader(VP8BitReader* br) {
VP8Get(br);
VP8GetValue(br, 6);
VP8GetValue(br, 3);
int use_lf_delta = VP8Get(br);
if (use_lf_delta) {
if (VP8Get(br)) {
int i;
for (i = 0; i < NUM_REF_LF_DELTAS; ++i) {
if (VP8Get(br)) {
VP8GetSignedValue(br, 6);
}
}
for (i = 0; i < NUM_MODE_LF_DELTAS; ++i) {
if (VP8Get(br)) {
VP8GetSignedValue(br, 6);
}
}
}
}
}
int GetQP(uint8_t* buf) {
VP8BitReader br;
const uint32_t bits = buf[0] | (buf[1] << 8) | (buf[2] << 16);
int key_frame = !(bits & 1);
// Size of first partition in bytes.
int partition_length = (bits >> 5);
// Skip past uncompressed header: 10bytes for key, 3bytes for delta frames.
if (key_frame) {
buf += 10;
} else {
buf += 3;
}
VP8InitBitReader(&br, buf, buf + partition_length);
if (key_frame) {
// Color space and pixel type.
VP8Get(&br);
VP8Get(&br);
}
ParseSegmentHeader(&br);
ParseFilterHeader(&br);
// Number of coefficient data partitions.
VP8GetValue(&br, 2);
// Base QP.
const int base_q0 = VP8GetValue(&br, 7);
return base_q0;
}
} // namespace vp8
} // namespace webrtc