vpx/vp8/encoder/encodeintra.c

369 lines
9.8 KiB
C
Raw Normal View History

2010-05-18 17:58:33 +02:00
/*
* Copyright (c) 2010 The WebM project authors. All Rights Reserved.
2010-05-18 17:58:33 +02:00
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
2010-05-18 17:58:33 +02:00
*/
#include "vpx_ports/config.h"
#include "vp8/common/idct.h"
2010-05-18 17:58:33 +02:00
#include "quantize.h"
#include "vp8/common/reconintra.h"
#include "vp8/common/reconintra4x4.h"
2010-05-18 17:58:33 +02:00
#include "encodemb.h"
#include "vp8/common/invtrans.h"
#include "vp8/common/recon.h"
2010-05-18 17:58:33 +02:00
#include "dct.h"
#include "vp8/common/g_common.h"
2010-05-18 17:58:33 +02:00
#include "encodeintra.h"
#ifdef ENC_DEBUG
extern int enc_debug;
#endif
2010-05-18 17:58:33 +02:00
#if CONFIG_RUNTIME_CPU_DETECT
#define IF_RTCD(x) (x)
#else
#define IF_RTCD(x) NULL
#endif
int vp8_encode_intra(VP8_COMP *cpi, MACROBLOCK *x, int use_16x16_pred)
{
int i;
int intra_pred_var = 0;
(void) cpi;
if (use_16x16_pred)
{
x->e_mbd.mode_info_context->mbmi.mode = DC_PRED;
#if CONFIG_COMP_INTRA_PRED
x->e_mbd.mode_info_context->mbmi.second_mode = (MB_PREDICTION_MODE) (DC_PRED - 1);
#endif
x->e_mbd.mode_info_context->mbmi.uv_mode = DC_PRED;
x->e_mbd.mode_info_context->mbmi.ref_frame = INTRA_FRAME;
vp8_encode_intra16x16mby(IF_RTCD(&cpi->rtcd), x);
}
else
{
for (i = 0; i < 16; i++)
{
x->e_mbd.block[i].bmi.as_mode.first = B_DC_PRED;
vp8_encode_intra4x4block(IF_RTCD(&cpi->rtcd), x, i);
}
}
intra_pred_var = VARIANCE_INVOKE(&cpi->rtcd.variance, getmbss)(x->src_diff);
return intra_pred_var;
}
void vp8_encode_intra4x4block(const VP8_ENCODER_RTCD *rtcd,
MACROBLOCK *x, int ib)
2010-05-18 17:58:33 +02:00
{
BLOCKD *b = &x->e_mbd.block[ib];
BLOCK *be = &x->block[ib];
#if CONFIG_COMP_INTRA_PRED
if (b->bmi.as_mode.second == (B_PREDICTION_MODE) (B_DC_PRED - 1))
{
#endif
RECON_INVOKE(&rtcd->common->recon, intra4x4_predict)
(b, b->bmi.as_mode.first, b->predictor);
#if CONFIG_COMP_INTRA_PRED
}
else
{
RECON_INVOKE(&rtcd->common->recon, comp_intra4x4_predict)
(b, b->bmi.as_mode.first, b->bmi.as_mode.second, b->predictor);
}
#endif
2010-05-18 17:58:33 +02:00
ENCODEMB_INVOKE(&rtcd->encodemb, subb)(be, b, 16);
x->vp8_short_fdct4x4(be->src_diff, be->coeff, 32);
x->quantize_b(be, b);
vp8_inverse_transform_b(IF_RTCD(&rtcd->common->idct), b, 32);
RECON_INVOKE(&rtcd->common->recon, recon)(b->predictor, b->diff, *(b->base_dst) + b->dst, b->dst_stride);
}
void vp8_encode_intra4x4mby(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *mb)
{
int i;
MACROBLOCKD *x = &mb->e_mbd;
#if !CONFIG_SUPERBLOCKS
2010-05-18 17:58:33 +02:00
vp8_intra_prediction_down_copy(x);
#endif
2010-05-18 17:58:33 +02:00
for (i = 0; i < 16; i++)
vp8_encode_intra4x4block(rtcd, mb, i);
2010-05-18 17:58:33 +02:00
return;
}
void vp8_encode_intra16x16mby(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x)
{
BLOCK *b = &x->block[0];
2010-05-18 17:58:33 +02:00
Improved coding using 8x8 transform In summary, this commit encompasses a series of changes in attempt to improve the 8x8 transform based coding to help overall compression quality, please refer to the detailed commit history below for what are the rationale underly the series of changes: a. A frame level flag to indicate if 8x8 transform is used at all. b. 8x8 transform is not used for key frames and small image size. c. On inter coded frame, macroblocks using modes B_PRED, SPLIT_MV and I8X8_PRED are forced to using 4x4 transform based coding, the rest uses 8x8 transform based coding. d. Encoder and decoder has the same assumption on the relationship between prediction modes and transform size, therefore no signaling is encoded in bitstream. e. Mode decision process now calculate the rate and distortion scores using their respective transforms. Overall test results: 1. HD set http://www.corp.google.com/~yaowu/no_crawl/t8x8/HD_t8x8_20120206.html (avg psnr: 3.09% glb psnr: 3.22%, ssim: 3.90%) 2. Cif set: http://www.corp.google.com/~yaowu/no_crawl/t8x8/cif_t8x8_20120206.html (avg psnr: -0.03%, glb psnr: -0.02%, ssim: -0.04%) It should be noted here, as 8x8 transform coding itself is disabled for cif size clips, the 0.03% loss is purely from the 1 bit/frame flag overhead on if 8x8 transform is used or not for the frame. ---patch history for future reference--- Patch 1: this commit tries to select transform size based on macroblock prediction mode. If the size of a prediction mode is 16x16, then the macroblock is forced to use 8x8 transform. If the prediction mode is B_PRED, SPLITMV or I8X8_PRED, then the macroblock is forced to use 4x4 transform. Tests on the following HD clips showed mixed results: (all hd clips only used first 100 frames in the test) http://www.corp.google.com/~yaowu/no_crawl/t8x8/hdmodebased8x8.html http://www.corp.google.com/~yaowu/no_crawl/t8x8/hdmodebased8x8_log.html while the results are mixed and overall negative, it is interesting to see 8x8 helped a few of the clips. Patch 2: this patch tries to hard-wire selection of transform size based on prediction modes without using segmentation to signal the transform size. encoder and decoder both takes the same assumption that all macroblocks use 8x8 transform except when prediciton mode is B_PRED, I8X8_PRED or SPLITMV. Test results are as follows: http://www.corp.google.com/~yaowu/no_crawl/t8x8/cifmodebase8x8_0125.html http://www.corp.google.com/~yaowu/no_crawl/t8x8/hdmodebased8x8_0125log.html Interestingly, by removing the overhead or coding the segmentation, the results on this limited HD set have turn positive on average. Patch 3: this patch disabled the usage of 8x8 transform on key frames, and kept the logic from patch 2 for inter frames only. test results on HD set turned decidedly positive with 8x8 transform enabled on inter frame with 16x16 prediction modes: (avg psnr: .81% glb psnr: .82 ssim: .55%) http://www.corp.google.com/~yaowu/no_crawl/t8x8/hdintermode8x8_0125.html results on cif set still negative overall Patch 4: continued from last patch, but now in mode decision process, the rate and distortion estimates are computed based on 8x8 transform results for MBs with modes associated with 8x8 transform. This patch also fixed a problem related to segment based eob coding when 8x8 transform is used. The patch significantly improved the results on HD clips: http://www.corp.google.com/~yaowu/no_crawl/t8x8/hd8x8RDintermode.html (avg psnr: 2.70% glb psnr: 2.76% ssim: 3.34%) results on cif also improved, though they are still negative compared to baseline that uses 4x4 transform only: http://www.corp.google.com/~yaowu/no_crawl/t8x8/cif8x8RDintermode.html (avg psnr: -.78% glb psnr: -.86% ssim: -.19%) Patch 5: This patch does 3 things: a. a bunch of decoder bug fixes, encodings and decodings were verified to have matched recon buffer on a number of encodes on cif size mobile and hd version of _pedestrian. b. the patch further improved the rate distortion calculation of MBS that use 8x8 transform. This provided some further gain on compression. c. the patch also got the experimental work SEG_LVL_EOB to work with 8x8 transformed macroblock, test results indicates it improves the cif set but hurt the HD set slightly. Tests results on HD clips: http://www.corp.google.com/~yaowu/no_crawl/t8x8/HD_t8x8_20120201.html (avg psnr: 3.19% glb psnr: 3.30% ssim: 3.93%) Test results on cif clips: http://www.corp.google.com/~yaowu/no_crawl/t8x8/cif_t8x8_20120201.html (avg psnr: -.47% glb psnr: -.51% ssim: +.28%) Patch 6: Added a frame level flag to indicate if 8x8 transform is allowed at all. temporarily the decision is based on frame size, can be optimized later one. This get the cif results to basically unchanged, with one bit per frame overhead on both cif and hd clips. Patch 8: Rebase and Merge to head by PGW. Fixed some suspect 4s that look like hey should be 64s in regard to segmented EOB. Perhaps #defines would be bette. Bulit and tested without T8x8 enabled and produces unchanged output. Patch 9: Corrected misalligned code/decode of "txfm_mode" bit. Limited testing for correct encode and decode with T8x8 configured on derf clips. Change-Id: I156e1405d25f81579d579dff8ab9af53944ec49c
2012-02-10 01:12:23 +01:00
int tx_type = x->e_mbd.mode_info_context->mbmi.txfm_size;
#if CONFIG_COMP_INTRA_PRED
if (x->e_mbd.mode_info_context->mbmi.second_mode == (MB_PREDICTION_MODE) (DC_PRED - 1))
#endif
RECON_INVOKE(&rtcd->common->recon, build_intra_predictors_mby)(&x->e_mbd);
#if CONFIG_COMP_INTRA_PRED
else
RECON_INVOKE(&rtcd->common->recon, build_comp_intra_predictors_mby)(&x->e_mbd);
#endif
2010-05-18 17:58:33 +02:00
ENCODEMB_INVOKE(&rtcd->encodemb, submby)(x->src_diff, *(b->base_src), x->e_mbd.predictor, b->src_stride);
if( tx_type == TX_8X8 )
vp8_transform_intra_mby_8x8(x);
else
vp8_transform_intra_mby(x);
2010-05-18 17:58:33 +02:00
if(tx_type == TX_8X8)
vp8_quantize_mby_8x8(x);
else
vp8_quantize_mby(x);
2010-05-18 17:58:33 +02:00
if (x->optimize)
{
if( tx_type == TX_8X8 )
vp8_optimize_mby_8x8(x, rtcd);
else
2010-05-18 17:58:33 +02:00
vp8_optimize_mby(x, rtcd);
}
2010-05-18 17:58:33 +02:00
if(tx_type == TX_8X8)
vp8_inverse_transform_mby_8x8(IF_RTCD(&rtcd->common->idct), &x->e_mbd);
else
vp8_inverse_transform_mby(IF_RTCD(&rtcd->common->idct), &x->e_mbd);
#ifdef ENC_DEBUG
if (enc_debug) {
int i;
printf("Intra qcoeff:\n");
printf("%d %d:\n", x->e_mbd.mb_to_left_edge, x->e_mbd.mb_to_top_edge);
for (i =0; i<400; i++) {
printf("%3d ", x->e_mbd.qcoeff[i]);
if (i%16 == 15) printf("\n");
}
printf("Intra dqcoeff:\n");
for (i =0; i<400; i++) {
printf("%3d ", x->e_mbd.dqcoeff[i]);
if (i%16 == 15) printf("\n");
}
printf("Intra diff:\n");
for (i =0; i<400; i++) {
printf("%3d ", x->e_mbd.diff[i]);
if (i%16 == 15) printf("\n");
}
printf("Intra predictor:\n");
for (i =0; i<400; i++) {
printf("%3d ", x->e_mbd.predictor[i]);
if (i%16 == 15) printf("\n");
}
printf("eobs:\n");
for (i=0;i<25;i++)
printf("%d ", x->e_mbd.block[i].eob);
printf("\n");
}
#endif
2010-05-18 17:58:33 +02:00
RECON_INVOKE(&rtcd->common->recon, recon_mby)
(IF_RTCD(&rtcd->common->recon), &x->e_mbd);
2010-05-18 17:58:33 +02:00
}
void vp8_encode_intra16x16mbuv(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x)
{
Improved coding using 8x8 transform In summary, this commit encompasses a series of changes in attempt to improve the 8x8 transform based coding to help overall compression quality, please refer to the detailed commit history below for what are the rationale underly the series of changes: a. A frame level flag to indicate if 8x8 transform is used at all. b. 8x8 transform is not used for key frames and small image size. c. On inter coded frame, macroblocks using modes B_PRED, SPLIT_MV and I8X8_PRED are forced to using 4x4 transform based coding, the rest uses 8x8 transform based coding. d. Encoder and decoder has the same assumption on the relationship between prediction modes and transform size, therefore no signaling is encoded in bitstream. e. Mode decision process now calculate the rate and distortion scores using their respective transforms. Overall test results: 1. HD set http://www.corp.google.com/~yaowu/no_crawl/t8x8/HD_t8x8_20120206.html (avg psnr: 3.09% glb psnr: 3.22%, ssim: 3.90%) 2. Cif set: http://www.corp.google.com/~yaowu/no_crawl/t8x8/cif_t8x8_20120206.html (avg psnr: -0.03%, glb psnr: -0.02%, ssim: -0.04%) It should be noted here, as 8x8 transform coding itself is disabled for cif size clips, the 0.03% loss is purely from the 1 bit/frame flag overhead on if 8x8 transform is used or not for the frame. ---patch history for future reference--- Patch 1: this commit tries to select transform size based on macroblock prediction mode. If the size of a prediction mode is 16x16, then the macroblock is forced to use 8x8 transform. If the prediction mode is B_PRED, SPLITMV or I8X8_PRED, then the macroblock is forced to use 4x4 transform. Tests on the following HD clips showed mixed results: (all hd clips only used first 100 frames in the test) http://www.corp.google.com/~yaowu/no_crawl/t8x8/hdmodebased8x8.html http://www.corp.google.com/~yaowu/no_crawl/t8x8/hdmodebased8x8_log.html while the results are mixed and overall negative, it is interesting to see 8x8 helped a few of the clips. Patch 2: this patch tries to hard-wire selection of transform size based on prediction modes without using segmentation to signal the transform size. encoder and decoder both takes the same assumption that all macroblocks use 8x8 transform except when prediciton mode is B_PRED, I8X8_PRED or SPLITMV. Test results are as follows: http://www.corp.google.com/~yaowu/no_crawl/t8x8/cifmodebase8x8_0125.html http://www.corp.google.com/~yaowu/no_crawl/t8x8/hdmodebased8x8_0125log.html Interestingly, by removing the overhead or coding the segmentation, the results on this limited HD set have turn positive on average. Patch 3: this patch disabled the usage of 8x8 transform on key frames, and kept the logic from patch 2 for inter frames only. test results on HD set turned decidedly positive with 8x8 transform enabled on inter frame with 16x16 prediction modes: (avg psnr: .81% glb psnr: .82 ssim: .55%) http://www.corp.google.com/~yaowu/no_crawl/t8x8/hdintermode8x8_0125.html results on cif set still negative overall Patch 4: continued from last patch, but now in mode decision process, the rate and distortion estimates are computed based on 8x8 transform results for MBs with modes associated with 8x8 transform. This patch also fixed a problem related to segment based eob coding when 8x8 transform is used. The patch significantly improved the results on HD clips: http://www.corp.google.com/~yaowu/no_crawl/t8x8/hd8x8RDintermode.html (avg psnr: 2.70% glb psnr: 2.76% ssim: 3.34%) results on cif also improved, though they are still negative compared to baseline that uses 4x4 transform only: http://www.corp.google.com/~yaowu/no_crawl/t8x8/cif8x8RDintermode.html (avg psnr: -.78% glb psnr: -.86% ssim: -.19%) Patch 5: This patch does 3 things: a. a bunch of decoder bug fixes, encodings and decodings were verified to have matched recon buffer on a number of encodes on cif size mobile and hd version of _pedestrian. b. the patch further improved the rate distortion calculation of MBS that use 8x8 transform. This provided some further gain on compression. c. the patch also got the experimental work SEG_LVL_EOB to work with 8x8 transformed macroblock, test results indicates it improves the cif set but hurt the HD set slightly. Tests results on HD clips: http://www.corp.google.com/~yaowu/no_crawl/t8x8/HD_t8x8_20120201.html (avg psnr: 3.19% glb psnr: 3.30% ssim: 3.93%) Test results on cif clips: http://www.corp.google.com/~yaowu/no_crawl/t8x8/cif_t8x8_20120201.html (avg psnr: -.47% glb psnr: -.51% ssim: +.28%) Patch 6: Added a frame level flag to indicate if 8x8 transform is allowed at all. temporarily the decision is based on frame size, can be optimized later one. This get the cif results to basically unchanged, with one bit per frame overhead on both cif and hd clips. Patch 8: Rebase and Merge to head by PGW. Fixed some suspect 4s that look like hey should be 64s in regard to segmented EOB. Perhaps #defines would be bette. Bulit and tested without T8x8 enabled and produces unchanged output. Patch 9: Corrected misalligned code/decode of "txfm_mode" bit. Limited testing for correct encode and decode with T8x8 configured on derf clips. Change-Id: I156e1405d25f81579d579dff8ab9af53944ec49c
2012-02-10 01:12:23 +01:00
int tx_type = x->e_mbd.mode_info_context->mbmi.txfm_size;
#if CONFIG_COMP_INTRA_PRED
if (x->e_mbd.mode_info_context->mbmi.second_uv_mode == (MB_PREDICTION_MODE) (DC_PRED - 1))
{
#endif
RECON_INVOKE(&rtcd->common->recon, build_intra_predictors_mbuv)(&x->e_mbd);
#if CONFIG_COMP_INTRA_PRED
}
else
{
RECON_INVOKE(&rtcd->common->recon, build_comp_intra_predictors_mbuv)(&x->e_mbd);
}
#endif
2010-05-18 17:58:33 +02:00
ENCODEMB_INVOKE(&rtcd->encodemb, submbuv)(x->src_diff, x->src.u_buffer, x->src.v_buffer, x->e_mbd.predictor, x->src.uv_stride);
if(tx_type == TX_8X8)
vp8_transform_mbuv_8x8(x);
else
vp8_transform_mbuv(x);
2010-05-18 17:58:33 +02:00
if(tx_type == TX_8X8)
vp8_quantize_mbuv_8x8(x);
else
vp8_quantize_mbuv(x);
#ifdef ENC_DEBUG
if (enc_debug) {
int i;
printf("vp8_encode_intra16x16mbuv\n");
printf("%d %d:\n", x->e_mbd.mb_to_left_edge, x->e_mbd.mb_to_top_edge);
printf("qcoeff:\n");
for (i =0; i<400; i++) {
printf("%3d ", x->e_mbd.qcoeff[i]);
if (i%16 == 15) printf("\n");
}
printf("dqcoeff:\n");
for (i =0; i<400; i++) {
printf("%3d ", x->e_mbd.dqcoeff[i]);
if (i%16 == 15) printf("\n");
}
printf("diff:\n");
for (i =0; i<400; i++) {
printf("%3d ", x->e_mbd.diff[i]);
if (i%16 == 15) printf("\n");
}
printf("predictor:\n");
for (i =0; i<400; i++) {
printf("%3d ", x->e_mbd.predictor[i]);
if (i%16 == 15) printf("\n");
}
printf("eobs:\n");
for (i=0;i<25;i++)
printf("%d ", x->e_mbd.block[i].eob);
printf("\n");
}
#endif
if (x->optimize)
{
if(tx_type == TX_8X8)
vp8_optimize_mbuv_8x8(x, rtcd);
else
2010-05-18 17:58:33 +02:00
vp8_optimize_mbuv(x, rtcd);
}
2010-05-18 17:58:33 +02:00
if(tx_type == TX_8X8)
vp8_inverse_transform_mbuv_8x8(IF_RTCD(&rtcd->common->idct), &x->e_mbd);
else
vp8_inverse_transform_mbuv(IF_RTCD(&rtcd->common->idct), &x->e_mbd);
2010-05-18 17:58:33 +02:00
vp8_recon_intra_mbuv(IF_RTCD(&rtcd->common->recon), &x->e_mbd);
}
void vp8_encode_intra8x8(const VP8_ENCODER_RTCD *rtcd,
MACROBLOCK *x, int ib)
{
BLOCKD *b = &x->e_mbd.block[ib];
BLOCK *be = &x->block[ib];
const int iblock[4]={0,1,4,5};
int i;
#if CONFIG_COMP_INTRA_PRED
if (b->bmi.as_mode.second == (MB_PREDICTION_MODE) (DC_PRED - 1))
{
#endif
RECON_INVOKE(&rtcd->common->recon, intra8x8_predict)
(b, b->bmi.as_mode.first, b->predictor);
#if CONFIG_COMP_INTRA_PRED
}
else
{
RECON_INVOKE(&rtcd->common->recon, comp_intra8x8_predict)
(b, b->bmi.as_mode.first, b->bmi.as_mode.second, b->predictor);
}
#endif
for(i=0;i<4;i++)
{
b = &x->e_mbd.block[ib + iblock[i]];
be = &x->block[ib + iblock[i]];
ENCODEMB_INVOKE(&rtcd->encodemb, subb)(be, b, 16);
x->vp8_short_fdct4x4(be->src_diff, be->coeff, 32);
x->quantize_b(be, b);
vp8_inverse_transform_b(IF_RTCD(&rtcd->common->idct), b, 32);
RECON_INVOKE(&rtcd->common->recon, recon)(b->predictor,
b->diff, *(b->base_dst) + b->dst, b->dst_stride);
}
}
extern const int vp8_i8x8_block[4];
void vp8_encode_intra8x8mby(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x)
{
int i, ib;
for(i=0;i<4;i++)
{
ib = vp8_i8x8_block[i];
vp8_encode_intra8x8(rtcd, x, ib);
}
}
void vp8_encode_intra_uv4x4(const VP8_ENCODER_RTCD *rtcd,
MACROBLOCK *x, int ib,
int mode, int second)
{
BLOCKD *b = &x->e_mbd.block[ib];
BLOCK *be = &x->block[ib];
#if CONFIG_COMP_INTRA_PRED
if (second == -1)
{
#endif
RECON_INVOKE(&rtcd->common->recon, intra_uv4x4_predict)
(b, mode, b->predictor);
#if CONFIG_COMP_INTRA_PRED
}
else
{
RECON_INVOKE(&rtcd->common->recon, comp_intra_uv4x4_predict)
(b, mode, second, b->predictor);
}
#endif
ENCODEMB_INVOKE(&rtcd->encodemb, subb)(be, b, 8);
x->vp8_short_fdct4x4(be->src_diff, be->coeff, 16);
x->quantize_b(be, b);
vp8_inverse_transform_b(IF_RTCD(&rtcd->common->idct), b, 16);
RECON_INVOKE(&rtcd->common->recon, recon_uv)(b->predictor,
b->diff, *(b->base_dst) + b->dst, b->dst_stride);
}
void vp8_encode_intra8x8mbuv(const VP8_ENCODER_RTCD *rtcd, MACROBLOCK *x)
{
int i, ib, mode, second;
BLOCKD *b;
for(i=0;i<4;i++)
{
ib = vp8_i8x8_block[i];
b = &x->e_mbd.block[ib];
mode = b->bmi.as_mode.first;
#if CONFIG_COMP_INTRA_PRED
second = b->bmi.as_mode.second;
#else
second = -1;
#endif
/*u */
vp8_encode_intra_uv4x4(rtcd, x, i+16, mode, second);
/*v */
vp8_encode_intra_uv4x4(rtcd, x, i+20, mode, second);
}
}