make QuantizeLevels() store the sum of squared error

(instead of MSE).
Useful for directly storing the alpha-PSNR (in another patch)

Change-Id: I4072864f9c53eb4f38366e8025a2816eb14f504e
This commit is contained in:
Pascal Massimino 2012-07-23 14:26:56 -07:00
parent e45a446ad5
commit 7d732f905b
2 changed files with 13 additions and 22 deletions

View File

@ -11,7 +11,6 @@
// Author: Skal (pascal.massimino@gmail.com)
#include <assert.h>
#include <math.h> // for sqrt()
#include "./quant_levels.h"
@ -27,8 +26,8 @@ extern "C" {
// -----------------------------------------------------------------------------
// Quantize levels.
int QuantizeLevels(uint8_t* data, int width, int height,
int num_levels, float* mse) {
int QuantizeLevels(uint8_t* const data, int width, int height,
int num_levels, uint64_t* const sse) {
int freq[NUM_SYMBOLS] = { 0 };
int q_level[NUM_SYMBOLS] = { 0 };
double inv_q_level[NUM_SYMBOLS] = { 0 };
@ -36,6 +35,7 @@ int QuantizeLevels(uint8_t* data, int width, int height,
const size_t data_size = height * width;
int i, num_levels_in, iter;
double last_err = 1.e38, err = 0.;
const double err_threshold = ERROR_THRESHOLD * data_size;
if (data == NULL) {
return 0;
@ -60,10 +60,7 @@ int QuantizeLevels(uint8_t* data, int width, int height,
}
}
if (num_levels_in <= num_levels) {
if (mse) *mse = 0.;
return 1; // nothing to do !
}
if (num_levels_in <= num_levels) goto End; // nothing to do!
// Start with uniformly spread centroids.
for (i = 0; i < num_levels; ++i) {
@ -78,7 +75,6 @@ int QuantizeLevels(uint8_t* data, int width, int height,
// k-Means iterations.
for (iter = 0; iter < MAX_ITER; ++iter) {
double err_count;
double q_sum[NUM_SYMBOLS] = { 0 };
double q_count[NUM_SYMBOLS] = { 0 };
int s, slot = 0;
@ -109,17 +105,14 @@ int QuantizeLevels(uint8_t* data, int width, int height,
// Compute convergence error.
err = 0.;
err_count = 0.;
for (s = min_s; s <= max_s; ++s) {
const double error = s - inv_q_level[q_level[s]];
err += freq[s] * error * error;
err_count += freq[s];
}
if (err_count > 0.) err /= err_count;
// Check for convergence: we stop as soon as the error is no
// longer improving.
if (last_err - err < ERROR_THRESHOLD) break;
if (last_err - err < err_threshold) break;
last_err = err;
}
@ -140,16 +133,14 @@ int QuantizeLevels(uint8_t* data, int width, int height,
data[n] = map[data[n]];
}
}
// Compute final mean squared error if needed.
if (mse != NULL) {
*mse = (float)sqrt(err);
}
End:
// Store sum of squared error if needed.
if (sse != NULL) *sse = (uint64_t)err;
return 1;
}
int DequantizeLevels(uint8_t* data, int width, int height) {
int DequantizeLevels(uint8_t* const data, int width, int height) {
if (data == NULL || width <= 0 || height <= 0) return 0;
// TODO(skal): implement gradient smoothing.
(void)data;

View File

@ -21,16 +21,16 @@ extern "C" {
#endif
// Replace the input 'data' of size 'width'x'height' with 'num-levels'
// quantized values. If not NULL, 'mse' will contain the mean-squared error.
// quantized values. If not NULL, 'sse' will contain the sum of squared error.
// Valid range for 'num_levels' is [2, 256].
// Returns false in case of error (data is NULL, or parameters are invalid).
int QuantizeLevels(uint8_t* data, int width, int height, int num_levels,
float* mse);
int QuantizeLevels(uint8_t* const data, int width, int height, int num_levels,
uint64_t* const sse);
// Apply post-processing to input 'data' of size 'width'x'height' assuming
// that the source was quantized to a reduced number of levels.
// Returns false in case of error (data is NULL, invalid parameters, ...).
int DequantizeLevels(uint8_t* data, int width, int height);
int DequantizeLevels(uint8_t* const data, int width, int height);
#if defined(__cplusplus) || defined(c_plusplus)
} // extern "C"