make QuantizeLevels() store the sum of squared error

(instead of MSE).
Useful for directly storing the alpha-PSNR (in another patch)

Change-Id: I4072864f9c53eb4f38366e8025a2816eb14f504e

src/utils/quant_levels.c

@@ -11,7 +11,6 @@
 // Author: Skal (pascal.massimino@gmail.com)
 
 #include <assert.h>
-#include <math.h>    // for sqrt()
 
 #include "./quant_levels.h"
 
@@ -27,8 +26,8 @@ extern "C" {
 // -----------------------------------------------------------------------------
 // Quantize levels.
 
-int QuantizeLevels(uint8_t* data, int width, int height,
-                   int num_levels, float* mse) {
+int QuantizeLevels(uint8_t* const data, int width, int height,
+                   int num_levels, uint64_t* const sse) {
   int freq[NUM_SYMBOLS] = { 0 };
   int q_level[NUM_SYMBOLS] = { 0 };
   double inv_q_level[NUM_SYMBOLS] = { 0 };
@@ -36,6 +35,7 @@ int QuantizeLevels(uint8_t* data, int width, int height,
   const size_t data_size = height * width;
   int i, num_levels_in, iter;
   double last_err = 1.e38, err = 0.;
+  const double err_threshold = ERROR_THRESHOLD * data_size;
 
   if (data == NULL) {
     return 0;
@@ -60,10 +60,7 @@ int QuantizeLevels(uint8_t* data, int width, int height,
     }
   }
 
-  if (num_levels_in <= num_levels) {
-    if (mse) *mse = 0.;
-    return 1;   // nothing to do !
-  }
+  if (num_levels_in <= num_levels) goto End;  // nothing to do!
 
   // Start with uniformly spread centroids.
   for (i = 0; i < num_levels; ++i) {
@@ -78,7 +75,6 @@ int QuantizeLevels(uint8_t* data, int width, int height,
 
   // k-Means iterations.
   for (iter = 0; iter < MAX_ITER; ++iter) {
-    double err_count;
     double q_sum[NUM_SYMBOLS] = { 0 };
     double q_count[NUM_SYMBOLS] = { 0 };
     int s, slot = 0;
@@ -109,17 +105,14 @@ int QuantizeLevels(uint8_t* data, int width, int height,
 
     // Compute convergence error.
     err = 0.;
-    err_count = 0.;
     for (s = min_s; s <= max_s; ++s) {
       const double error = s - inv_q_level[q_level[s]];
       err += freq[s] * error * error;
-      err_count += freq[s];
     }
-    if (err_count > 0.) err /= err_count;
 
     // Check for convergence: we stop as soon as the error is no
     // longer improving.
-    if (last_err - err < ERROR_THRESHOLD) break;
+    if (last_err - err < err_threshold) break;
     last_err = err;
   }
 
@@ -140,16 +133,14 @@ int QuantizeLevels(uint8_t* data, int width, int height,
       data[n] = map[data[n]];
     }
   }
-
-  // Compute final mean squared error if needed.
-  if (mse != NULL) {
-    *mse = (float)sqrt(err);
-  }
+ End:
+  // Store sum of squared error if needed.
+  if (sse != NULL) *sse = (uint64_t)err;
 
   return 1;
 }
 
-int DequantizeLevels(uint8_t* data, int width, int height) {
+int DequantizeLevels(uint8_t* const data, int width, int height) {
   if (data == NULL || width <= 0 || height <= 0) return 0;
   // TODO(skal): implement gradient smoothing.
   (void)data;

src/utils/quant_levels.h

@@ -21,16 +21,16 @@ extern "C" {
 #endif
 
 // Replace the input 'data' of size 'width'x'height' with 'num-levels'
-// quantized values. If not NULL, 'mse' will contain the mean-squared error.
+// quantized values. If not NULL, 'sse' will contain the sum of squared error.
 // Valid range for 'num_levels' is [2, 256].
 // Returns false in case of error (data is NULL, or parameters are invalid).
-int QuantizeLevels(uint8_t* data, int width, int height, int num_levels,
-                   float* mse);
+int QuantizeLevels(uint8_t* const data, int width, int height, int num_levels,
+                   uint64_t* const sse);
 
 // Apply post-processing to input 'data' of size 'width'x'height' assuming
 // that the source was quantized to a reduced number of levels.
 // Returns false in case of error (data is NULL, invalid parameters, ...).
-int DequantizeLevels(uint8_t* data, int width, int height);
+int DequantizeLevels(uint8_t* const data, int width, int height);
 
 #if defined(__cplusplus) || defined(c_plusplus)
 }    // extern "C"