Merge pull request #3342 from alalek:perf_stability_check

modules/ts/include/opencv2/ts/ts_perf.hpp

@@ -419,9 +419,11 @@ private:
     static int64 timeLimitDefault;
     static unsigned int iterationsLimitDefault;
 
+    unsigned int minIters;
     unsigned int nIters;
     unsigned int currentIter;
     unsigned int runsPerIteration;
+    unsigned int perfValidationStage;
 
     performance_metrics metrics;
     void validateMetrics();

modules/ts/src/ts_perf.cpp

@@ -1,5 +1,16 @@
 #include "precomp.hpp"
 
+#include <map>
+#include <iostream>
+#include <fstream>
+
+#if defined WIN32 || defined _WIN32 || defined WIN64 || defined _WIN64
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+#endif
+
 #ifdef HAVE_CUDA
 #include "opencv2/core/cuda.hpp"
 #endif
@@ -35,11 +46,11 @@ static bool         param_verify_sanity;
 static bool         param_collect_impl;
 #endif
 extern bool         test_ipp_check;
+
 #ifdef HAVE_CUDA
 static int          param_cuda_device;
 #endif
 
-
 #ifdef ANDROID
 static int          param_affinity_mask;
 static bool         log_power_checkpoints;
@@ -59,6 +70,8 @@ static void setCurrentThreadAffinityMask(int mask)
 }
 #endif
 
+static double perf_stability_criteria = 0.03; // 3%
+
 namespace {
 
 class PerfEnvironment: public ::testing::Environment
@@ -635,6 +648,82 @@ void performance_metrics::clear()
     terminationReason = TERM_UNKNOWN;
 }
 
+/*****************************************************************************************\
+*                                   Performance validation results
+\*****************************************************************************************/
+
+static bool perf_validation_enabled = false;
+
+static std::string perf_validation_results_directory;
+static std::map<std::string, float> perf_validation_results;
+static std::string perf_validation_results_outfile;
+
+static double perf_validation_criteria = 0.03; // 3 %
+static double perf_validation_time_threshold_ms = 0.1;
+static int perf_validation_idle_delay_ms = 3000; // 3 sec
+
+static void loadPerfValidationResults(const std::string& fileName)
+{
+    perf_validation_results.clear();
+    std::ifstream infile(fileName.c_str());
+    while (!infile.eof())
+    {
+        std::string name;
+        float value = 0;
+        if (!(infile >> value))
+        {
+            if (infile.eof())
+                break; // it is OK
+            std::cout << "ERROR: Can't load performance validation results from " << fileName << "!" << std::endl;
+            return;
+        }
+        infile.ignore(1);
+        if (!(std::getline(infile, name)))
+        {
+            std::cout << "ERROR: Can't load performance validation results from " << fileName << "!" << std::endl;
+            return;
+        }
+        if (!name.empty() && name[name.size() - 1] == '\r') // CRLF processing on Linux
+            name.resize(name.size() - 1);
+        perf_validation_results[name] = value;
+    }
+    std::cout << "Performance validation results loaded from " << fileName << " (" << perf_validation_results.size() << " entries)" << std::endl;
+}
+
+static void savePerfValidationResult(const std::string& name, float value)
+{
+    perf_validation_results[name] = value;
+}
+
+static void savePerfValidationResults()
+{
+    if (!perf_validation_results_outfile.empty())
+    {
+        std::ofstream outfile((perf_validation_results_directory + perf_validation_results_outfile).c_str());
+        std::map<std::string, float>::const_iterator i;
+        for (i = perf_validation_results.begin(); i != perf_validation_results.end(); ++i)
+        {
+            outfile << i->second << ';';
+            outfile << i->first << std::endl;
+        }
+        outfile.close();
+        std::cout << "Performance validation results saved (" << perf_validation_results.size() << " entries)" << std::endl;
+    }
+}
+
+class PerfValidationEnvironment : public ::testing::Environment
+{
+public:
+    virtual ~PerfValidationEnvironment() {}
+    virtual void SetUp() {}
+
+    virtual void TearDown()
+    {
+        savePerfValidationResults();
+    }
+};
+
+
 
 /*****************************************************************************************\
 *                                   ::perf::TestBase
@@ -666,6 +755,8 @@ void TestBase::Init(const std::vector<std::string> & availableImpls,
         "{   perf_list_impls             |false    |list available implementation variants and exit}"
         "{   perf_run_cpu                |false    |deprecated, equivalent to --perf_impl=plain}"
         "{   perf_strategy               |default  |specifies performance measuring strategy: default, base or simple (weak restrictions)}"
+        "{   perf_read_validation_results |        |specifies file name with performance results from previous run}"
+        "{   perf_write_validation_results |       |specifies file name to write performance validation results}"
 #ifdef ANDROID
         "{   perf_time_limit             |6.0      |default time limit for a single test (in seconds)}"
         "{   perf_affinity_mask          |0        |set affinity mask for the main thread}"
@@ -789,6 +880,26 @@ void TestBase::Init(const std::vector<std::string> & availableImpls,
     }
 #endif
 
+    {
+        const char* path = getenv("OPENCV_PERF_VALIDATION_DIR");
+        if (path)
+            perf_validation_results_directory = path;
+    }
+
+    std::string fileName_perf_validation_results_src = args.get<std::string>("perf_read_validation_results");
+    if (!fileName_perf_validation_results_src.empty())
+    {
+        perf_validation_enabled = true;
+        loadPerfValidationResults(perf_validation_results_directory + fileName_perf_validation_results_src);
+    }
+
+    perf_validation_results_outfile = args.get<std::string>("perf_write_validation_results");
+    if (!perf_validation_results_outfile.empty())
+    {
+        perf_validation_enabled = true;
+        ::testing::AddGlobalTestEnvironment(new PerfValidationEnvironment());
+    }
+
     if (!args.check())
     {
         args.printErrors();
@@ -878,7 +989,9 @@ TestBase::TestBase(): testStrategy(PERF_STRATEGY_DEFAULT), declare(this)
 {
     lastTime = totalTime = timeLimit = 0;
     nIters = currentIter = runsPerIteration = 0;
+    minIters = param_min_samples;
     verified = false;
+    perfValidationStage = 0;
 }
 #ifdef _MSC_VER
 # pragma warning(pop)
@@ -1004,7 +1117,7 @@ bool TestBase::next()
                 has_next = false;
                 break;
             }
-            if (currentIter < param_min_samples)
+            if (currentIter < minIters)
             {
                 has_next = true;
                 break;
@@ -1012,14 +1125,96 @@ bool TestBase::next()
 
             calcMetrics();
 
-            double criteria = 0.03;  // 3%
             if (fabs(metrics.mean) > 1e-6)
-                has_next = metrics.stddev > criteria * fabs(metrics.mean);
+                has_next = metrics.stddev > perf_stability_criteria * fabs(metrics.mean);
             else
                 has_next = true;
         }
     } while (false);
 
+    if (perf_validation_enabled && !has_next)
+    {
+        calcMetrics();
+        double median_ms = metrics.median * 1000.0f / metrics.frequency;
+
+        const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
+        std::string name = (test_info == 0) ? "" :
+                std::string(test_info->test_case_name()) + "--" + test_info->name();
+
+        if (!perf_validation_results.empty() && !name.empty())
+        {
+            std::map<std::string, float>::iterator i = perf_validation_results.find(name);
+            bool isSame = false;
+            bool found = false;
+            bool grow = false;
+            if (i != perf_validation_results.end())
+            {
+                found = true;
+                double prev_result = i->second;
+                grow = median_ms > prev_result;
+                isSame = fabs(median_ms - prev_result) <= perf_validation_criteria * fabs(median_ms);
+                if (!isSame)
+                {
+                    if (perfValidationStage == 0)
+                    {
+                        printf("Performance is changed (samples = %d, median):\n    %.2f ms (current)\n    %.2f ms (previous)\n", (int)times.size(), median_ms, prev_result);
+                    }
+                }
+            }
+            else
+            {
+                if (perfValidationStage == 0)
+                    printf("New performance result is detected\n");
+            }
+            if (!isSame)
+            {
+                if (perfValidationStage < 2)
+                {
+                    if (perfValidationStage == 0 && currentIter <= minIters * 3 && currentIter < nIters)
+                    {
+                        unsigned int new_minIters = std::max(minIters * 5, currentIter * 3);
+                        printf("Increase minIters from %u to %u\n", minIters, new_minIters);
+                        minIters = new_minIters;
+                        has_next = true;
+                        perfValidationStage++;
+                    }
+                    else if (found && currentIter >= nIters &&
+                            median_ms > perf_validation_time_threshold_ms &&
+                            (grow || metrics.stddev > perf_stability_criteria * fabs(metrics.mean)))
+                    {
+                        printf("Performance is unstable, it may be a result of overheat problems\n");
+                        printf("Idle delay for %d ms... \n", perf_validation_idle_delay_ms);
+#if defined WIN32 || defined _WIN32 || defined WIN64 || defined _WIN64
+                        Sleep(perf_validation_idle_delay_ms);
+#else
+                        usleep(perf_validation_idle_delay_ms * 1000);
+#endif
+                        has_next = true;
+                        minIters = std::min(minIters * 5, nIters);
+                        // reset collected samples
+                        currentIter = 0;
+                        times.clear();
+                        metrics.clear();
+                        perfValidationStage += 2;
+                    }
+                    if (!has_next)
+                    {
+                        printf("Assume that current result is valid\n");
+                    }
+                }
+                else
+                {
+                    printf("Re-measured performance result: %.2f ms\n", median_ms);
+                }
+            }
+        }
+
+        if (!has_next && !name.empty())
+        {
+            savePerfValidationResult(name, (float)median_ms);
+        }
+    }
+
 #ifdef ANDROID
     if (log_power_checkpoints)
     {
@@ -1223,9 +1418,10 @@ void TestBase::validateMetrics()
     else if (getCurrentPerformanceStrategy() == PERF_STRATEGY_SIMPLE)
     {
         double mean = metrics.mean * 1000.0f / metrics.frequency;
+        double median = metrics.median * 1000.0f / metrics.frequency;
         double stddev = metrics.stddev * 1000.0f / metrics.frequency;
         double percents = stddev / mean * 100.f;
-        printf("[ PERFSTAT ]    (samples = %d, mean = %.2f, stddev = %.2f (%.1f%%))\n", (int)metrics.samples, mean, stddev, percents);
+        printf("[ PERFSTAT ]    (samples = %d, mean = %.2f, median = %.2f, stddev = %.2f (%.1f%%))\n", (int)metrics.samples, mean, median, stddev, percents);
     }
     else
     {