diff --git a/modules/ts/include/opencv2/ts/ts_perf.hpp b/modules/ts/include/opencv2/ts/ts_perf.hpp index 76fecba5b..aac8ae502 100644 --- a/modules/ts/include/opencv2/ts/ts_perf.hpp +++ b/modules/ts/include/opencv2/ts/ts_perf.hpp @@ -419,9 +419,11 @@ private: static int64 timeLimitDefault; static unsigned int iterationsLimitDefault; + unsigned int minIters; unsigned int nIters; unsigned int currentIter; unsigned int runsPerIteration; + unsigned int perfValidationStage; performance_metrics metrics; void validateMetrics(); diff --git a/modules/ts/src/ts_perf.cpp b/modules/ts/src/ts_perf.cpp index c970101e5..f2eae265a 100644 --- a/modules/ts/src/ts_perf.cpp +++ b/modules/ts/src/ts_perf.cpp @@ -1,5 +1,16 @@ #include "precomp.hpp" +#include +#include +#include + +#if defined WIN32 || defined _WIN32 || defined WIN64 || defined _WIN64 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif + #ifdef HAVE_CUDA #include "opencv2/core/cuda.hpp" #endif @@ -35,11 +46,11 @@ static bool param_verify_sanity; static bool param_collect_impl; #endif extern bool test_ipp_check; + #ifdef HAVE_CUDA static int param_cuda_device; #endif - #ifdef ANDROID static int param_affinity_mask; static bool log_power_checkpoints; @@ -59,6 +70,8 @@ static void setCurrentThreadAffinityMask(int mask) } #endif +static double perf_stability_criteria = 0.03; // 3% + namespace { class PerfEnvironment: public ::testing::Environment @@ -635,6 +648,82 @@ void performance_metrics::clear() terminationReason = TERM_UNKNOWN; } +/*****************************************************************************************\ +* Performance validation results +\*****************************************************************************************/ + +static bool perf_validation_enabled = false; + +static std::string perf_validation_results_directory; +static std::map perf_validation_results; +static std::string perf_validation_results_outfile; + +static double perf_validation_criteria = 0.03; // 3 % +static double perf_validation_time_threshold_ms = 0.1; +static int perf_validation_idle_delay_ms = 3000; // 3 sec + +static void loadPerfValidationResults(const std::string& fileName) +{ + perf_validation_results.clear(); + std::ifstream infile(fileName.c_str()); + while (!infile.eof()) + { + std::string name; + float value = 0; + if (!(infile >> value)) + { + if (infile.eof()) + break; // it is OK + std::cout << "ERROR: Can't load performance validation results from " << fileName << "!" << std::endl; + return; + } + infile.ignore(1); + if (!(std::getline(infile, name))) + { + std::cout << "ERROR: Can't load performance validation results from " << fileName << "!" << std::endl; + return; + } + if (!name.empty() && name[name.size() - 1] == '\r') // CRLF processing on Linux + name.resize(name.size() - 1); + perf_validation_results[name] = value; + } + std::cout << "Performance validation results loaded from " << fileName << " (" << perf_validation_results.size() << " entries)" << std::endl; +} + +static void savePerfValidationResult(const std::string& name, float value) +{ + perf_validation_results[name] = value; +} + +static void savePerfValidationResults() +{ + if (!perf_validation_results_outfile.empty()) + { + std::ofstream outfile((perf_validation_results_directory + perf_validation_results_outfile).c_str()); + std::map::const_iterator i; + for (i = perf_validation_results.begin(); i != perf_validation_results.end(); ++i) + { + outfile << i->second << ';'; + outfile << i->first << std::endl; + } + outfile.close(); + std::cout << "Performance validation results saved (" << perf_validation_results.size() << " entries)" << std::endl; + } +} + +class PerfValidationEnvironment : public ::testing::Environment +{ +public: + virtual ~PerfValidationEnvironment() {} + virtual void SetUp() {} + + virtual void TearDown() + { + savePerfValidationResults(); + } +}; + + /*****************************************************************************************\ * ::perf::TestBase @@ -666,6 +755,8 @@ void TestBase::Init(const std::vector & availableImpls, "{ perf_list_impls |false |list available implementation variants and exit}" "{ perf_run_cpu |false |deprecated, equivalent to --perf_impl=plain}" "{ perf_strategy |default |specifies performance measuring strategy: default, base or simple (weak restrictions)}" + "{ perf_read_validation_results | |specifies file name with performance results from previous run}" + "{ perf_write_validation_results | |specifies file name to write performance validation results}" #ifdef ANDROID "{ perf_time_limit |6.0 |default time limit for a single test (in seconds)}" "{ perf_affinity_mask |0 |set affinity mask for the main thread}" @@ -789,6 +880,26 @@ void TestBase::Init(const std::vector & availableImpls, } #endif + { + const char* path = getenv("OPENCV_PERF_VALIDATION_DIR"); + if (path) + perf_validation_results_directory = path; + } + + std::string fileName_perf_validation_results_src = args.get("perf_read_validation_results"); + if (!fileName_perf_validation_results_src.empty()) + { + perf_validation_enabled = true; + loadPerfValidationResults(perf_validation_results_directory + fileName_perf_validation_results_src); + } + + perf_validation_results_outfile = args.get("perf_write_validation_results"); + if (!perf_validation_results_outfile.empty()) + { + perf_validation_enabled = true; + ::testing::AddGlobalTestEnvironment(new PerfValidationEnvironment()); + } + if (!args.check()) { args.printErrors(); @@ -878,7 +989,9 @@ TestBase::TestBase(): testStrategy(PERF_STRATEGY_DEFAULT), declare(this) { lastTime = totalTime = timeLimit = 0; nIters = currentIter = runsPerIteration = 0; + minIters = param_min_samples; verified = false; + perfValidationStage = 0; } #ifdef _MSC_VER # pragma warning(pop) @@ -1004,7 +1117,7 @@ bool TestBase::next() has_next = false; break; } - if (currentIter < param_min_samples) + if (currentIter < minIters) { has_next = true; break; @@ -1012,14 +1125,96 @@ bool TestBase::next() calcMetrics(); - double criteria = 0.03; // 3% if (fabs(metrics.mean) > 1e-6) - has_next = metrics.stddev > criteria * fabs(metrics.mean); + has_next = metrics.stddev > perf_stability_criteria * fabs(metrics.mean); else has_next = true; } } while (false); + if (perf_validation_enabled && !has_next) + { + calcMetrics(); + double median_ms = metrics.median * 1000.0f / metrics.frequency; + + const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); + std::string name = (test_info == 0) ? "" : + std::string(test_info->test_case_name()) + "--" + test_info->name(); + + if (!perf_validation_results.empty() && !name.empty()) + { + std::map::iterator i = perf_validation_results.find(name); + bool isSame = false; + bool found = false; + bool grow = false; + if (i != perf_validation_results.end()) + { + found = true; + double prev_result = i->second; + grow = median_ms > prev_result; + isSame = fabs(median_ms - prev_result) <= perf_validation_criteria * fabs(median_ms); + if (!isSame) + { + if (perfValidationStage == 0) + { + printf("Performance is changed (samples = %d, median):\n %.2f ms (current)\n %.2f ms (previous)\n", (int)times.size(), median_ms, prev_result); + } + } + } + else + { + if (perfValidationStage == 0) + printf("New performance result is detected\n"); + } + if (!isSame) + { + if (perfValidationStage < 2) + { + if (perfValidationStage == 0 && currentIter <= minIters * 3 && currentIter < nIters) + { + unsigned int new_minIters = std::max(minIters * 5, currentIter * 3); + printf("Increase minIters from %u to %u\n", minIters, new_minIters); + minIters = new_minIters; + has_next = true; + perfValidationStage++; + } + else if (found && currentIter >= nIters && + median_ms > perf_validation_time_threshold_ms && + (grow || metrics.stddev > perf_stability_criteria * fabs(metrics.mean))) + { + printf("Performance is unstable, it may be a result of overheat problems\n"); + printf("Idle delay for %d ms... \n", perf_validation_idle_delay_ms); +#if defined WIN32 || defined _WIN32 || defined WIN64 || defined _WIN64 + Sleep(perf_validation_idle_delay_ms); +#else + usleep(perf_validation_idle_delay_ms * 1000); +#endif + has_next = true; + minIters = std::min(minIters * 5, nIters); + // reset collected samples + currentIter = 0; + times.clear(); + metrics.clear(); + perfValidationStage += 2; + } + if (!has_next) + { + printf("Assume that current result is valid\n"); + } + } + else + { + printf("Re-measured performance result: %.2f ms\n", median_ms); + } + } + } + + if (!has_next && !name.empty()) + { + savePerfValidationResult(name, (float)median_ms); + } + } + #ifdef ANDROID if (log_power_checkpoints) { @@ -1223,9 +1418,10 @@ void TestBase::validateMetrics() else if (getCurrentPerformanceStrategy() == PERF_STRATEGY_SIMPLE) { double mean = metrics.mean * 1000.0f / metrics.frequency; + double median = metrics.median * 1000.0f / metrics.frequency; double stddev = metrics.stddev * 1000.0f / metrics.frequency; double percents = stddev / mean * 100.f; - printf("[ PERFSTAT ] (samples = %d, mean = %.2f, stddev = %.2f (%.1f%%))\n", (int)metrics.samples, mean, stddev, percents); + printf("[ PERFSTAT ] (samples = %d, mean = %.2f, median = %.2f, stddev = %.2f (%.1f%%))\n", (int)metrics.samples, mean, median, stddev, percents); } else {