generic-library/opencv
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #3342 from alalek:perf_stability_check

Browse Source
This commit is contained in:
Alexander Alekhin 2014-10-24 16:00:20 +00:00
commit 1fbad26fe3
2 changed files with 203 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/ts/include/opencv2/ts/ts_perf.hpp
View File

@ -419,9 +419,11 @@ private:
static int64 timeLimitDefault;
static unsigned int iterationsLimitDefault;
unsigned int minIters;
unsigned int nIters;
unsigned int currentIter;
unsigned int runsPerIteration;
unsigned int perfValidationStage;
performance_metrics metrics;
void validateMetrics();

206
modules/ts/src/ts_perf.cpp
View File

@ -1,5 +1,16 @@
#include "precomp.hpp"
#include <map>
#include <iostream>
#include <fstream>
#if defined WIN32 || defined _WIN32 || defined WIN64 || defined _WIN64
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#endif
#ifdef HAVE_CUDA
#include "opencv2/core/cuda.hpp"
#endif
@ -35,11 +46,11 @@ static bool param_verify_sanity;
static bool param_collect_impl;
#endif
extern bool test_ipp_check;
#ifdef HAVE_CUDA
static int param_cuda_device;
#endif
#ifdef ANDROID
static int param_affinity_mask;
static bool log_power_checkpoints;
@ -59,6 +70,8 @@ static void setCurrentThreadAffinityMask(int mask)
}
#endif
static double perf_stability_criteria = 0.03; // 3%
namespace {
class PerfEnvironment: public ::testing::Environment
@ -635,6 +648,82 @@ void performance_metrics::clear()
terminationReason = TERM_UNKNOWN;
}
/*****************************************************************************************\
* Performance validation results
\*****************************************************************************************/
static bool perf_validation_enabled = false;
static std::string perf_validation_results_directory;
static std::map<std::string, float> perf_validation_results;
static std::string perf_validation_results_outfile;
static double perf_validation_criteria = 0.03; // 3 %
static double perf_validation_time_threshold_ms = 0.1;
static int perf_validation_idle_delay_ms = 3000; // 3 sec
static void loadPerfValidationResults(const std::string& fileName)
{
perf_validation_results.clear();
std::ifstream infile(fileName.c_str());
while (!infile.eof())
{
std::string name;
float value = 0;
if (!(infile >> value))
{
if (infile.eof())
break; // it is OK
std::cout << "ERROR: Can't load performance validation results from " << fileName << "!" << std::endl;
return;
}
infile.ignore(1);
if (!(std::getline(infile, name)))
{
std::cout << "ERROR: Can't load performance validation results from " << fileName << "!" << std::endl;
return;
}
if (!name.empty() && name[name.size() - 1] == '\r') // CRLF processing on Linux
name.resize(name.size() - 1);
perf_validation_results[name] = value;
}
std::cout << "Performance validation results loaded from " << fileName << " (" << perf_validation_results.size() << " entries)" << std::endl;
}
static void savePerfValidationResult(const std::string& name, float value)
{
perf_validation_results[name] = value;
}
static void savePerfValidationResults()
{
if (!perf_validation_results_outfile.empty())
{
std::ofstream outfile((perf_validation_results_directory + perf_validation_results_outfile).c_str());
std::map<std::string, float>::const_iterator i;
for (i = perf_validation_results.begin(); i != perf_validation_results.end(); ++i)
{
outfile << i->second << ';';
outfile << i->first << std::endl;
}
outfile.close();
std::cout << "Performance validation results saved (" << perf_validation_results.size() << " entries)" << std::endl;
}
}
class PerfValidationEnvironment : public ::testing::Environment
{
public:
virtual ~PerfValidationEnvironment() {}
virtual void SetUp() {}
virtual void TearDown()
{
savePerfValidationResults();
}
};
/*****************************************************************************************\
* ::perf::TestBase
@ -666,6 +755,8 @@ void TestBase::Init(const std::vector<std::string> & availableImpls,
"{ perf_list_impls |false |list available implementation variants and exit}"
"{ perf_run_cpu |false |deprecated, equivalent to --perf_impl=plain}"
"{ perf_strategy |default |specifies performance measuring strategy: default, base or simple (weak restrictions)}"
"{ perf_read_validation_results | |specifies file name with performance results from previous run}"
"{ perf_write_validation_results | |specifies file name to write performance validation results}"
#ifdef ANDROID
"{ perf_time_limit |6.0 |default time limit for a single test (in seconds)}"
"{ perf_affinity_mask |0 |set affinity mask for the main thread}"
@ -789,6 +880,26 @@ void TestBase::Init(const std::vector<std::string> & availableImpls,
}
#endif
{
const char* path = getenv("OPENCV_PERF_VALIDATION_DIR");
if (path)
perf_validation_results_directory = path;
}
std::string fileName_perf_validation_results_src = args.get<std::string>("perf_read_validation_results");
if (!fileName_perf_validation_results_src.empty())
{
perf_validation_enabled = true;
loadPerfValidationResults(perf_validation_results_directory + fileName_perf_validation_results_src);
}
perf_validation_results_outfile = args.get<std::string>("perf_write_validation_results");
if (!perf_validation_results_outfile.empty())
{
perf_validation_enabled = true;
::testing::AddGlobalTestEnvironment(new PerfValidationEnvironment());
}
if (!args.check())
{
args.printErrors();
@ -878,7 +989,9 @@ TestBase::TestBase(): testStrategy(PERF_STRATEGY_DEFAULT), declare(this)
{
lastTime = totalTime = timeLimit = 0;
nIters = currentIter = runsPerIteration = 0;
minIters = param_min_samples;
verified = false;
perfValidationStage = 0;
}
#ifdef _MSC_VER
# pragma warning(pop)
@ -1004,7 +1117,7 @@ bool TestBase::next()
has_next = false;
break;
}
if (currentIter < param_min_samples)
if (currentIter < minIters)
{
has_next = true;
break;
@ -1012,14 +1125,96 @@ bool TestBase::next()
calcMetrics();
double criteria = 0.03; // 3%
if (fabs(metrics.mean) > 1e-6)
has_next = metrics.stddev > criteria * fabs(metrics.mean);
has_next = metrics.stddev > perf_stability_criteria * fabs(metrics.mean);
else
has_next = true;
}
} while (false);
if (perf_validation_enabled && !has_next)
{
calcMetrics();
double median_ms = metrics.median * 1000.0f / metrics.frequency;
const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info();
std::string name = (test_info == 0) ? "" :
std::string(test_info->test_case_name()) + "--" + test_info->name();
if (!perf_validation_results.empty() && !name.empty())
{
std::map<std::string, float>::iterator i = perf_validation_results.find(name);
bool isSame = false;
bool found = false;
bool grow = false;
if (i != perf_validation_results.end())
{
found = true;
double prev_result = i->second;
grow = median_ms > prev_result;
isSame = fabs(median_ms - prev_result) <= perf_validation_criteria * fabs(median_ms);
if (!isSame)
{
if (perfValidationStage == 0)
{
printf("Performance is changed (samples = %d, median):\n %.2f ms (current)\n %.2f ms (previous)\n", (int)times.size(), median_ms, prev_result);
}
}
}
else
{
if (perfValidationStage == 0)
printf("New performance result is detected\n");
}
if (!isSame)
{
if (perfValidationStage < 2)
{
if (perfValidationStage == 0 && currentIter <= minIters * 3 && currentIter < nIters)
{
unsigned int new_minIters = std::max(minIters * 5, currentIter * 3);
printf("Increase minIters from %u to %u\n", minIters, new_minIters);
minIters = new_minIters;
has_next = true;
perfValidationStage++;
}
else if (found && currentIter >= nIters &&
median_ms > perf_validation_time_threshold_ms &&
(grow || metrics.stddev > perf_stability_criteria * fabs(metrics.mean)))
{
printf("Performance is unstable, it may be a result of overheat problems\n");
printf("Idle delay for %d ms... \n", perf_validation_idle_delay_ms);
#if defined WIN32 || defined _WIN32 || defined WIN64 || defined _WIN64
Sleep(perf_validation_idle_delay_ms);
#else
usleep(perf_validation_idle_delay_ms * 1000);
#endif
has_next = true;
minIters = std::min(minIters * 5, nIters);
// reset collected samples
currentIter = 0;
times.clear();
metrics.clear();
perfValidationStage += 2;
}
if (!has_next)
{
printf("Assume that current result is valid\n");
}
}
else
{
printf("Re-measured performance result: %.2f ms\n", median_ms);
}
}
}
if (!has_next && !name.empty())
{
savePerfValidationResult(name, (float)median_ms);
}
}
#ifdef ANDROID
if (log_power_checkpoints)
{
@ -1223,9 +1418,10 @@ void TestBase::validateMetrics()
else if (getCurrentPerformanceStrategy() == PERF_STRATEGY_SIMPLE)
{
double mean = metrics.mean * 1000.0f / metrics.frequency;
double median = metrics.median * 1000.0f / metrics.frequency;
double stddev = metrics.stddev * 1000.0f / metrics.frequency;
double percents = stddev / mean * 100.f;
printf("[ PERFSTAT ] (samples = %d, mean = %.2f, stddev = %.2f (%.1f%%))\n", (int)metrics.samples, mean, stddev, percents);
printf("[ PERFSTAT ] (samples = %d, mean = %.2f, median = %.2f, stddev = %.2f (%.1f%%))\n", (int)metrics.samples, mean, median, stddev, percents);
}
else
{