generic-library/opencv
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add accuracy tests while running perf

Browse Source
This commit is contained in:
yao
2013-05-17 13:18:46 +08:00
parent 7d9041460a
commit 5f20fce6fd
22 changed files with 1185 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

451
modules/ocl/perf/precomp.cpp
View File

@@ -41,6 +41,10 @@
//M*/
#include "precomp.hpp"
#if GTEST_OS_WINDOWS
#define NOMINMAX
# include <windows.h>
#endif
// This program test most of the functions in ocl module and generate data metrix of x-factor in .csv files
// All images needed in this test are in samples/gpu folder.
@@ -110,6 +114,7 @@ void TestSystem::finishCurrentSubtest()
return;
}
int is_accurate = is_accurate_;
double cpu_time = cpu_elapsed_ / getTickFrequency() * 1000.0;
double gpu_time = gpu_elapsed_ / getTickFrequency() * 1000.0;
double gpu_full_time = gpu_full_elapsed_ / getTickFrequency() * 1000.0;
@@ -166,8 +171,8 @@ void TestSystem::finishCurrentSubtest()
deviation = std::sqrt(sum / gpu_times_.size());
}
printMetrics(cpu_time, gpu_time, gpu_full_time, speedup, fullspeedup);
writeMetrics(cpu_time, gpu_time, gpu_full_time, speedup, fullspeedup, gpu_min, gpu_max, deviation);
printMetrics(is_accurate, cpu_time, gpu_time, gpu_full_time, speedup, fullspeedup);
writeMetrics(is_accurate, cpu_time, gpu_time, gpu_full_time, speedup, fullspeedup, gpu_min, gpu_max, deviation);
num_subtests_called_++;
resetCurrentSubtest();
@@ -184,10 +189,19 @@ double TestSystem::meanTime(const vector<int64> &samples)
void TestSystem::printHeading()
{
cout << endl;
cout << setiosflags(ios_base::left);
cout << TAB << setw(10) << "CPU, ms" << setw(10) << "GPU, ms"
<< setw(14) << "SPEEDUP" << setw(14) << "GPUTOTAL, ms" << setw(14) << "TOTALSPEEDUP"
<< "DESCRIPTION\n";
cout<< setiosflags(ios_base::left);
#if 0
cout<<TAB<<setw(7)<< "Accu." << setw(10) << "CPU (ms)" << setw(10) << "GPU, ms"
<< setw(8) << "Speedup"<< setw(10)<<"GPUTotal" << setw(10) << "Total"
<< "Description\n";
cout<<TAB<<setw(7)<<""<<setw(10)<<""<<setw(10)<<""<<setw(8)<<""<<setw(10)<<"(ms)"<<setw(10)<<"Speedup\n";
#endif
cout<<TAB<< setw(10) << "CPU (ms)" << setw(10) << "GPU, ms"
<< setw(8) << "Speedup"<< setw(10)<<"GPUTotal" << setw(10) << "Total"
<< "Description\n";
cout<<TAB<<setw(10)<<""<<setw(10)<<""<<setw(8)<<""<<setw(10)<<"(ms)"<<setw(10)<<"Speedup\n";
cout << resetiosflags(ios_base::left);
}
@@ -198,9 +212,14 @@ void TestSystem::writeHeading()
{
recordname_ += "_OCL.csv";
record_ = fopen(recordname_.c_str(), "w");
if(record_ == NULL)
{
cout<<".csv file open failed.\n";
exit(0);
}
}
fprintf(record_, "NAME,DESCRIPTION,CPU (ms),GPU (ms),SPEEDUP,GPUTOTAL (ms),TOTALSPEEDUP,GPU Min (ms),GPU Max (ms), Standard deviation (ms)\n");
fprintf(record_, "NAME,DESCRIPTION,ACCURACY,CPU (ms),GPU (ms),SPEEDUP,GPUTOTAL (ms),TOTALSPEEDUP,GPU Min (ms),GPU Max (ms), Standard deviation (ms)\n");
fflush(record_);
}
@@ -209,54 +228,82 @@ void TestSystem::printSummary()
{
cout << setiosflags(ios_base::fixed);
cout << "\naverage GPU speedup: x"
<< setprecision(3) << speedup_total_ / std::max(1, num_subtests_called_)
<< endl;
<< setprecision(3) << speedup_total_ / std::max(1, num_subtests_called_)
<< endl;
cout << "\nGPU exceeded: "
<< setprecision(3) << speedup_faster_count_
<< "\nGPU passed: "
<< setprecision(3) << speedup_equal_count_
<< "\nGPU failed: "
<< setprecision(3) << speedup_slower_count_
<< endl;
<< setprecision(3) << speedup_faster_count_
<< "\nGPU passed: "
<< setprecision(3) << speedup_equal_count_
<< "\nGPU failed: "
<< setprecision(3) << speedup_slower_count_
<< endl;
cout << "\nGPU exceeded rate: "
<< setprecision(3) << (float)speedup_faster_count_ / std::max(1, num_subtests_called_) * 100
<< "%"
<< "\nGPU passed rate: "
<< setprecision(3) << (float)speedup_equal_count_ / std::max(1, num_subtests_called_) * 100
<< "%"
<< "\nGPU failed rate: "
<< setprecision(3) << (float)speedup_slower_count_ / std::max(1, num_subtests_called_) * 100
<< "%"
<< endl;
<< setprecision(3) << (float)speedup_faster_count_ / std::max(1, num_subtests_called_) * 100
<< "%"
<< "\nGPU passed rate: "
<< setprecision(3) << (float)speedup_equal_count_ / std::max(1, num_subtests_called_) * 100
<< "%"
<< "\nGPU failed rate: "
<< setprecision(3) << (float)speedup_slower_count_ / std::max(1, num_subtests_called_) * 100
<< "%"
<< endl;
cout << "\naverage GPUTOTAL speedup: x"
<< setprecision(3) << speedup_full_total_ / std::max(1, num_subtests_called_)
<< endl;
<< setprecision(3) << speedup_full_total_ / std::max(1, num_subtests_called_)
<< endl;
cout << "\nGPUTOTAL exceeded: "
<< setprecision(3) << speedup_full_faster_count_
<< "\nGPUTOTAL passed: "
<< setprecision(3) << speedup_full_equal_count_
<< "\nGPUTOTAL failed: "
<< setprecision(3) << speedup_full_slower_count_
<< endl;
<< setprecision(3) << speedup_full_faster_count_
<< "\nGPUTOTAL passed: "
<< setprecision(3) << speedup_full_equal_count_
<< "\nGPUTOTAL failed: "
<< setprecision(3) << speedup_full_slower_count_
<< endl;
cout << "\nGPUTOTAL exceeded rate: "
<< setprecision(3) << (float)speedup_full_faster_count_ / std::max(1, num_subtests_called_) * 100
<< "%"
<< "\nGPUTOTAL passed rate: "
<< setprecision(3) << (float)speedup_full_equal_count_ / std::max(1, num_subtests_called_) * 100
<< "%"
<< "\nGPUTOTAL failed rate: "
<< setprecision(3) << (float)speedup_full_slower_count_ / std::max(1, num_subtests_called_) * 100
<< "%"
<< endl;
<< setprecision(3) << (float)speedup_full_faster_count_ / std::max(1, num_subtests_called_) * 100
<< "%"
<< "\nGPUTOTAL passed rate: "
<< setprecision(3) << (float)speedup_full_equal_count_ / std::max(1, num_subtests_called_) * 100
<< "%"
<< "\nGPUTOTAL failed rate: "
<< setprecision(3) << (float)speedup_full_slower_count_ / std::max(1, num_subtests_called_) * 100
<< "%"
<< endl;
cout << resetiosflags(ios_base::fixed);
}
void TestSystem::printMetrics(double cpu_time, double gpu_time, double gpu_full_time, double speedup, double fullspeedup)
{
cout << TAB << setiosflags(ios_base::left);
stringstream stream;
enum GTestColor {
COLOR_DEFAULT,
COLOR_RED,
COLOR_GREEN,
COLOR_YELLOW
};
#if GTEST_OS_WINDOWS&&!GTEST_OS_WINDOWS_MOBILE
// Returns the character attribute for the given color.
WORD GetColorAttribute(GTestColor color) {
switch (color) {
case COLOR_RED: return FOREGROUND_RED;
case COLOR_GREEN: return FOREGROUND_GREEN;
case COLOR_YELLOW: return FOREGROUND_RED | FOREGROUND_GREEN;
default: return 0;
}
}
#else
static const char* GetAnsiColorCode(GTestColor color) {
switch (color) {
case COLOR_RED: return "1";
case COLOR_GREEN: return "2";
case COLOR_YELLOW: return "3";
default: return NULL;
};
}
#endif
static void printMetricsUti(double cpu_time, double gpu_time, double gpu_full_time, double speedup, double fullspeedup, std::stringstream& stream, std::stringstream& cur_subtest_description)
{
//cout <<TAB<< setw(7) << stream.str();
cout <<TAB;
stream.str("");
stream << cpu_time;
cout << setw(10) << stream.str();
@@ -266,21 +313,86 @@ void TestSystem::printMetrics(double cpu_time, double gpu_time, double gpu_full_
stream.str("");
stream << "x" << setprecision(3) << speedup;
cout << setw(14) << stream.str();
cout << setw(8) << stream.str();
stream.str("");
stream << gpu_full_time;
cout << setw(14) << stream.str();
cout << setw(10) << stream.str();
stream.str("");
stream << "x" << setprecision(3) << fullspeedup;
cout << setw(14) << stream.str();
cout << setw(10) << stream.str();
cout << cur_subtest_description_.str();
cout << cur_subtest_description.str();
cout << resetiosflags(ios_base::left) << endl;
}
void TestSystem::writeMetrics(double cpu_time, double gpu_time, double gpu_full_time, double speedup, double fullspeedup, double gpu_min, double gpu_max, double std_dev)
void TestSystem::printMetrics(int is_accurate, double cpu_time, double gpu_time, double gpu_full_time, double speedup, double fullspeedup)
{
cout << setiosflags(ios_base::left);
stringstream stream;
#if 0
if(is_accurate == 1)
stream << "Pass";
else if(is_accurate_ == 0)
stream << "Fail";
else if(is_accurate == -1)
stream << " ";
else
{
std::cout<<"is_accurate errer: "<<is_accurate<<"\n";
exit(-1);
}
#endif
std::stringstream &cur_subtest_description = getCurSubtestDescription();
#if GTEST_OS_WINDOWS&&!GTEST_OS_WINDOWS_MOBILE
WORD color;
const HANDLE stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE);
// Gets the current text color.
CONSOLE_SCREEN_BUFFER_INFO buffer_info;
GetConsoleScreenBufferInfo(stdout_handle, &buffer_info);
const WORD old_color_attrs = buffer_info.wAttributes;
// We need to flush the stream buffers into the console before each
// SetConsoleTextAttribute call lest it affect the text that is already
// printed but has not yet reached the console.
fflush(stdout);
if(is_accurate == 1||is_accurate == -1)
{
color = old_color_attrs;
printMetricsUti(cpu_time, gpu_time, gpu_full_time, speedup, fullspeedup, stream, cur_subtest_description);
}else
{
color = GetColorAttribute(COLOR_RED);
SetConsoleTextAttribute(stdout_handle,
color| FOREGROUND_INTENSITY);
printMetricsUti(cpu_time, gpu_time, gpu_full_time, speedup, fullspeedup, stream, cur_subtest_description);
fflush(stdout);
// Restores the text color.
SetConsoleTextAttribute(stdout_handle, old_color_attrs);
}
#else
GTestColor color = COLOR_RED;
if(is_accurate == 1|| is_accurate == -1)
{
printMetricsUti(cpu_time, gpu_time, gpu_full_time, speedup, fullspeedup, stream, cur_subtest_description);
}else
{
printf("\033[0;3%sm", GetAnsiColorCode(color));
printMetricsUti(cpu_time, gpu_time, gpu_full_time, speedup, fullspeedup, stream, cur_subtest_description);
printf("\033[m"); // Resets the terminal to default.
}
#endif
}
void TestSystem::writeMetrics(int is_accurate, double cpu_time, double gpu_time, double gpu_full_time, double speedup, double fullspeedup, double gpu_min, double gpu_max, double std_dev)
{
if (!record_)
{
@@ -288,10 +400,24 @@ void TestSystem::writeMetrics(double cpu_time, double gpu_time, double gpu_full_
record_ = fopen(recordname_.c_str(), "w");
}
fprintf(record_, "%s,%s,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f\n", itname_changed_ ? itname_.c_str() : "",
cur_subtest_description_.str().c_str(),
cpu_time, gpu_time, speedup, gpu_full_time, fullspeedup,
gpu_min, gpu_max, std_dev);
string _is_accurate_;
if(is_accurate == 1)
_is_accurate_ = "Pass";
else if(is_accurate == 0)
_is_accurate_ = "Fail";
else if(is_accurate == -1)
_is_accurate_ = " ";
else
{
std::cout<<"is_accurate errer: "<<is_accurate<<"\n";
exit(-1);
}
fprintf(record_, "%s,%s,%s,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f\n", itname_changed_ ? itname_.c_str() : "",
cur_subtest_description_.str().c_str(),
_is_accurate_.c_str(), cpu_time, gpu_time, speedup, gpu_full_time, fullspeedup,
gpu_min, gpu_max, std_dev);
if (itname_changed_)
{
@@ -310,31 +436,31 @@ void TestSystem::writeSummary()
}
fprintf(record_, "\nAverage GPU speedup: %.3f\n"
"exceeded: %d (%.3f%%)\n"
"passed: %d (%.3f%%)\n"
"failed: %d (%.3f%%)\n"
"\nAverage GPUTOTAL speedup: %.3f\n"
"exceeded: %d (%.3f%%)\n"
"passed: %d (%.3f%%)\n"
"failed: %d (%.3f%%)\n",
speedup_total_ / std::max(1, num_subtests_called_),
speedup_faster_count_, (float)speedup_faster_count_ / std::max(1, num_subtests_called_) * 100,
speedup_equal_count_, (float)speedup_equal_count_ / std::max(1, num_subtests_called_) * 100,
speedup_slower_count_, (float)speedup_slower_count_ / std::max(1, num_subtests_called_) * 100,
speedup_full_total_ / std::max(1, num_subtests_called_),
speedup_full_faster_count_, (float)speedup_full_faster_count_ / std::max(1, num_subtests_called_) * 100,
speedup_full_equal_count_, (float)speedup_full_equal_count_ / std::max(1, num_subtests_called_) * 100,
speedup_full_slower_count_, (float)speedup_full_slower_count_ / std::max(1, num_subtests_called_) * 100
);
"exceeded: %d (%.3f%%)\n"
"passed: %d (%.3f%%)\n"
"failed: %d (%.3f%%)\n"
"\nAverage GPUTOTAL speedup: %.3f\n"
"exceeded: %d (%.3f%%)\n"
"passed: %d (%.3f%%)\n"
"failed: %d (%.3f%%)\n",
speedup_total_ / std::max(1, num_subtests_called_),
speedup_faster_count_, (float)speedup_faster_count_ / std::max(1, num_subtests_called_) * 100,
speedup_equal_count_, (float)speedup_equal_count_ / std::max(1, num_subtests_called_) * 100,
speedup_slower_count_, (float)speedup_slower_count_ / std::max(1, num_subtests_called_) * 100,
speedup_full_total_ / std::max(1, num_subtests_called_),
speedup_full_faster_count_, (float)speedup_full_faster_count_ / std::max(1, num_subtests_called_) * 100,
speedup_full_equal_count_, (float)speedup_full_equal_count_ / std::max(1, num_subtests_called_) * 100,
speedup_full_slower_count_, (float)speedup_full_slower_count_ / std::max(1, num_subtests_called_) * 100
);
fflush(record_);
}
void TestSystem::printError(const std::string &msg)
{
if(msg != "CL_INVALID_BUFFER_SIZE")
{
cout << TAB << "[error: " << msg << "] " << cur_subtest_description_.str() << endl;
}
if(msg != "CL_INVALID_BUFFER_SIZE")
{
cout << TAB << "[error: " << msg << "] " << cur_subtest_description_.str() << endl;
}
}
void gen(Mat &mat, int rows, int cols, int type, Scalar low, Scalar high)
@@ -343,7 +469,134 @@ void gen(Mat &mat, int rows, int cols, int type, Scalar low, Scalar high)
RNG rng(0);
rng.fill(mat, RNG::UNIFORM, low, high);
}
#if 0
void gen(Mat &mat, int rows, int cols, int type, int low, int high, int n)
{
assert(n > 0&&n <= cols * rows);
assert(type == CV_8UC1||type == CV_8UC3||type == CV_8UC4
||type == CV_32FC1||type == CV_32FC3||type == CV_32FC4);
RNG rng;
//generate random position without duplication
std::vector<int> pos;
for(int i = 0; i < cols * rows; i++)
{
pos.push_back(i);
}
for(int i = 0; i < cols * rows; i++)
{
int temp = i + rng.uniform(0, cols * rows - 1 - i);
int temp1 = pos[temp];
pos[temp]= pos[i];
pos[i] = temp1;
}
std::vector<int> selected_pos;
for(int i = 0; i < n; i++)
{
selected_pos.push_back(pos[i]);
}
pos.clear();
//end of generating random y without duplication
if(type == CV_8UC1)
{
typedef struct coorStruct_
{
int x;
int y;
uchar xy;
}coorStruct;
coorStruct coor_struct;
std::vector<coorStruct> coor;
for(int i = 0; i < n; i++)
{
coor_struct.x = -1;
coor_struct.y = -1;
coor_struct.xy = (uchar)rng.uniform(low, high);
coor.push_back(coor_struct);
}
for(int i = 0; i < n; i++)
{
coor[i].y = selected_pos[i]/cols;
coor[i].x = selected_pos[i]%cols;
}
selected_pos.clear();
mat.create(rows, cols, type);
mat.setTo(0);
for(int i = 0; i < n; i++)
{
mat.at<unsigned char>(coor[i].y, coor[i].x) = coor[i].xy;
}
}
if(type == CV_8UC4 || type == CV_8UC3)
{
mat.create(rows, cols, type);
mat.setTo(0);
typedef struct Coor
{
int x;
int y;
uchar r;
uchar g;
uchar b;
uchar alpha;
}coor;
std::vector<coor> coor_vect;
coor xy_coor;
for(int i = 0; i < n; i++)
{
xy_coor.r = (uchar)rng.uniform(low, high);
xy_coor.g = (uchar)rng.uniform(low, high);
xy_coor.b = (uchar)rng.uniform(low, high);
if(type == CV_8UC4)
xy_coor.alpha = (uchar)rng.uniform(low, high);
coor_vect.push_back(xy_coor);
}
for(int i = 0; i < n; i++)
{
coor_vect[i].y = selected_pos[i]/((int)mat.step1()/mat.elemSize());
coor_vect[i].x = selected_pos[i]%((int)mat.step1()/mat.elemSize());
//printf("coor_vect[%d] = (%d, %d)\n", i, coor_vect[i].y, coor_vect[i].x);
}
if(type == CV_8UC4)
{
for(int i = 0; i < n; i++)
{
mat.at<unsigned char>(coor_vect[i].y, 4 * coor_vect[i].x) = coor_vect[i].r;
mat.at<unsigned char>(coor_vect[i].y, 4 * coor_vect[i].x + 1) = coor_vect[i].g;
mat.at<unsigned char>(coor_vect[i].y, 4 * coor_vect[i].x + 2) = coor_vect[i].b;
mat.at<unsigned char>(coor_vect[i].y, 4 * coor_vect[i].x + 3) = coor_vect[i].alpha;
}
}else if(type == CV_8UC3)
{
for(int i = 0; i < n; i++)
{
mat.at<unsigned char>(coor_vect[i].y, 3 * coor_vect[i].x) = coor_vect[i].r;
mat.at<unsigned char>(coor_vect[i].y, 3 * coor_vect[i].x + 1) = coor_vect[i].g;
mat.at<unsigned char>(coor_vect[i].y, 3 * coor_vect[i].x + 2) = coor_vect[i].b;
}
}
}
}
#endif
string abspath(const string &relpath)
{
@@ -352,11 +605,57 @@ string abspath(const string &relpath)
int CV_CDECL cvErrorCallback(int /*status*/, const char * /*func_name*/,
const char *err_msg, const char * /*file_name*/,
int /*line*/, void * /*userdata*/)
const char *err_msg, const char * /*file_name*/,
int /*line*/, void * /*userdata*/)
{
TestSystem::instance().printError(err_msg);
return 0;
}
double checkNorm(const Mat &m)
{
return norm(m, NORM_INF);
}
double checkNorm(const Mat &m1, const Mat &m2)
{
return norm(m1, m2, NORM_INF);
}
double checkSimilarity(const Mat &m1, const Mat &m2)
{
Mat diff;
matchTemplate(m1, m2, diff, CV_TM_CCORR_NORMED);
return std::abs(diff.at<float>(0, 0) - 1.f);
}
int ExpectedMatNear(cv::Mat dst, cv::Mat cpu_dst, double eps)
{
assert(dst.type() == cpu_dst.type());
assert(dst.size() == cpu_dst.size());
if(checkNorm(cv::Mat(dst), cv::Mat(cpu_dst)) < eps ||checkNorm(cv::Mat(dst), cv::Mat(cpu_dst)) == eps)
return 1;
return 0;
}
int ExceptDoubleNear(double val1, double val2, double abs_error)
{
const double diff = fabs(val1 - val2);
if (diff <= abs_error)
return 1;
return 0;
}
int ExceptedMatSimilar(cv::Mat dst, cv::Mat cpu_dst, double eps)
{
assert(dst.type() == cpu_dst.type());
assert(dst.size() == cpu_dst.size());
if(checkSimilarity(cv::Mat(cpu_dst), cv::Mat(dst)) <= eps)
return 1;
return 0;
}