From f1873bbca19400110f6a0fa4333453165ebdc963 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 12 Nov 2013 18:15:50 +0400 Subject: [PATCH 1/2] perf tests: allow to skip performance tests --- modules/ts/include/opencv2/ts/ts_perf.hpp | 3 ++ modules/ts/src/ts_perf.cpp | 47 ++++++++++++++++------- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/modules/ts/include/opencv2/ts/ts_perf.hpp b/modules/ts/include/opencv2/ts/ts_perf.hpp index 6a0e9215b..9238b3e34 100644 --- a/modules/ts/include/opencv2/ts/ts_perf.hpp +++ b/modules/ts/include/opencv2/ts/ts_perf.hpp @@ -243,6 +243,7 @@ typedef struct CV_EXPORTS performance_metrics TERM_TIME = 1, TERM_INTERRUPT = 2, TERM_EXCEPTION = 3, + TERM_SKIP_TEST = 4, // there are some limitations and test should be skipped TERM_UNKNOWN = -1 }; @@ -279,6 +280,8 @@ public: static enum PERF_STRATEGY getPerformanceStrategy(); static enum PERF_STRATEGY setPerformanceStrategy(enum PERF_STRATEGY strategy); + class PerfSkipTestException: public cv::Exception {}; + protected: virtual void PerfTestBody() = 0; diff --git a/modules/ts/src/ts_perf.cpp b/modules/ts/src/ts_perf.cpp index 319076ca8..3bcb5c11a 100644 --- a/modules/ts/src/ts_perf.cpp +++ b/modules/ts/src/ts_perf.cpp @@ -1175,7 +1175,14 @@ void TestBase::reportMetrics(bool toJUnitXML) { performance_metrics& m = calcMetrics(); - if (toJUnitXML) + if (m.terminationReason == performance_metrics::TERM_SKIP_TEST) + { + if (toJUnitXML) + { + RecordProperty("custom_status", "skipped"); + } + } + else if (toJUnitXML) { RecordProperty("bytesIn", (int)m.bytesIn); RecordProperty("bytesOut", (int)m.bytesOut); @@ -1267,21 +1274,30 @@ void TestBase::SetUp() void TestBase::TearDown() { - if (!HasFailure() && !verified) - ADD_FAILURE() << "The test has no sanity checks. There should be at least one check at the end of performance test."; - - validateMetrics(); - if (HasFailure()) - reportMetrics(false); + if (metrics.terminationReason == performance_metrics::TERM_SKIP_TEST) + { + LOGI("\tTest was skipped"); + GTEST_SUCCEED() << "Test was skipped"; + } else { - const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); - const char* type_param = test_info->type_param(); - const char* value_param = test_info->value_param(); - if (value_param) printf("[ VALUE ] \t%s\n", value_param), fflush(stdout); - if (type_param) printf("[ TYPE ] \t%s\n", type_param), fflush(stdout); - reportMetrics(true); + if (!HasFailure() && !verified) + ADD_FAILURE() << "The test has no sanity checks. There should be at least one check at the end of performance test."; + + validateMetrics(); + if (HasFailure()) + { + reportMetrics(false); + return; + } } + + const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); + const char* type_param = test_info->type_param(); + const char* value_param = test_info->value_param(); + if (value_param) printf("[ VALUE ] \t%s\n", value_param), fflush(stdout); + if (type_param) printf("[ TYPE ] \t%s\n", type_param), fflush(stdout); + reportMetrics(true); } std::string TestBase::getDataPath(const std::string& relativePath) @@ -1331,6 +1347,11 @@ void TestBase::RunPerfTestBody() { this->PerfTestBody(); } + catch(PerfSkipTestException&) + { + metrics.terminationReason = performance_metrics::TERM_SKIP_TEST; + return; + } catch(PerfEarlyExitException&) { metrics.terminationReason = performance_metrics::TERM_INTERRUPT; From fb326a58fc0b9c1b8bb64e115ab85b2354616729 Mon Sep 17 00:00:00 2001 From: Alexander Alekhin Date: Tue, 12 Nov 2013 18:59:43 +0400 Subject: [PATCH 2/2] ocl: perf tests: skip tests if we fail check for maxMemoryAllocSize --- modules/ocl/perf/perf_arithm.cpp | 3 +++ modules/ocl/perf/perf_filters.cpp | 18 ++++++++++++++ modules/ocl/perf/perf_imgwarp.cpp | 5 +++- modules/ocl/perf/perf_matrix_operation.cpp | 4 ++++ modules/ocl/perf/perf_precomp.hpp | 28 +++++++++++++++++++--- modules/ocl/perf/perf_pyramid.cpp | 10 ++++++-- modules/ocl/perf/perf_split_merge.cpp | 9 +++++-- 7 files changed, 69 insertions(+), 8 deletions(-) diff --git a/modules/ocl/perf/perf_arithm.cpp b/modules/ocl/perf/perf_arithm.cpp index 24eab3b91..2699b44a7 100644 --- a/modules/ocl/perf/perf_arithm.cpp +++ b/modules/ocl/perf/perf_arithm.cpp @@ -1066,6 +1066,9 @@ PERF_TEST_P(RepeatFixture, Repeat, const int nx = 3, ny = 2; const Size dstSize(srcSize.width * nx, srcSize.height * ny); + checkDeviceMaxMemoryAllocSize(srcSize, type); + checkDeviceMaxMemoryAllocSize(dstSize, type); + Mat src(srcSize, type), dst(dstSize, type); declare.in(src, WARMUP_RNG).out(dst); diff --git a/modules/ocl/perf/perf_filters.cpp b/modules/ocl/perf/perf_filters.cpp index ef0ea7b45..7e5389df6 100644 --- a/modules/ocl/perf/perf_filters.cpp +++ b/modules/ocl/perf/perf_filters.cpp @@ -61,6 +61,8 @@ PERF_TEST_P(BlurFixture, Blur, const Size srcSize = get<0>(params), ksize(3, 3); const int type = get<1>(params), bordertype = BORDER_CONSTANT; + checkDeviceMaxMemoryAllocSize(srcSize, type); + Mat src(srcSize, type), dst(srcSize, type); declare.in(src, WARMUP_RNG).out(dst); @@ -99,6 +101,8 @@ PERF_TEST_P(LaplacianFixture, Laplacian, const Size srcSize = get<0>(params); const int type = get<1>(params), ksize = 3; + checkDeviceMaxMemoryAllocSize(srcSize, type); + Mat src(srcSize, type), dst(srcSize, type); declare.in(src, WARMUP_RNG).out(dst); @@ -138,6 +142,8 @@ PERF_TEST_P(ErodeFixture, Erode, const int type = get<1>(params), ksize = 3; const Mat ker = getStructuringElement(MORPH_RECT, Size(ksize, ksize)); + checkDeviceMaxMemoryAllocSize(srcSize, type); + Mat src(srcSize, type), dst(srcSize, type); declare.in(src, WARMUP_RNG).out(dst).in(ker); @@ -176,6 +182,8 @@ PERF_TEST_P(SobelFixture, Sobel, const Size srcSize = get<0>(params); const int type = get<1>(params), dx = 1, dy = 1; + checkDeviceMaxMemoryAllocSize(srcSize, type, sizeof(float) * 2); + Mat src(srcSize, type), dst(srcSize, type); declare.in(src, WARMUP_RNG).out(dst); @@ -217,6 +225,8 @@ PERF_TEST_P(ScharrFixture, Scharr, const Size srcSize = get<0>(params); const int type = get<1>(params), dx = 1, dy = 0; + checkDeviceMaxMemoryAllocSize(srcSize, type, sizeof(float) * 2); + Mat src(srcSize, type), dst(srcSize, type); declare.in(src, WARMUP_RNG).out(dst); @@ -258,6 +268,8 @@ PERF_TEST_P(GaussianBlurFixture, GaussianBlur, const Size srcSize = get<0>(params); const int type = get<1>(params), ksize = 7; + checkDeviceMaxMemoryAllocSize(srcSize, type); + Mat src(srcSize, type), dst(srcSize, type); declare.in(src, WARMUP_RNG).out(dst); @@ -295,6 +307,8 @@ PERF_TEST_P(filter2DFixture, filter2D, const Size srcSize = get<0>(params); const int type = get<1>(params), ksize = 3; + checkDeviceMaxMemoryAllocSize(srcSize, type); + Mat src(srcSize, type), dst(srcSize, type), kernel(ksize, ksize, CV_32SC1); declare.in(src, WARMUP_RNG).in(kernel).out(dst); randu(kernel, -3.0, 3.0); @@ -335,6 +349,8 @@ PERF_TEST_P(BilateralFixture, Bilateral, const int type = get<1>(params), d = 7; const double sigmacolor = 50.0, sigmaspace = 50.0; + checkDeviceMaxMemoryAllocSize(srcSize, type); + Mat src(srcSize, type), dst(srcSize, type); declare.in(src, WARMUP_RNG).out(dst); @@ -374,6 +390,8 @@ PERF_TEST_P(adaptiveBilateralFixture, adaptiveBilateral, const double sigmaspace = 10.0; Size ksize(9, 9); + checkDeviceMaxMemoryAllocSize(srcSize, type); + Mat src(srcSize, type), dst(srcSize, type); declare.in(src, WARMUP_RNG).out(dst); diff --git a/modules/ocl/perf/perf_imgwarp.cpp b/modules/ocl/perf/perf_imgwarp.cpp index 0aff45e9a..01d4fd275 100644 --- a/modules/ocl/perf/perf_imgwarp.cpp +++ b/modules/ocl/perf/perf_imgwarp.cpp @@ -154,9 +154,12 @@ PERF_TEST_P(resizeFixture, resize, const Size srcSize = get<0>(params); const int type = get<1>(params), interType = get<2>(params); double scale = get<3>(params); + const Size dstSize(cvRound(srcSize.width * scale), cvRound(srcSize.height * scale)); + + checkDeviceMaxMemoryAllocSize(srcSize, type); + checkDeviceMaxMemoryAllocSize(dstSize, type); Mat src(srcSize, type), dst; - const Size dstSize(cvRound(srcSize.width * scale), cvRound(srcSize.height * scale)); dst.create(dstSize, type); declare.in(src, WARMUP_RNG).out(dst); if (interType == INTER_LINEAR && type == CV_8UC4 && OCL_SIZE_4000 == srcSize) diff --git a/modules/ocl/perf/perf_matrix_operation.cpp b/modules/ocl/perf/perf_matrix_operation.cpp index f2baa7ffc..5ca322e22 100644 --- a/modules/ocl/perf/perf_matrix_operation.cpp +++ b/modules/ocl/perf/perf_matrix_operation.cpp @@ -63,6 +63,10 @@ PERF_TEST_P(ConvertToFixture, ConvertTo, Mat src(srcSize, type), dst; const int dstType = CV_MAKE_TYPE(CV_32F, src.channels()); + + checkDeviceMaxMemoryAllocSize(srcSize, type); + checkDeviceMaxMemoryAllocSize(srcSize, dstType); + dst.create(srcSize, dstType); declare.in(src, WARMUP_RNG).out(dst); diff --git a/modules/ocl/perf/perf_precomp.hpp b/modules/ocl/perf/perf_precomp.hpp index 2d9639a85..a8663df99 100644 --- a/modules/ocl/perf/perf_precomp.hpp +++ b/modules/ocl/perf/perf_precomp.hpp @@ -103,8 +103,30 @@ using namespace cv; CV_TEST_FAIL_NO_IMPL(); #endif -#define OCL_TEST_CYCLE_N(n) for(declare.iterations(n); startTimer(), next(); ocl::finish(), stopTimer()) -#define OCL_TEST_CYCLE() for(; startTimer(), next(); ocl::finish(), stopTimer()) -#define OCL_TEST_CYCLE_MULTIRUN(runsNum) for(declare.runs(runsNum); startTimer(), next(); stopTimer()) for(int r = 0; r < runsNum; ocl::finish(), ++r) +#define OCL_TEST_CYCLE_N(n) for(declare.iterations(n); startTimer(), next(); cv::ocl::finish(), stopTimer()) +#define OCL_TEST_CYCLE() for(; startTimer(), next(); cv::ocl::finish(), stopTimer()) +#define OCL_TEST_CYCLE_MULTIRUN(runsNum) for(declare.runs(runsNum); startTimer(), next(); stopTimer()) for(int r = 0; r < runsNum; cv::ocl::finish(), ++r) + +namespace cvtest { +namespace ocl { +inline void checkDeviceMaxMemoryAllocSize(const Size& size, int type, int factor = 1) +{ + assert(factor > 0); + if (!(IMPL_OCL == perf::TestBase::getSelectedImpl())) + return; // OpenCL devices are not used + int cn = CV_MAT_CN(type); + int cn_ocl = cn == 3 ? 4 : cn; + int type_ocl = CV_MAKE_TYPE(CV_MAT_DEPTH(type), cn_ocl); + size_t memSize = size.area() * CV_ELEM_SIZE(type_ocl); + const cv::ocl::DeviceInfo& devInfo = cv::ocl::Context::getContext()->getDeviceInfo(); + if (memSize * factor >= devInfo.maxMemAllocSize) + { + throw perf::TestBase::PerfSkipTestException(); + } +} +} // namespace cvtest::ocl +} // namespace cvtest + +using namespace cvtest::ocl; #endif diff --git a/modules/ocl/perf/perf_pyramid.cpp b/modules/ocl/perf/perf_pyramid.cpp index c799853db..820dd6062 100644 --- a/modules/ocl/perf/perf_pyramid.cpp +++ b/modules/ocl/perf/perf_pyramid.cpp @@ -60,9 +60,12 @@ PERF_TEST_P(pyrDownFixture, pyrDown, const Size_MatType_t params = GetParam(); const Size srcSize = get<0>(params); const int type = get<1>(params); + Size dstSize((srcSize.height + 1) >> 1, (srcSize.width + 1) >> 1); + + checkDeviceMaxMemoryAllocSize(srcSize, type); + checkDeviceMaxMemoryAllocSize(dstSize, type); Mat src(srcSize, type), dst; - Size dstSize((srcSize.height + 1) >> 1, (srcSize.width + 1) >> 1); dst.create(dstSize, type); declare.in(src, WARMUP_RNG).out(dst); @@ -97,9 +100,12 @@ PERF_TEST_P(pyrUpFixture, pyrUp, const Size_MatType_t params = GetParam(); const Size srcSize = get<0>(params); const int type = get<1>(params); + Size dstSize(srcSize.height << 1, srcSize.width << 1); + + checkDeviceMaxMemoryAllocSize(srcSize, type); + checkDeviceMaxMemoryAllocSize(dstSize, type); Mat src(srcSize, type), dst; - Size dstSize(srcSize.height << 1, srcSize.width << 1); dst.create(dstSize, type); declare.in(src, WARMUP_RNG).out(dst); diff --git a/modules/ocl/perf/perf_split_merge.cpp b/modules/ocl/perf/perf_split_merge.cpp index f2f7c4115..ecfc49e33 100644 --- a/modules/ocl/perf/perf_split_merge.cpp +++ b/modules/ocl/perf/perf_split_merge.cpp @@ -60,8 +60,10 @@ PERF_TEST_P(MergeFixture, Merge, const Size_MatType_t params = GetParam(); const Size srcSize = get<0>(params); const int depth = get<1>(params), channels = 3; - const int dstType = CV_MAKE_TYPE(depth, channels); + + checkDeviceMaxMemoryAllocSize(srcSize, dstType); + Mat dst(srcSize, dstType); vector src(channels); for (vector::iterator i = src.begin(), end = src.end(); i != end; ++i) @@ -105,8 +107,11 @@ PERF_TEST_P(SplitFixture, Split, const Size_MatType_t params = GetParam(); const Size srcSize = get<0>(params); const int depth = get<1>(params), channels = 3; + const int type = CV_MAKE_TYPE(depth, channels); - Mat src(srcSize, CV_MAKE_TYPE(depth, channels)); + checkDeviceMaxMemoryAllocSize(srcSize, type); + + Mat src(srcSize, type); declare.in(src, WARMUP_RNG); if (RUN_OCL_IMPL)