Merge pull request #1909 from alalek:ocl_tapi_perf

modules/core/perf/opencl/perf_arithm.cpp

@@ -0,0 +1,75 @@
+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                           License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2010-2013, Advanced Micro Devices, Inc., all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of the copyright holders may not be used to endorse or promote products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors "as is" and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are disclaimed.
+// In no event shall the OpenCV Foundation or contributors be liable for any direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#include "perf_precomp.hpp"
+#include "opencv2/ts/ocl_perf.hpp"
+
+#ifdef HAVE_OPENCL
+
+namespace cvtest {
+namespace ocl {
+
+///////////// Add ////////////////////////
+
+typedef Size_MatType AddFixture;
+
+OCL_PERF_TEST_P(AddFixture, Add,
+            ::testing::Combine(OCL_TEST_SIZES,
+                               OCL_TEST_TYPES))
+{
+    const Size srcSize = GET_PARAM(0);
+    const int type = GET_PARAM(1);
+
+    checkDeviceMaxMemoryAllocSize(srcSize, type);
+
+    UMat src1(srcSize, type), src2(srcSize, type), dst(srcSize, type);
+    randu(src1);
+    randu(src2);
+    declare.in(src1, src2).out(dst);
+
+    OCL_TEST_CYCLE() cv::add(src1, src2, dst);
+
+    SANITY_CHECK(dst);
+}
+
+} } // namespace cvtest::ocl
+
+#endif // HAVE_OPENCL

modules/imgproc/test/ocl/test_color.cpp

@@ -44,7 +44,6 @@
 //M*/
 
 #include "test_precomp.hpp"
-#include "cvconfig.h"
 #include "opencv2/ts/ocl_test.hpp"
 
 #ifdef HAVE_OPENCL

modules/imgproc/test/ocl/test_warp.cpp

@@ -52,7 +52,6 @@
 //M*/
 
 #include "test_precomp.hpp"
-#include "cvconfig.h"
 #include "opencv2/ts/ocl_test.hpp"
 
 #ifdef HAVE_OPENCL

modules/ocl/perf/main.cpp

@@ -70,7 +70,7 @@ static const char * impls[] =
 
 int main(int argc, char ** argv)
 {
-    ::perf::TestBase::setPerformanceStrategy(::perf::PERF_STRATEGY_SIMPLE);
+    ::perf::TestBase::setModulePerformanceStrategy(::perf::PERF_STRATEGY_SIMPLE);
 
     CV_PERF_TEST_MAIN_INTERNALS(ocl, impls, dumpOpenCLDevice())
 }

modules/ts/include/opencv2/ts/ocl_perf.hpp

@@ -0,0 +1,111 @@
+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                           License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2010-2013, Advanced Micro Devices, Inc., all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of the copyright holders may not be used to endorse or promote products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors "as is" and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are disclaimed.
+// In no event shall the OpenCV Foundation or contributors be liable for any direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#ifndef __OPENCV_TS_OCL_PERF_HPP__
+#define __OPENCV_TS_OCL_PERF_HPP__
+
+#include "ocl_test.hpp"
+#include "ts_perf.hpp"
+
+#ifdef HAVE_OPENCL
+
+namespace cvtest {
+namespace ocl {
+
+using namespace perf;
+
+#define OCL_PERF_STRATEGY PERF_STRATEGY_SIMPLE
+
+#define OCL_PERF_TEST_P(fixture, name, params) SIMPLE_PERF_TEST_P(fixture, name, params)
+
+#define SIMPLE_PERF_TEST_P(fixture, name, params)\
+    class OCL##_##fixture##_##name : public fixture {\
+    public:\
+        OCL##_##fixture##_##name() {}\
+    protected:\
+        virtual void PerfTestBody();\
+    };\
+    TEST_P(OCL##_##fixture##_##name, name){ declare.strategy(OCL_PERF_STRATEGY); RunPerfTestBody(); }\
+    INSTANTIATE_TEST_CASE_P(/*none*/, OCL##_##fixture##_##name, params);\
+    void OCL##_##fixture##_##name::PerfTestBody()
+
+
+#define OCL_SIZE_1000 Size(1000, 1000)
+#define OCL_SIZE_2000 Size(2000, 2000)
+#define OCL_SIZE_4000 Size(4000, 4000)
+
+#define OCL_TEST_SIZES ::testing::Values(OCL_SIZE_1000, OCL_SIZE_2000, OCL_SIZE_4000)
+#define OCL_TEST_TYPES ::testing::Values(CV_8UC1, CV_32FC1, CV_8UC4, CV_32FC4)
+
+#define OCL_PERF_ENUM ::testing::Values
+
+// TODO Replace finish call to dstUMat.wait()
+#define OCL_TEST_CYCLE() \
+    for (; startTimer(), next(); cvtest::ocl::perf::safeFinish(), stopTimer())
+
+#define OCL_TEST_CYCLE_MULTIRUN(runsNum) \
+    for (declare.runs(runsNum); startTimer(), next(); cvtest::ocl::perf::safeFinish(), stopTimer()) \
+        for (int r = 0; r < runsNum; cvtest::ocl::perf::safeFinish(), ++r)
+
+namespace perf {
+
+// Check for current device limitation
+CV_EXPORTS void checkDeviceMaxMemoryAllocSize(const Size& size, int type, int factor = 1);
+
+// Initialize Mat with random numbers. Range is depends on the data type.
+// TODO Parameter type is actually OutputArray
+CV_EXPORTS void randu(InputOutputArray dst);
+
+inline void safeFinish()
+{
+    if (cv::ocl::useOpenCL())
+        cv::ocl::finish2();
+}
+
+} // namespace perf
+using namespace perf;
+
+} // namespace cvtest::ocl
+} // namespace cvtest
+
+#endif // HAVE_OPENCL
+
+#endif // __OPENCV_TS_OCL_PERF_HPP__

modules/ts/include/opencv2/ts/ocl_test.hpp

@@ -42,11 +42,11 @@
 #ifndef __OPENCV_TS_OCL_TEST_HPP__
 #define __OPENCV_TS_OCL_TEST_HPP__
 
-#ifdef HAVE_OPENCL
+#include "cvconfig.h" // to get definition of HAVE_OPENCL
-
-#include "cvconfig.h"
 #include "opencv2/opencv_modules.hpp"
 
+#ifdef HAVE_OPENCL
+
 #include "opencv2/ts.hpp"
 
 #include "opencv2/highgui.hpp"
@@ -123,7 +123,7 @@ using perf::MatType;
 
 #define OCL_RNG_SEED 123456
 
-struct TestUtils
+struct CV_EXPORTS TestUtils
 {
     cv::RNG rng;
 
@@ -270,7 +270,7 @@ struct TestUtils
 #define UMAT_UPLOAD_OUTPUT_PARAMETER(name) UMAT_UPLOAD_INPUT_PARAMETER(name)
 
 template <typename T>
-struct TSTestWithParam : public TestUtils, public ::testing::TestWithParam<T>
+struct CV_EXPORTS TSTestWithParam : public TestUtils, public ::testing::TestWithParam<T>
 {
 
 };

modules/ts/include/opencv2/ts/ts_perf.hpp

@@ -251,6 +251,7 @@ typedef struct CV_EXPORTS performance_metrics
 \*****************************************************************************************/
 enum PERF_STRATEGY
 {
+    PERF_STRATEGY_DEFAULT = -1,
     PERF_STRATEGY_BASE = 0,
     PERF_STRATEGY_SIMPLE = 1,
 };
@@ -271,8 +272,8 @@ public:
     static std::string getDataPath(const std::string& relativePath);
     static std::string getSelectedImpl();
 
-    static enum PERF_STRATEGY getPerformanceStrategy();
+    static enum PERF_STRATEGY getCurrentModulePerformanceStrategy();
-    static enum PERF_STRATEGY setPerformanceStrategy(enum PERF_STRATEGY strategy);
+    static enum PERF_STRATEGY setModulePerformanceStrategy(enum PERF_STRATEGY strategy);
 
     class PerfSkipTestException: public cv::Exception {};
 
@@ -286,7 +287,7 @@ protected:
     void stopTimer();
     bool next();
 
-    //_declareHelper declare;
+    PERF_STRATEGY getCurrentPerformanceStrategy() const;
 
     enum WarmUpType
     {
@@ -300,6 +301,7 @@ protected:
     static void warmup(cv::InputOutputArray a, WarmUpType wtype = WARMUP_READ);
 
     performance_metrics& calcMetrics();
+
     void RunPerfTestBody();
 private:
     typedef std::vector<std::pair<int, cv::Size> > SizeVector;
@@ -310,6 +312,8 @@ private:
     unsigned int getTotalInputSize() const;
     unsigned int getTotalOutputSize() const;
 
+    enum PERF_STRATEGY testStrategy;
+
     TimeVector times;
     int64 lastTime;
     int64 totalTime;
@@ -349,6 +353,8 @@ private:
         _declareHelper& time(double timeLimitSecs);
         _declareHelper& tbb_threads(int n = -1);
         _declareHelper& runs(unsigned int runsNumber);
+
+        _declareHelper& strategy(enum PERF_STRATEGY s);
     private:
         TestBase* test;
         _declareHelper(TestBase* t);

modules/ts/src/ocl_perf.cpp

@@ -0,0 +1,97 @@
+/*M///////////////////////////////////////////////////////////////////////////////////////
+//
+//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
+//
+//  By downloading, copying, installing or using the software you agree to this license.
+//  If you do not agree to this license, do not download, install,
+//  copy or use the software.
+//
+//
+//                           License Agreement
+//                For Open Source Computer Vision Library
+//
+// Copyright (C) 2010-2013, Advanced Micro Devices, Inc., all rights reserved.
+// Third party copyrights are property of their respective owners.
+//
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+//
+//   * Redistribution's of source code must retain the above copyright notice,
+//     this list of conditions and the following disclaimer.
+//
+//   * Redistribution's in binary form must reproduce the above copyright notice,
+//     this list of conditions and the following disclaimer in the documentation
+//     and/or other materials provided with the distribution.
+//
+//   * The name of the copyright holders may not be used to endorse or promote products
+//     derived from this software without specific prior written permission.
+//
+// This software is provided by the copyright holders and contributors "as is" and
+// any express or implied warranties, including, but not limited to, the implied
+// warranties of merchantability and fitness for a particular purpose are disclaimed.
+// In no event shall the OpenCV Foundation or contributors be liable for any direct,
+// indirect, incidental, special, exemplary, or consequential damages
+// (including, but not limited to, procurement of substitute goods or services;
+// loss of use, data, or profits; or business interruption) however caused
+// and on any theory of liability, whether in contract, strict liability,
+// or tort (including negligence or otherwise) arising in any way out of
+// the use of this software, even if advised of the possibility of such damage.
+//
+//M*/
+
+#include "precomp.hpp"
+
+#include "opencv2/ts/ocl_perf.hpp"
+
+#ifdef HAVE_OPENCL
+
+namespace cvtest {
+namespace ocl {
+
+namespace perf {
+
+void checkDeviceMaxMemoryAllocSize(const Size& size, int type, int factor)
+{
+    assert(factor > 0);
+    if (!cv::ocl::useOpenCL())
+        return;
+    int cn = CV_MAT_CN(type);
+    int cn_ocl = cn == 3 ? 4 : cn;
+    int type_ocl = CV_MAKE_TYPE(CV_MAT_DEPTH(type), cn_ocl);
+    size_t memSize = size.area() * CV_ELEM_SIZE(type_ocl);
+    const cv::ocl::Device& dev = cv::ocl::Device::getDefault();
+    if (memSize * factor >= dev.maxMemAllocSize())
+    {
+        throw ::perf::TestBase::PerfSkipTestException();
+    }
+}
+
+void randu(InputOutputArray dst)
+{
+    if (dst.depth() == CV_8U)
+    {
+        cv::randu(dst, 0, 256);
+    }
+    else if (dst.depth() == CV_8S)
+    {
+        cv::randu(dst, -128, 128);
+    }
+    else if (dst.depth() == CV_16U)
+    {
+        cv::randu(dst, 0, 1024);
+    }
+    else if (dst.depth() == CV_32F || dst.depth() == CV_64F)
+    {
+        cv::randu(dst, -1.0, 1.0);
+    }
+    else // (dst.depth() == CV_16S || dst.depth() == CV_32S)
+    {
+        cv::randu(dst, -4096, 4096);
+    }
+}
+
+} // namespace perf
+
+}} // namespace cvtest::ocl
+
+#endif // HAVE_OPENCL

modules/ts/src/ts_perf.cpp

@@ -19,7 +19,8 @@ static std::vector<std::string> available_impls;
 
 static std::string  param_impl;
 
-static enum PERF_STRATEGY param_strategy = PERF_STRATEGY_BASE;
+static enum PERF_STRATEGY strategyForce = PERF_STRATEGY_DEFAULT;
+static enum PERF_STRATEGY strategyModule = PERF_STRATEGY_BASE;
 
 static double       param_max_outliers;
 static double       param_max_deviation;
@@ -684,11 +685,11 @@ void TestBase::Init(const std::vector<std::string> & availableImpls,
     }
     else if (perf_strategy == "base")
     {
-        param_strategy = PERF_STRATEGY_BASE;
+        strategyForce = PERF_STRATEGY_BASE;
     }
     else if (perf_strategy == "simple")
     {
-        param_strategy = PERF_STRATEGY_SIMPLE;
+        strategyForce = PERF_STRATEGY_SIMPLE;
     }
     else
     {
@@ -788,16 +789,16 @@ std::string TestBase::getSelectedImpl()
     return param_impl;
 }
 
-enum PERF_STRATEGY TestBase::getPerformanceStrategy()
+enum PERF_STRATEGY TestBase::setModulePerformanceStrategy(enum PERF_STRATEGY strategy)
 {
-    return param_strategy;
+    enum PERF_STRATEGY ret = strategyModule;
+    strategyModule = strategy;
+    return ret;
 }
 
-enum PERF_STRATEGY TestBase::setPerformanceStrategy(enum PERF_STRATEGY strategy)
+enum PERF_STRATEGY TestBase::getCurrentModulePerformanceStrategy()
 {
-    enum PERF_STRATEGY ret = param_strategy;
+    return strategyForce == PERF_STRATEGY_DEFAULT ? strategyModule : strategyForce;
-    param_strategy = strategy;
-    return ret;
 }
 
 
@@ -830,7 +831,7 @@ int64 TestBase::_calibrate()
     _helper h;
     h.PerfTestBody();
     double compensation = h.getMetrics().min;
-    if (param_strategy == PERF_STRATEGY_SIMPLE)
+    if (getCurrentModulePerformanceStrategy() == PERF_STRATEGY_SIMPLE)
     {
         CV_Assert(compensation < 0.01 * cv::getTickFrequency());
         compensation = 0.0f; // simple strategy doesn't require any compensation
@@ -843,7 +844,7 @@ int64 TestBase::_calibrate()
 # pragma warning(push)
 # pragma warning(disable:4355)  // 'this' : used in base member initializer list
 #endif
-TestBase::TestBase(): declare(this)
+TestBase::TestBase(): testStrategy(PERF_STRATEGY_DEFAULT), declare(this)
 {
 }
 #ifdef _MSC_VER
@@ -864,9 +865,18 @@ void TestBase::declareArray(SizeVector& sizes, cv::InputOutputArray a, WarmUpTyp
 
 void TestBase::warmup(cv::InputOutputArray a, WarmUpType wtype)
 {
-    if (a.empty()) return;
+    if (a.empty())
-    if (a.kind() != cv::_InputArray::STD_VECTOR_MAT && a.kind() != cv::_InputArray::STD_VECTOR_VECTOR)
+    {
+        return;
+    }
+    else if (a.isUMat())
+    {
+        return; // TODO current warmup_impl is not useful for GPU-based data
+    }
+    else if (a.kind() != cv::_InputArray::STD_VECTOR_MAT && a.kind() != cv::_InputArray::STD_VECTOR_VECTOR)
+    {
         warmup_impl(a.getMat(), wtype);
+    }
     else
     {
         size_t total = a.total();
@@ -896,6 +906,14 @@ cv::Size TestBase::getSize(cv::InputArray a)
     return cv::Size();
 }
 
+PERF_STRATEGY TestBase::getCurrentPerformanceStrategy() const
+{
+    if (strategyForce == PERF_STRATEGY_DEFAULT)
+        return (testStrategy == PERF_STRATEGY_DEFAULT) ? strategyModule : testStrategy;
+    else
+        return strategyForce;
+}
+
 bool TestBase::next()
 {
     static int64 lastActivityPrintTime = 0;
@@ -924,13 +942,13 @@ bool TestBase::next()
             break;
         }
 
-        if (param_strategy == PERF_STRATEGY_BASE)
+        if (getCurrentPerformanceStrategy() == PERF_STRATEGY_BASE)
         {
             has_next = currentIter < nIters && totalTime < timeLimit;
         }
         else
         {
-            assert(param_strategy == PERF_STRATEGY_SIMPLE);
+            assert(getCurrentPerformanceStrategy() == PERF_STRATEGY_SIMPLE);
             if (totalTime - lastActivityPrintTime >= cv::getTickFrequency() * 10)
             {
                 std::cout << '.' << std::endl;
@@ -1053,7 +1071,7 @@ performance_metrics& TestBase::calcMetrics()
     TimeVector::const_iterator start = times.begin();
     TimeVector::const_iterator end = times.end();
 
-    if (param_strategy == PERF_STRATEGY_BASE)
+    if (getCurrentPerformanceStrategy() == PERF_STRATEGY_BASE)
     {
         //estimate mean and stddev for log(time)
         double gmean = 0;
@@ -1084,7 +1102,7 @@ performance_metrics& TestBase::calcMetrics()
             ++end, --metrics.outliers;
         }
     }
-    else if (param_strategy == PERF_STRATEGY_SIMPLE)
+    else if (getCurrentPerformanceStrategy() == PERF_STRATEGY_SIMPLE)
     {
         metrics.outliers = static_cast<int>(times.size() * param_max_outliers / 100);
         for (unsigned int i = 0; i < metrics.outliers; i++)
@@ -1143,7 +1161,7 @@ void TestBase::validateMetrics()
     ASSERT_GE(m.samples, 1u)
       << "  No time measurements was performed.\nstartTimer() and stopTimer() commands are required for performance tests.";
 
-    if (param_strategy == PERF_STRATEGY_BASE)
+    if (getCurrentPerformanceStrategy() == PERF_STRATEGY_BASE)
     {
         EXPECT_GE(m.samples, param_min_samples)
           << "  Only a few samples are collected.\nPlease increase number of iterations or/and time limit to get reliable performance measurements.";
@@ -1157,7 +1175,7 @@ void TestBase::validateMetrics()
         EXPECT_LE(m.outliers, std::max((unsigned int)cvCeil(m.samples * param_max_outliers / 100.), 1u))
           << "  Test results are not reliable (too many outliers).";
     }
-    else if (param_strategy == PERF_STRATEGY_SIMPLE)
+    else if (getCurrentPerformanceStrategy() == PERF_STRATEGY_SIMPLE)
     {
         double mean = metrics.mean * 1000.0f / metrics.frequency;
         double stddev = metrics.stddev * 1000.0f / metrics.frequency;
@@ -1479,6 +1497,12 @@ TestBase::_declareHelper& TestBase::_declareHelper::out(cv::InputOutputArray a1,
     return *this;
 }
 
+TestBase::_declareHelper& TestBase::_declareHelper::strategy(enum PERF_STRATEGY s)
+{
+    test->testStrategy = s;
+    return *this;
+}
+
 TestBase::_declareHelper::_declareHelper(TestBase* t) : test(t)
 {
 }