SVMSGD class added

This commit is contained in:
joao.faro 2015-09-02 17:14:40 +01:00 committed by Marina Noskova
parent b529b37d06
commit a2f0963d66
3 changed files with 330 additions and 10 deletions

View File

@ -415,16 +415,6 @@
pages = {2548--2555},
organization = {IEEE}
}
@ARTICLE{Louhichi07,
author = {Louhichi, H. and Fournel, T. and Lavest, J. M. and Ben Aissia, H.},
title = {Self-calibration of Scheimpflug cameras: an easy protocol},
year = {2007},
pages = {26162622},
journal = {Meas. Sci. Technol.},
volume = {18},
number = {8},
publisher = {IOP Publishing Ltd}
}
@ARTICLE{LibSVM,
author = {Chang, Chih-Chung and Lin, Chih-Jen},
title = {LIBSVM: a library for support vector machines},
@ -874,3 +864,11 @@
year={2007},
organization={IEEE}
}
@incollection{bottou2010large,
title={Large-scale machine learning with stochastic gradient descent},
author={Bottou, L{\'e}on},
booktitle={Proceedings of COMPSTAT'2010},
pages={177--186},
year={2010},
publisher={Springer}
}

View File

@ -1513,6 +1513,127 @@ CV_EXPORTS void randMVNormal( InputArray mean, InputArray cov, int nsamples, Out
CV_EXPORTS void createConcentricSpheresTestSet( int nsamples, int nfeatures, int nclasses,
OutputArray samples, OutputArray responses);
/****************************************************************************************\
* Stochastic Gradient Descent SVM Classifier *
\****************************************************************************************/
/*!
@brief Stochastic Gradient Descent SVM classifier
SVMSGD provides a fast and easy-to-use implementation of the SVM classifier using the Stochastic Gradient Descent approach, as presented in @cite bottou2010large.
The gradient descent show amazing performance for large-scale problems, reducing the computing time. This allows a fast and reliable online update of the classifier for each new feature which
is fundamental when dealing with variations of data over time (like weather and illumination changes in videosurveillance, for example).
First, create the SVMSGD object. To enable the online update, a value for updateFrequency should be defined.
Then the SVM model can be trained using the train features and the correspondent labels.
After that, the label of a new feature vector can be predicted using the predict function. If the updateFrequency was defined in the constructor, the predict function will update the weights automatically.
@code
// Initialize object
SVMSGD SvmSgd;
// Train the Stochastic Gradient Descent SVM
SvmSgd.train(trainFeatures, labels);
// Predict label for the new feature vector (1xM)
predictedLabel = SvmSgd.predict(newFeatureVector);
@endcode
*/
class CV_EXPORTS_W SVMSGD {
public:
/** @brief SGDSVM constructor.
@param lambda regularization
@param learnRate learning rate
@param nIterations number of training iterations
*/
SVMSGD(float lambda = 0.000001, float learnRate = 2, uint nIterations = 100000);
/** @brief SGDSVM constructor.
@param updateFrequency online update frequency
@param learnRateDecay learn rate decay over time: learnRate = learnRate * learnDecay
@param lambda regularization
@param learnRate learning rate
@param nIterations number of training iterations
*/
SVMSGD(uint updateFrequency, float learnRateDecay = 1, float lambda = 0.000001, float learnRate = 2, uint nIterations = 100000);
virtual ~SVMSGD();
virtual SVMSGD* clone() const;
/** @brief Train the SGDSVM classifier.
The function trains the SGDSVM classifier using the train features and the correspondent labels (-1 or 1).
@param trainFeatures features used for training. Each row is a new sample.
@param labels mat (size Nx1 with N = number of features) with the label of each training feature.
*/
virtual void train(cv::Mat trainFeatures, cv::Mat labels);
/** @brief Predict the label of a new feature vector.
The function predicts and returns the label of a new feature vector, using the previously trained SVM model.
@param newFeature new feature vector used for prediction
*/
virtual float predict(cv::Mat newFeature);
/** @brief Returns the weights of the trained model.
*/
virtual std::vector<float> getWeights(){ return _weights; };
/** @brief Sets the weights of the trained model.
@param weights weights used to predict the label of a new feature vector.
*/
virtual void setWeights(std::vector<float> weights){ _weights = weights; };
private:
void updateWeights();
void generateRandomIndex();
float calcInnerProduct(float *rowDataPointer);
void updateWeights(float innerProduct, float *rowDataPointer, int label);
// Vector with SVM weights
std::vector<float> _weights;
// Random index generation
long long int _randomNumber;
unsigned int _randomIndex;
// Number of features and samples
unsigned int _nFeatures;
unsigned int _nTrainSamples;
// Parameters for learning
float _lambda; //regularization
float _learnRate; //learning rate
unsigned int _nIterations; //number of training iterations
// Vars to control the features slider matrix
bool _onlineUpdate;
bool _initPredict;
uint _slidingWindowSize;
uint _predictSlidingWindowSize;
float* _labelSlider;
float _learnRateDecay;
// Mat with features slider and correspondent counter
unsigned int _sliderCounter;
cv::Mat _featuresSlider;
};
//! @} ml
}

201
modules/ml/src/svmsgd.cpp Normal file
View File

@ -0,0 +1,201 @@
/*M///////////////////////////////////////////////////////////////////////////////////////
//
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
// By downloading, copying, installing or using the software you agree to this license.
// If you do not agree to this license, do not download, install,
// copy or use the software.
//
//
// License Agreement
// For Open Source Computer Vision Library
//
// Copyright (C) 2000, Intel Corporation, all rights reserved.
// Copyright (C) 2014, Itseez Inc, all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistribution's of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// * Redistribution's in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// * The name of the copyright holders may not be used to endorse or promote products
// derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/
#include "precomp.hpp"
/****************************************************************************************\
* Stochastic Gradient Descent SVM Classifier *
\****************************************************************************************/
namespace cv {
namespace ml {
SVMSGD::SVMSGD(float lambda, float learnRate, uint nIterations){
// Initialize with random seed
_randomNumber = 1;
// Initialize constants
_slidingWindowSize = 0;
_nFeatures = 0;
_predictSlidingWindowSize = 1;
// Initialize sliderCounter at index 0
_sliderCounter = 0;
// Parameters for learning
_lambda = lambda; // regularization
_learnRate = learnRate; // learning rate (ideally should be large at beginning and decay each iteration)
_nIterations = nIterations; // number of training iterations
// True only in the first predict iteration
_initPredict = true;
// Online update flag
_onlineUpdate = false;
}
SVMSGD::SVMSGD(uint updateFrequency, float learnRateDecay, float lambda, float learnRate, uint nIterations){
// Initialize with random seed
_randomNumber = 1;
// Initialize constants
_slidingWindowSize = 0;
_nFeatures = 0;
_predictSlidingWindowSize = updateFrequency;
// Initialize sliderCounter at index 0
_sliderCounter = 0;
// Parameters for learning
_lambda = lambda; // regularization
_learnRate = learnRate; // learning rate (ideally should be large at beginning and decay each iteration)
_nIterations = nIterations; // number of training iterations
// True only in the first predict iteration
_initPredict = true;
// Online update flag
_onlineUpdate = true;
// Learn rate decay: _learnRate = _learnRate * _learnDecay
_learnRateDecay = learnRateDecay;
}
SVMSGD::~SVMSGD(){
}
SVMSGD* SVMSGD::clone() const{
return new SVMSGD(*this);
}
void SVMSGD::train(cv::Mat trainFeatures, cv::Mat labels){
// Initialize _nFeatures
_slidingWindowSize = trainFeatures.rows;
_nFeatures = trainFeatures.cols;
float innerProduct;
// Initialize weights vector with zeros
if (_weights.size()==0){
_weights.reserve(_nFeatures);
for (uint feat = 0; feat < _nFeatures; ++feat){
_weights.push_back(0.0);
}
}
// Stochastic gradient descent SVM
for (uint iter = 0; iter < _nIterations; ++iter){
generateRandomIndex();
innerProduct = calcInnerProduct(trainFeatures.ptr<float>(_randomIndex));
int label = (labels.at<int>(_randomIndex,0) > 0) ? 1 : -1; // ensure that labels are -1 or 1
updateWeights(innerProduct, trainFeatures.ptr<float>(_randomIndex), label );
}
}
float SVMSGD::predict(cv::Mat newFeature){
float innerProduct;
if (_initPredict){
_nFeatures = newFeature.cols;
_slidingWindowSize = _predictSlidingWindowSize;
_featuresSlider = cv::Mat::zeros(_slidingWindowSize, _nFeatures, CV_32F);
_initPredict = false;
_labelSlider = new float[_predictSlidingWindowSize]();
_learnRate = _learnRate * _learnRateDecay;
}
innerProduct = calcInnerProduct(newFeature.ptr<float>(0));
// Resultant label (-1 or 1)
int label = (innerProduct>=0) ? 1 : -1;
if (_onlineUpdate){
// Update the featuresSlider with newFeature and _labelSlider with label
newFeature.row(0).copyTo(_featuresSlider.row(_sliderCounter));
_labelSlider[_sliderCounter] = float(label);
// Update weights with a random index
if (_sliderCounter == _slidingWindowSize-1){
generateRandomIndex();
updateWeights(innerProduct, _featuresSlider.ptr<float>(_randomIndex), int(_labelSlider[_randomIndex]) );
}
// _sliderCounter++ if < _slidingWindowSize
_sliderCounter = (_sliderCounter == _slidingWindowSize-1) ? 0 : (_sliderCounter+1);
}
return float(label);
}
void SVMSGD::generateRandomIndex(){
// Choose random sample, using Mikolov's fast almost-uniform random number
_randomNumber = _randomNumber * (unsigned long long) 25214903917 + 11;
_randomIndex = uint(_randomNumber % (unsigned long long) _slidingWindowSize);
}
float SVMSGD::calcInnerProduct(float *rowDataPointer){
float innerProduct = 0;
for (uint feat = 0; feat < _nFeatures; ++feat){
innerProduct += _weights[feat] * rowDataPointer[feat];
}
return innerProduct;
}
void SVMSGD::updateWeights(float innerProduct, float *rowDataPointer, int label){
if (label * innerProduct > 1) {
// Not a support vector, only apply weight decay
for (uint feat = 0; feat < _nFeatures; feat++) {
_weights[feat] -= _learnRate * _lambda * _weights[feat];
}
} else {
// It's a support vector, add it to the weights
for (uint feat = 0; feat < _nFeatures; feat++) {
_weights[feat] -= _learnRate * (_lambda * _weights[feat] - label * rowDataPointer[feat]);
}
}
}
}
}