1793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler/*M/////////////////////////////////////////////////////////////////////////////////////// 2793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// 3793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. 4793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// 5793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// By downloading, copying, installing or using the software you agree to this license. 6793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// If you do not agree to this license, do not download, install, 7793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// copy or use the software. 8793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// 9793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// 10793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// Intel License Agreement 11793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// For Open Source Computer Vision Library 12793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// 13793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// Copyright (C) 2000, Intel Corporation, all rights reserved. 14793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// Third party copyrights are property of their respective owners. 15793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// 16793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// Redistribution and use in source and binary forms, with or without modification, 17793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// are permitted provided that the following conditions are met: 18793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// 19793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// * Redistribution's of source code must retain the above copyright notice, 20793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// this list of conditions and the following disclaimer. 21793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// 22793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// * Redistribution's in binary form must reproduce the above copyright notice, 23793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// this list of conditions and the following disclaimer in the documentation 24793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// and/or other materials provided with the distribution. 25793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// 26793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// * The name of Intel Corporation may not be used to endorse or promote products 27793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// derived from this software without specific prior written permission. 28793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// 29793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// This software is provided by the copyright holders and contributors "as is" and 30793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// any express or implied warranties, including, but not limited to, the implied 31793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// warranties of merchantability and fitness for a particular purpose are disclaimed. 32793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// In no event shall the Intel Corporation or contributors be liable for any direct, 33793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// indirect, incidental, special, exemplary, or consequential damages 34793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// (including, but not limited to, procurement of substitute goods or services; 35793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// loss of use, data, or profits; or business interruption) however caused 36793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// and on any theory of liability, whether in contract, strict liability, 37793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// or tort (including negligence or otherwise) arising in any way out of 38793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// the use of this software, even if advised of the possibility of such damage. 39793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler// 40793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler//M*/ 41793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler 42793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler#include "test_precomp.hpp" 43793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler 44793ee12c6df9cad3806238d32528c49a3ff9331dNoah Preslerusing namespace cv; 45793ee12c6df9cad3806238d32528c49a3ff9331dNoah Preslerusing namespace std; 46793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler 47793ee12c6df9cad3806238d32528c49a3ff9331dNoah PreslerCV_AMLTest::CV_AMLTest( const char* _modelName ) : CV_MLBaseTest( _modelName ) 48793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler{ 49793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler validationFN = "avalidation.xml"; 50793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler} 51793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler 52793ee12c6df9cad3806238d32528c49a3ff9331dNoah Preslerint CV_AMLTest::run_test_case( int testCaseIdx ) 53793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler{ 54793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler int code = cvtest::TS::OK; 55793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler code = prepare_test_case( testCaseIdx ); 56793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler 57793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler if (code == cvtest::TS::OK) 58793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler { 59793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler //#define GET_STAT 60793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler#ifdef GET_STAT 61793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler const char* data_name = ((CvFileNode*)cvGetSeqElem( dataSetNames, testCaseIdx ))->data.str.ptr; 62793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler printf("%s, %s ", name, data_name); 63793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler const int icount = 100; 64793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler float res[icount]; 65793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler for (int k = 0; k < icount; k++) 66793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler { 67793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler#endif 68793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler data->shuffleTrainTest(); 69793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler code = train( testCaseIdx ); 70793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler#ifdef GET_STAT 71793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler float case_result = get_error(); 72793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler 73793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler res[k] = case_result; 74793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler } 75793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler float mean = 0, sigma = 0; 76793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler for (int k = 0; k < icount; k++) 77793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler { 78793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler mean += res[k]; 79793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler } 80793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler mean = mean /icount; 81793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler for (int k = 0; k < icount; k++) 82793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler { 83793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler sigma += (res[k] - mean)*(res[k] - mean); 84793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler } 85793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler sigma = sqrt(sigma/icount); 86793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler printf("%f, %f\n", mean, sigma); 87793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler#endif 88793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler } 89793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler return code; 90793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler} 91793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler 92793ee12c6df9cad3806238d32528c49a3ff9331dNoah Preslerint CV_AMLTest::validate_test_results( int testCaseIdx ) 93793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler{ 94793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler int iters; 95793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler float mean, sigma; 96793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler // read validation params 97793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler FileNode resultNode = 98793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler validationFS.getFirstTopLevelNode()["validation"][modelName][dataSetNames[testCaseIdx]]["result"]; 99793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler resultNode["iter_count"] >> iters; 100793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler if ( iters > 0) 101793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler { 102793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler resultNode["mean"] >> mean; 103793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler resultNode["sigma"] >> sigma; 104793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler model->save(format("/Users/vp/tmp/dtree/testcase_%02d.cur.yml", testCaseIdx)); 105793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler float curErr = get_test_error( testCaseIdx ); 106793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler const int coeff = 4; 107793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler ts->printf( cvtest::TS::LOG, "Test case = %d; test error = %f; mean error = %f (diff=%f), %d*sigma = %f\n", 108793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler testCaseIdx, curErr, mean, abs( curErr - mean), coeff, coeff*sigma ); 109793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler if ( abs( curErr - mean) > coeff*sigma ) 110793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler { 111793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler ts->printf( cvtest::TS::LOG, "abs(%f - %f) > %f - OUT OF RANGE!\n", curErr, mean, coeff*sigma, coeff ); 112793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler return cvtest::TS::FAIL_BAD_ACCURACY; 113793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler } 114793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler else 115793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler ts->printf( cvtest::TS::LOG, ".\n" ); 116793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler 117793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler } 118793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler else 119793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler { 120793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler ts->printf( cvtest::TS::LOG, "validation info is not suitable" ); 121793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler return cvtest::TS::FAIL_INVALID_TEST_DATA; 122793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler } 123793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler return cvtest::TS::OK; 124793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler} 125793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler 126793ee12c6df9cad3806238d32528c49a3ff9331dNoah PreslerTEST(ML_DTree, regression) { CV_AMLTest test( CV_DTREE ); test.safe_run(); } 127793ee12c6df9cad3806238d32528c49a3ff9331dNoah PreslerTEST(ML_Boost, regression) { CV_AMLTest test( CV_BOOST ); test.safe_run(); } 128793ee12c6df9cad3806238d32528c49a3ff9331dNoah PreslerTEST(ML_RTrees, regression) { CV_AMLTest test( CV_RTREES ); test.safe_run(); } 129793ee12c6df9cad3806238d32528c49a3ff9331dNoah PreslerTEST(DISABLED_ML_ERTrees, regression) { CV_AMLTest test( CV_ERTREES ); test.safe_run(); } 130793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler 131793ee12c6df9cad3806238d32528c49a3ff9331dNoah Presler/* End of file. */ 132